From 2f81029e096bd0d1214ec4a74ce3ce6731f5ba30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20M=C3=BCller?= Date: Sun, 9 Jun 2024 12:17:30 +0200 Subject: [PATCH 1/8] Add initial code for revision cleanup --- src/node/types/Revision.ts | 9 ++ src/node/utils/Cleanup.ts | 185 +++++++++++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+) create mode 100644 src/node/types/Revision.ts create mode 100644 src/node/utils/Cleanup.ts diff --git a/src/node/types/Revision.ts b/src/node/types/Revision.ts new file mode 100644 index 00000000000..8a9d65e29cf --- /dev/null +++ b/src/node/types/Revision.ts @@ -0,0 +1,9 @@ +import {AChangeSet} from "./PadType"; + +export type Revision = { + changeset: AChangeSet, + meta: { + author: string, + timestamp: number, + } +} diff --git a/src/node/utils/Cleanup.ts b/src/node/utils/Cleanup.ts new file mode 100644 index 00000000000..5bc5176e79d --- /dev/null +++ b/src/node/utils/Cleanup.ts @@ -0,0 +1,185 @@ +'use strict' + +import {AChangeSet, PadType} from "../types/PadType"; +import {MapArrayType} from "../types/MapType"; +import {ChangeSet} from "../types/ChangeSet"; +import {Revision} from "../types/Revision"; + +const promises = require('./promises'); +const AttributePool = require('ep_etherpad-lite/static/js/AttributePool'); + +const padManager = require('ep_etherpad-lite/node/db/PadManager'); +const db = require('ep_etherpad-lite/node/db/DB'); +const Changeset = require('ep_etherpad-lite/static/js/Changeset'); +const padMessageHandler = require('ep_etherpad-lite/node/handler/PadMessageHandler'); + + +const composePadChangesets = async (pad: PadType, startNum: number, endNum: number) => { + // fetch all changesets we need + const headNum = pad.getHeadRevisionNumber(); + endNum = Math.min(endNum, headNum + 1); + startNum = Math.max(startNum, 0); + + // create an array for all changesets, we will + // replace the values with the changeset later + const changesetsNeeded = []; + for (let r = startNum; r < endNum; r++) { + changesetsNeeded.push(r); + } + + // get all changesets + const changesets: MapArrayType = {}; + await Promise.all(changesetsNeeded.map( + (revNum) => pad.getRevisionChangeset(revNum) + .then((changeset) => changesets[revNum] = changeset))); + + // compose Changesets + let r; + try { + let changeset = changesets[startNum]; + const pool = pad.apool(); + + for (r = startNum + 1; r < endNum; r++) { + const cs = changesets[r]; + changeset = Changeset.compose(changeset, cs, pool); + } + return changeset; + } catch (e) { + // r-1 indicates the rev that was build starting with startNum, applying startNum+1, +2, +3 + throw e; + } +}; + +exports.deleteAllRevisions = async (padID: string): Promise => { + + const randomPadId = padID + 'aertdfdf' + Math.random().toString(10) + + let pad = await padManager.getPad(padID); + await pad.copyPadWithoutHistory(randomPadId, false); + pad = await padManager.getPad(randomPadId); + await pad.copyPadWithoutHistory(padID, true); + await pad.remove(); +} + +const createRevision = async (aChangeset: AChangeSet, timestamp: number, isKeyRev: boolean, authorId = '', atext: any = null, pool: any = null) => { + + if (authorId !== '') pool.putAttrib(['author', authorId]); + + return { + changeset: aChangeset, + meta: { + author: authorId, + timestamp: timestamp, + ...isKeyRev ? { + pool: pool, + atext: atext, + } : {}, + }, + }; +} + +exports.deleteRevisions = async (padId: string, keepRevisions: number): Promise => { + + let pad = await padManager.getPad(padId); + pad.check() + + console.log('Initial pad is valid') + + padMessageHandler.kickSessionsFromPad(padId) + + const cleanupUntilRevision = pad.head - keepRevisions + console.log('Composing changesets: ', cleanupUntilRevision) + const changeset = await composePadChangesets(pad, 0, cleanupUntilRevision + 1) + + const revisions: Revision[] = []; + + for (let rev = 0; rev <= pad.head; ++rev) { + revisions[rev] = await pad.getRevision(rev) + } + + console.log('Loaded revisions: ', revisions.length) + + await promises.timesLimit(cleanupUntilRevision, 500, async (i: string) => { + console.log('Delete revision: ', i) + await db.remove(`pad:${padId}:revs:${i}`, null); + }); + + let padContent = await db.get(`pad:${padId}`) + padContent.head = keepRevisions + await db.set(`pad:${padId}`, padContent); + + let newAText = Changeset.makeAText('\n'); + let newPool = new AttributePool() + + for (let rev = 0; rev <= cleanupUntilRevision; ++rev) { + newAText = Changeset.applyToAText(revisions[rev].changeset, newAText, newPool); + } + + const revision = await createRevision( + changeset, + revisions[cleanupUntilRevision].meta.timestamp, + 0 === pad.getKeyRevisionNumber(0), + '', + newAText, + newPool + ); + console.log('Create revision 0: ', revision); + + const p: Promise[] = []; + + p.push(db.set(`pad:${padId}:revs:0`, revision)) + + p.push(promises.timesLimit(keepRevisions, 500, async (i: number) => { + const rev = i + cleanupUntilRevision + 1 + const newRev = rev - cleanupUntilRevision; + console.log('Map revision: ' + rev + ' => ' + newRev) + + newAText = Changeset.applyToAText(revisions[rev].changeset, newAText, newPool); + + const revision = await createRevision( + revisions[rev].changeset, + revisions[rev].meta.timestamp, + newRev === pad.getKeyRevisionNumber(newRev), + revisions[rev].meta.author, + newAText, + newPool + ); + console.log('Create revision: ', newRev, revision); + + await db.set(`pad:${padId}:revs:${newRev}`, revision); + })); + + await Promise.all(p) + + console.log('Finished migration. Checking pad now') + + + padManager.unloadPad(padId); + + let newPad = await padManager.getPad(padId); + newPad.check(); +} + +exports.checkTodos = async () => { + await new Promise(resolve => setTimeout(resolve, 5000)); + + await Promise.all((await padManager.listAllPads()).padIDs.map(async (padId: string) => { + const pad = await padManager.getPad(padId); + + console.log('pad user count', padId, padMessageHandler.padUsersCount(padId)) + const revisionDate = await pad.getRevisionDate(pad.getHeadRevisionNumber()) + console.log('pad last modified', padId, Date.now() - revisionDate) + + if (pad.head < 10000 || padMessageHandler.padUsersCount(padId) > 0 || Date.now() < revisionDate + 1000 * 60 * 60 * 24) { + return + } + + try { + await exports.deleteRevisions(padId, 100) + console.log('successful cleaned up pad: ', padId) + } catch (err: any) { + console.error(`Error in pad ${padId}: ${err.stack || err}`); + return; + } + })); +} From 826f57c28df1aae629862ff509277d11e6fa3bfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20M=C3=BCller?= Date: Fri, 16 Aug 2024 21:53:43 +0200 Subject: [PATCH 2/8] Some improvements - code cleanup --- src/node/handler/PadMessageHandler.ts | 4 +-- src/node/utils/Cleanup.ts | 47 +++------------------------ 2 files changed, 6 insertions(+), 45 deletions(-) diff --git a/src/node/handler/PadMessageHandler.ts b/src/node/handler/PadMessageHandler.ts index 3909496075d..e5a92da19f4 100644 --- a/src/node/handler/PadMessageHandler.ts +++ b/src/node/handler/PadMessageHandler.ts @@ -1142,7 +1142,7 @@ const getChangesetInfo = async (pad: PadType, startNum: number, endNum:number, g getPadLines(pad, startNum - 1), // Get all needed composite Changesets. ...compositesChangesetNeeded.map(async (item) => { - const changeset = await composePadChangesets(pad, item.start, item.end); + const changeset = await exports.composePadChangesets(pad, item.start, item.end); composedChangesets[`${item.start}/${item.end}`] = changeset; }), // Get all needed revision Dates. @@ -1208,7 +1208,7 @@ const getPadLines = async (pad: PadType, revNum: number) => { * Tries to rebuild the composePadChangeset function of the original Etherpad * https://github.com/ether/pad/blob/master/etherpad/src/etherpad/control/pad/pad_changeset_control.js#L241 */ -const composePadChangesets = async (pad: PadType, startNum: number, endNum: number) => { +exports.composePadChangesets = async (pad: PadType, startNum: number, endNum: number) => { // fetch all changesets we need const headNum = pad.getHeadRevisionNumber(); endNum = Math.min(endNum, headNum + 1); diff --git a/src/node/utils/Cleanup.ts b/src/node/utils/Cleanup.ts index 5bc5176e79d..3bcfd9ef9fe 100644 --- a/src/node/utils/Cleanup.ts +++ b/src/node/utils/Cleanup.ts @@ -1,8 +1,6 @@ 'use strict' -import {AChangeSet, PadType} from "../types/PadType"; -import {MapArrayType} from "../types/MapType"; -import {ChangeSet} from "../types/ChangeSet"; +import {AChangeSet} from "../types/PadType"; import {Revision} from "../types/Revision"; const promises = require('./promises'); @@ -13,43 +11,6 @@ const db = require('ep_etherpad-lite/node/db/DB'); const Changeset = require('ep_etherpad-lite/static/js/Changeset'); const padMessageHandler = require('ep_etherpad-lite/node/handler/PadMessageHandler'); - -const composePadChangesets = async (pad: PadType, startNum: number, endNum: number) => { - // fetch all changesets we need - const headNum = pad.getHeadRevisionNumber(); - endNum = Math.min(endNum, headNum + 1); - startNum = Math.max(startNum, 0); - - // create an array for all changesets, we will - // replace the values with the changeset later - const changesetsNeeded = []; - for (let r = startNum; r < endNum; r++) { - changesetsNeeded.push(r); - } - - // get all changesets - const changesets: MapArrayType = {}; - await Promise.all(changesetsNeeded.map( - (revNum) => pad.getRevisionChangeset(revNum) - .then((changeset) => changesets[revNum] = changeset))); - - // compose Changesets - let r; - try { - let changeset = changesets[startNum]; - const pool = pad.apool(); - - for (r = startNum + 1; r < endNum; r++) { - const cs = changesets[r]; - changeset = Changeset.compose(changeset, cs, pool); - } - return changeset; - } catch (e) { - // r-1 indicates the rev that was build starting with startNum, applying startNum+1, +2, +3 - throw e; - } -}; - exports.deleteAllRevisions = async (padID: string): Promise => { const randomPadId = padID + 'aertdfdf' + Math.random().toString(10) @@ -61,7 +22,7 @@ exports.deleteAllRevisions = async (padID: string): Promise => { await pad.remove(); } -const createRevision = async (aChangeset: AChangeSet, timestamp: number, isKeyRev: boolean, authorId = '', atext: any = null, pool: any = null) => { +const createRevision = async (aChangeset: AChangeSet, timestamp: number, isKeyRev: boolean, authorId: string, atext: any, pool: any) => { if (authorId !== '') pool.putAttrib(['author', authorId]); @@ -81,7 +42,7 @@ const createRevision = async (aChangeset: AChangeSet, timestamp: number, isKeyRe exports.deleteRevisions = async (padId: string, keepRevisions: number): Promise => { let pad = await padManager.getPad(padId); - pad.check() + await pad.check() console.log('Initial pad is valid') @@ -89,7 +50,7 @@ exports.deleteRevisions = async (padId: string, keepRevisions: number): Promise< const cleanupUntilRevision = pad.head - keepRevisions console.log('Composing changesets: ', cleanupUntilRevision) - const changeset = await composePadChangesets(pad, 0, cleanupUntilRevision + 1) + const changeset = await padMessageHandler.composePadChangesets(pad, 0, cleanupUntilRevision + 1) const revisions: Revision[] = []; From e4df2ac6afa70e5536ce955fd492caeb66fcba32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20M=C3=BCller?= Date: Thu, 5 Sep 2024 20:09:15 +0200 Subject: [PATCH 3/8] Cleanup logging --- src/node/utils/Cleanup.ts | 51 +++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/src/node/utils/Cleanup.ts b/src/node/utils/Cleanup.ts index 3bcfd9ef9fe..fcff58d9bc2 100644 --- a/src/node/utils/Cleanup.ts +++ b/src/node/utils/Cleanup.ts @@ -4,12 +4,12 @@ import {AChangeSet} from "../types/PadType"; import {Revision} from "../types/Revision"; const promises = require('./promises'); -const AttributePool = require('ep_etherpad-lite/static/js/AttributePool'); - const padManager = require('ep_etherpad-lite/node/db/PadManager'); const db = require('ep_etherpad-lite/node/db/DB'); const Changeset = require('ep_etherpad-lite/static/js/Changeset'); const padMessageHandler = require('ep_etherpad-lite/node/handler/PadMessageHandler'); +const log4js = require('log4js'); +const logger = log4js.getLogger('cleanup'); exports.deleteAllRevisions = async (padID: string): Promise => { @@ -41,15 +41,17 @@ const createRevision = async (aChangeset: AChangeSet, timestamp: number, isKeyRe exports.deleteRevisions = async (padId: string, keepRevisions: number): Promise => { + logger.debug('Start cleanup revisions', padId) + let pad = await padManager.getPad(padId); await pad.check() - console.log('Initial pad is valid') + logger.debug('Initial pad is valid') padMessageHandler.kickSessionsFromPad(padId) const cleanupUntilRevision = pad.head - keepRevisions - console.log('Composing changesets: ', cleanupUntilRevision) + logger.debug('Composing changesets: ', cleanupUntilRevision) const changeset = await padMessageHandler.composePadChangesets(pad, 0, cleanupUntilRevision + 1) const revisions: Revision[] = []; @@ -58,10 +60,9 @@ exports.deleteRevisions = async (padId: string, keepRevisions: number): Promise< revisions[rev] = await pad.getRevision(rev) } - console.log('Loaded revisions: ', revisions.length) + logger.debug('Loaded revisions: ', revisions.length) - await promises.timesLimit(cleanupUntilRevision, 500, async (i: string) => { - console.log('Delete revision: ', i) + await promises.timesLimit(pad.head + 1, 500, async (i: string) => { await db.remove(`pad:${padId}:revs:${i}`, null); }); @@ -70,10 +71,10 @@ exports.deleteRevisions = async (padId: string, keepRevisions: number): Promise< await db.set(`pad:${padId}`, padContent); let newAText = Changeset.makeAText('\n'); - let newPool = new AttributePool() + let pool = pad.apool() for (let rev = 0; rev <= cleanupUntilRevision; ++rev) { - newAText = Changeset.applyToAText(revisions[rev].changeset, newAText, newPool); + newAText = Changeset.applyToAText(revisions[rev].changeset, newAText, pool); } const revision = await createRevision( @@ -82,9 +83,8 @@ exports.deleteRevisions = async (padId: string, keepRevisions: number): Promise< 0 === pad.getKeyRevisionNumber(0), '', newAText, - newPool + pool ); - console.log('Create revision 0: ', revision); const p: Promise[] = []; @@ -93,9 +93,8 @@ exports.deleteRevisions = async (padId: string, keepRevisions: number): Promise< p.push(promises.timesLimit(keepRevisions, 500, async (i: number) => { const rev = i + cleanupUntilRevision + 1 const newRev = rev - cleanupUntilRevision; - console.log('Map revision: ' + rev + ' => ' + newRev) - newAText = Changeset.applyToAText(revisions[rev].changeset, newAText, newPool); + newAText = Changeset.applyToAText(revisions[rev].changeset, newAText, pool); const revision = await createRevision( revisions[rev].changeset, @@ -103,43 +102,47 @@ exports.deleteRevisions = async (padId: string, keepRevisions: number): Promise< newRev === pad.getKeyRevisionNumber(newRev), revisions[rev].meta.author, newAText, - newPool + pool ); - console.log('Create revision: ', newRev, revision); await db.set(`pad:${padId}:revs:${newRev}`, revision); })); await Promise.all(p) - console.log('Finished migration. Checking pad now') - + logger.debug('Finished migration. Checking pad now') padManager.unloadPad(padId); let newPad = await padManager.getPad(padId); - newPad.check(); + await newPad.check(); } exports.checkTodos = async () => { await new Promise(resolve => setTimeout(resolve, 5000)); + // TODO: Move to settings + const settings = { + minHead: 100, + keepRevisions: 100, + minAge: 1,//1000 * 60 * 60 * 24, + } + await Promise.all((await padManager.listAllPads()).padIDs.map(async (padId: string) => { + // TODO: Handle concurrency const pad = await padManager.getPad(padId); - console.log('pad user count', padId, padMessageHandler.padUsersCount(padId)) const revisionDate = await pad.getRevisionDate(pad.getHeadRevisionNumber()) - console.log('pad last modified', padId, Date.now() - revisionDate) - if (pad.head < 10000 || padMessageHandler.padUsersCount(padId) > 0 || Date.now() < revisionDate + 1000 * 60 * 60 * 24) { + if (pad.head < settings.minHead || padMessageHandler.padUsersCount(padId) > 0 || Date.now() < revisionDate + settings.minAge) { return } try { - await exports.deleteRevisions(padId, 100) - console.log('successful cleaned up pad: ', padId) + await exports.deleteRevisions(padId, settings.keepRevisions) + logger.info('successful cleaned up pad: ', padId) } catch (err: any) { - console.error(`Error in pad ${padId}: ${err.stack || err}`); + logger.error(`Error in pad ${padId}: ${err.stack || err}`); return; } })); From 593d1021b60caf30506e9c3c5a29d84d83c4e282 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20M=C3=BCller?= Date: Tue, 10 Sep 2024 16:55:14 +0200 Subject: [PATCH 4/8] Add button in admin backend to cleanup revisions of a specific pad --- admin/public/ep_admin_pads/de.json | 1 + admin/public/ep_admin_pads/en.json | 1 + admin/src/pages/PadPage.tsx | 23 ++++++++++++++++++++++- src/node/hooks/express/adminsettings.ts | 21 +++++++++++++++++++++ src/node/utils/Cleanup.ts | 15 ++++++++++++--- src/node/utils/Settings.ts | 7 +++++++ 6 files changed, 64 insertions(+), 4 deletions(-) diff --git a/admin/public/ep_admin_pads/de.json b/admin/public/ep_admin_pads/de.json index afb553caf46..67dd73ddf07 100644 --- a/admin/public/ep_admin_pads/de.json +++ b/admin/public/ep_admin_pads/de.json @@ -14,6 +14,7 @@ "ep_adminpads2_autoupdate.title": "Aktiviert oder deaktiviert automatische Aktualisierungen für die aktuelle Abfrage.", "ep_adminpads2_confirm": "Willst du das Pad {{padID}} wirklich löschen?", "ep_adminpads2_delete.value": "Löschen", + "ep_adminpads2_cleanup": "Historie aufräumen", "ep_adminpads2_last-edited": "Zuletzt bearbeitet", "ep_adminpads2_loading": "Lädt...", "ep_adminpads2_manage-pads": "Pads verwalten", diff --git a/admin/public/ep_admin_pads/en.json b/admin/public/ep_admin_pads/en.json index 8a9044b1b84..76354c6403f 100644 --- a/admin/public/ep_admin_pads/en.json +++ b/admin/public/ep_admin_pads/en.json @@ -4,6 +4,7 @@ "ep_adminpads2_autoupdate.title": "Enables or disables automatic updates for the current query.", "ep_adminpads2_confirm": "Do you really want to delete the pad {{padID}}?", "ep_adminpads2_delete.value": "Delete", + "ep_adminpads2_cleanup": "Cleanup revisions", "ep_adminpads2_last-edited": "Last edited", "ep_adminpads2_loading": "Loading…", "ep_adminpads2_manage-pads": "Manage pads", diff --git a/admin/src/pages/PadPage.tsx b/admin/src/pages/PadPage.tsx index e663603cdd9..595dd9376ae 100644 --- a/admin/src/pages/PadPage.tsx +++ b/admin/src/pages/PadPage.tsx @@ -6,7 +6,7 @@ import {useDebounce} from "../utils/useDebounce.ts"; import {determineSorting} from "../utils/sorting.ts"; import * as Dialog from "@radix-ui/react-dialog"; import {IconButton} from "../components/IconButton.tsx"; -import {ChevronLeft, ChevronRight, Eye, Trash2} from "lucide-react"; +import {ChevronLeft, ChevronRight, Eye, Trash2, FileStack} from "lucide-react"; import {SearchField} from "../components/SearchField.tsx"; export const PadPage = ()=>{ @@ -68,12 +68,30 @@ export const PadPage = ()=>{ results: newPads }) }) + + settingsSocket.on('results:cleanupPadRevisions', (data)=>{ + let newPads = useStore.getState().pads?.results ?? [] + + newPads.forEach((pad)=>{ + if (pad.padName === data.padId) { + pad.revisionNumber = data.keepRevisions + } + }) + + useStore.getState().setPads({ + results: newPads, + total: useStore.getState().pads!.total + }) + }) }, [settingsSocket, pads]); const deletePad = (padID: string)=>{ settingsSocket?.emit('deletePad', padID) } + const cleanupPad = (padID: string)=>{ + settingsSocket?.emit('cleanupPadRevisions', padID) + } return
@@ -150,6 +168,9 @@ export const PadPage = ()=>{ setPadToDelete(pad.padName) setDeleteDialog(true) }}/> + } title={} onClick={()=>{ + cleanupPad(pad.padName) + }}/> } title="view" onClick={()=>window.open(`/p/${pad.padName}`, '_blank')}/>
diff --git a/src/node/hooks/express/adminsettings.ts b/src/node/hooks/express/adminsettings.ts index 63d901f2126..3d30e8fb5ab 100644 --- a/src/node/hooks/express/adminsettings.ts +++ b/src/node/hooks/express/adminsettings.ts @@ -13,6 +13,7 @@ const settings = require('../../utils/Settings'); const UpdateCheck = require('../../utils/UpdateCheck'); const padManager = require('../../db/PadManager'); const api = require('../../db/API'); +const cleanup = require('../../utils/Cleanup'); const queryPadLimit = 12; @@ -252,6 +253,26 @@ exports.socketio = (hookName: string, {io}: any) => { } }) + socket.on('cleanupPadRevisions', async (padId: string) => { + const padExists = await padManager.doesPadExists(padId); + if (padExists) { + logger.info(`Cleanup pad revisions: ${padId}`); + try { + const result = await cleanup.deleteRevisions(padId, settings.cleanup.keepRevisions) + if (result) { + socket.emit('results:cleanupPadRevisions', { + padId: padId, + keepRevisions: settings.cleanup.keepRevisions, + }); + logger.info('successful cleaned up pad: ', padId) + } + } catch (err: any) { + logger.error(`Error in pad ${padId}: ${err.stack || err}`); + return; + } + } + }) + socket.on('restartServer', async () => { logger.info('Admin request to restart server through a socket on /admin/settings'); settings.reloadSettings(); diff --git a/src/node/utils/Cleanup.ts b/src/node/utils/Cleanup.ts index fcff58d9bc2..d0fa0db7d0c 100644 --- a/src/node/utils/Cleanup.ts +++ b/src/node/utils/Cleanup.ts @@ -39,7 +39,7 @@ const createRevision = async (aChangeset: AChangeSet, timestamp: number, isKeyRe }; } -exports.deleteRevisions = async (padId: string, keepRevisions: number): Promise => { +exports.deleteRevisions = async (padId: string, keepRevisions: number): Promise => { logger.debug('Start cleanup revisions', padId) @@ -48,6 +48,11 @@ exports.deleteRevisions = async (padId: string, keepRevisions: number): Promise< logger.debug('Initial pad is valid') + if (pad.head < keepRevisions) { + logger.debug('Pad has not enough revisions') + return false + } + padMessageHandler.kickSessionsFromPad(padId) const cleanupUntilRevision = pad.head - keepRevisions @@ -116,6 +121,8 @@ exports.deleteRevisions = async (padId: string, keepRevisions: number): Promise< let newPad = await padManager.getPad(padId); await newPad.check(); + + return true } exports.checkTodos = async () => { @@ -139,8 +146,10 @@ exports.checkTodos = async () => { } try { - await exports.deleteRevisions(padId, settings.keepRevisions) - logger.info('successful cleaned up pad: ', padId) + const result = await exports.deleteRevisions(padId, settings.keepRevisions) + if (result) { + logger.info('successful cleaned up pad: ', padId) + } } catch (err: any) { logger.error(`Error in pad ${padId}: ${err.stack || err}`); return; diff --git a/src/node/utils/Settings.ts b/src/node/utils/Settings.ts index 4ff117ad367..535ccbd8ce7 100644 --- a/src/node/utils/Settings.ts +++ b/src/node/utils/Settings.ts @@ -380,6 +380,13 @@ exports.sso = { */ exports.showSettingsInAdminPage = true; +/* + * Settings for cleanup of pads + */ +exports.cleanup = { + keepRevisions: 100, +} + /* * By default, when caret is moved out of viewport, it scrolls the minimum * height needed to make this line visible. From 3330ed8d1b344ff722b4ab673386a9acf7fe3703 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20M=C3=BCller?= Date: Tue, 10 Sep 2024 17:25:51 +0200 Subject: [PATCH 5/8] Disable cleanup by default and show errors in admin area --- admin/src/pages/PadPage.tsx | 21 +++++++++++++++++++++ settings.json.docker | 8 ++++++++ settings.json.template | 8 ++++++++ src/node/hooks/express/adminsettings.ts | 14 ++++++++++++++ src/node/utils/Cleanup.ts | 2 +- src/node/utils/Settings.ts | 1 + 6 files changed, 53 insertions(+), 1 deletion(-) diff --git a/admin/src/pages/PadPage.tsx b/admin/src/pages/PadPage.tsx index 595dd9376ae..b5db854f567 100644 --- a/admin/src/pages/PadPage.tsx +++ b/admin/src/pages/PadPage.tsx @@ -23,6 +23,7 @@ export const PadPage = ()=>{ const pads = useStore(state=>state.pads) const [currentPage, setCurrentPage] = useState(0) const [deleteDialog, setDeleteDialog] = useState(false) + const [errorText, setErrorText] = useState(null) const [padToDelete, setPadToDelete] = useState('') const pages = useMemo(()=>{ if(!pads){ @@ -72,6 +73,11 @@ export const PadPage = ()=>{ settingsSocket.on('results:cleanupPadRevisions', (data)=>{ let newPads = useStore.getState().pads?.results ?? [] + if (data.error) { + setErrorText(data.error) + return + } + newPads.forEach((pad)=>{ if (pad.padName === data.padId) { pad.revisionNumber = data.keepRevisions @@ -118,6 +124,21 @@ export const PadPage = ()=>{ + + + + +
+
Error occured: {errorText}
+
+ +
+
+ + +

setSearchTerm(v.target.value)} placeholder={t('ep_admin_pads:ep_adminpads2_search-heading')}/> diff --git a/settings.json.docker b/settings.json.docker index 109f36bfd6c..da1d51c13a8 100644 --- a/settings.json.docker +++ b/settings.json.docker @@ -171,6 +171,14 @@ */ "showSettingsInAdminPage": "${SHOW_SETTINGS_IN_ADMIN_PAGE:true}", + /* + * Settings for cleanup of pads + */ + "cleanup": { + "enabled": false, + "keepRevisions": 5 + }, + /* The authentication method used by the server. The default value is sso diff --git a/settings.json.template b/settings.json.template index fe10e51b9cc..2d856f42e4c 100644 --- a/settings.json.template +++ b/settings.json.template @@ -162,6 +162,14 @@ */ "showSettingsInAdminPage": true, + /* + * Settings for cleanup of pads + */ + "cleanup": { + "enabled": false, + "keepRevisions": 5 + }, + /* * Node native SSL support * diff --git a/src/node/hooks/express/adminsettings.ts b/src/node/hooks/express/adminsettings.ts index 3d30e8fb5ab..4c60a05ad20 100644 --- a/src/node/hooks/express/adminsettings.ts +++ b/src/node/hooks/express/adminsettings.ts @@ -254,6 +254,13 @@ exports.socketio = (hookName: string, {io}: any) => { }) socket.on('cleanupPadRevisions', async (padId: string) => { + if (!settings.cleanup.enabled) { + socket.emit('results:cleanupPadRevisions', { + error: 'Cleanup disabled. Enable cleanup in settings.json: cleanup.enabled => true', + }); + return; + } + const padExists = await padManager.doesPadExists(padId); if (padExists) { logger.info(`Cleanup pad revisions: ${padId}`); @@ -265,9 +272,16 @@ exports.socketio = (hookName: string, {io}: any) => { keepRevisions: settings.cleanup.keepRevisions, }); logger.info('successful cleaned up pad: ', padId) + } else { + socket.emit('results:cleanupPadRevisions', { + error: 'Error cleaning up pad', + }); } } catch (err: any) { logger.error(`Error in pad ${padId}: ${err.stack || err}`); + socket.emit('results:cleanupPadRevisions', { + error: err.toString(), + }); return; } } diff --git a/src/node/utils/Cleanup.ts b/src/node/utils/Cleanup.ts index d0fa0db7d0c..06361d5db8a 100644 --- a/src/node/utils/Cleanup.ts +++ b/src/node/utils/Cleanup.ts @@ -48,7 +48,7 @@ exports.deleteRevisions = async (padId: string, keepRevisions: number): Promise< logger.debug('Initial pad is valid') - if (pad.head < keepRevisions) { + if (pad.head <= keepRevisions) { logger.debug('Pad has not enough revisions') return false } diff --git a/src/node/utils/Settings.ts b/src/node/utils/Settings.ts index 535ccbd8ce7..4d7b421e1c8 100644 --- a/src/node/utils/Settings.ts +++ b/src/node/utils/Settings.ts @@ -384,6 +384,7 @@ exports.showSettingsInAdminPage = true; * Settings for cleanup of pads */ exports.cleanup = { + enabled: false, keepRevisions: 100, } From 967d430c326c720711c69908879fa998b0e60395 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20M=C3=BCller?= Date: Tue, 10 Sep 2024 23:07:57 +0200 Subject: [PATCH 6/8] Improve cleanup code --- src/node/utils/Cleanup.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/node/utils/Cleanup.ts b/src/node/utils/Cleanup.ts index 06361d5db8a..4dd52fbca9a 100644 --- a/src/node/utils/Cleanup.ts +++ b/src/node/utils/Cleanup.ts @@ -61,7 +61,7 @@ exports.deleteRevisions = async (padId: string, keepRevisions: number): Promise< const revisions: Revision[] = []; - for (let rev = 0; rev <= pad.head; ++rev) { + for (let rev = cleanupUntilRevision; rev <= pad.head; ++rev) { revisions[rev] = await pad.getRevision(rev) } @@ -78,9 +78,7 @@ exports.deleteRevisions = async (padId: string, keepRevisions: number): Promise< let newAText = Changeset.makeAText('\n'); let pool = pad.apool() - for (let rev = 0; rev <= cleanupUntilRevision; ++rev) { - newAText = Changeset.applyToAText(revisions[rev].changeset, newAText, pool); - } + newAText = Changeset.applyToAText(changeset, newAText, pool); const revision = await createRevision( changeset, From 1f80a3246a572c47e055a3d88ce06edc6ca174e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20M=C3=BCller?= Date: Tue, 10 Sep 2024 23:19:33 +0200 Subject: [PATCH 7/8] Load revisions for cleanup in parallel --- src/node/utils/Cleanup.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/node/utils/Cleanup.ts b/src/node/utils/Cleanup.ts index 4dd52fbca9a..31d2ec5f512 100644 --- a/src/node/utils/Cleanup.ts +++ b/src/node/utils/Cleanup.ts @@ -61,9 +61,10 @@ exports.deleteRevisions = async (padId: string, keepRevisions: number): Promise< const revisions: Revision[] = []; - for (let rev = cleanupUntilRevision; rev <= pad.head; ++rev) { + await promises.timesLimit(keepRevisions + 1, 500, async (i: number) => { + const rev = i + cleanupUntilRevision revisions[rev] = await pad.getRevision(rev) - } + }); logger.debug('Loaded revisions: ', revisions.length) From 2f280fb4818999337de1a555d6e8913f3c7eef57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20M=C3=BCller?= Date: Sat, 14 Sep 2024 13:43:52 +0200 Subject: [PATCH 8/8] Consider saved revisions during pad cleanup --- src/node/utils/Cleanup.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/node/utils/Cleanup.ts b/src/node/utils/Cleanup.ts index 31d2ec5f512..7e480020dda 100644 --- a/src/node/utils/Cleanup.ts +++ b/src/node/utils/Cleanup.ts @@ -74,6 +74,17 @@ exports.deleteRevisions = async (padId: string, keepRevisions: number): Promise< let padContent = await db.get(`pad:${padId}`) padContent.head = keepRevisions + if (padContent.savedRevisions) { + let newSavedRevisions = [] + + for (let i = 0; i < padContent.savedRevisions.length; i++) { + if (padContent.savedRevisions[i].revNum > cleanupUntilRevision) { + padContent.savedRevisions[i].revNum = padContent.savedRevisions[i].revNum - cleanupUntilRevision + newSavedRevisions.push(padContent.savedRevisions[i]) + } + } + padContent.savedRevisions = newSavedRevisions + } await db.set(`pad:${padId}`, padContent); let newAText = Changeset.makeAText('\n');