From b71c69e81dcb92abeff3aaab8236cc773f7a83a8 Mon Sep 17 00:00:00 2001 From: PTR <47147771+PTR-inc@users.noreply.github.com> Date: Sun, 10 Nov 2024 14:49:11 +0100 Subject: [PATCH] Fix autobackup defaults and zip level for performance (#6518) * Fix autobackup defaults and zip level for performance * Add zipcompression configuration option --- db.js | 28 ++++++++++------------------ meshcentral-config-schema.json | 5 +++++ meshcentral.js | 6 +++--- meshuser.js | 3 +-- 4 files changed, 19 insertions(+), 23 deletions(-) diff --git a/db.js b/db.js index 01958a2f..0d27dd68 100644 --- a/db.js +++ b/db.js @@ -3434,7 +3434,8 @@ module.exports.CreateDB = function (parent, func) { obj.performBackup = function (func) { parent.debug('db','Entering performBackup'); try { - if (obj.performingBackup) return 1; + if (obj.performingBackup) return 'Backup alreay in progress.'; + if (parent.config.settings.autobackup.backupintervalhours == -1) { if (func) { func('Unable to create backup if backuppath is set to the data folder.'); return 'Backup aborted.' }}; obj.performingBackup = true; let backupPath = parent.backuppath; let dataPath = parent.datapath; @@ -3446,11 +3447,10 @@ module.exports.CreateDB = function (parent, func) { obj.newAutoBackupFile = path.join(backupPath, ((typeof parent.config.settings.autobackup.backupname == 'string') ? parent.config.settings.autobackup.backupname : 'meshcentral-autobackup-') + fileSuffix + '.zip'); if ((obj.databaseType == DB_MONGOJS) || (obj.databaseType == DB_MONGODB)) { - // Perform a MongoDump in the datadir + // Perform a MongoDump const dbname = (parent.args.mongodbname) ? (parent.args.mongodbname) : 'meshcentral'; const dburl = parent.args.mongodb; - //const obj.newDBDumpFile = 'mongodump-' + fileSuffix; obj.newDBDumpFile = path.join(backupPath, (dbname + '-mongodump-' + fileSuffix + '.archive')); var cmd = buildMongoDumpCommand(); @@ -3490,16 +3490,6 @@ module.exports.CreateDB = function (parent, func) { } else if (obj.databaseType == DB_SQLITE) { //.db3 suffix to escape escape backupfile glob to exclude the sqlite db files obj.newDBDumpFile = path.join(backupPath, databaseName + '-sqlitedump-' + fileSuffix + '.db3'); - /*undocumented in node-sqlite3 API, check https://github.com/TryGhost/node-sqlite3/blob/593c9d498be2510d286349134537e3bf89401c4a/test/backup.test.js - var backup = obj.file.backup(obj.newDBDumpFile); - backup.step(-1, function (err) { - if (err) { console.log('SQLite start-backup error: ' + err); obj.backupStatus |=BACKUPFAIL_DBDUMP; obj.createBackupfile(func); }; - backup.finish(function (err) { - if (err) { console.log('SQLite backup error: ' + err); obj.backupStatus |=BACKUPFAIL_DBDUMP;}; - obj.createBackupfile(func); - }); - }); - */ // do a VACUUM INTO in favor of the backup API to compress the export, see https://www.sqlite.org/backup.html obj.file.exec('VACUUM INTO \'' + obj.newDBDumpFile + '\'', function (err) { if (err) { console.log('SQLite start-backup error: ' + err); obj.backupStatus |=BACKUPFAIL_DBDUMP;}; @@ -3529,19 +3519,21 @@ module.exports.CreateDB = function (parent, func) { obj.createBackupfile(func); } } catch (ex) { console.log(ex); }; - return(0); + return 'Starting auto-backup...'; }; obj.createBackupfile = function(func) { parent.debug('db', 'Entering createFileBackup'); let archiver = require('archiver'); let archive = null; + let zipLevel = Math.min(Math.max(Number(parent.config.settings.autobackup.zipcompression ? parent.config.settings.autobackup.zipcompression : 5),1),9); + //if password defined, create encrypted zip if (parent.config.settings.autobackup && (typeof parent.config.settings.autobackup.zippassword == 'string')) { try { //Only register format once, otherwise it triggers an error if (archiver.isRegisteredFormat('zip-encrypted') == false) { archiver.registerFormat('zip-encrypted', require('archiver-zip-encrypted')); } - archive = archiver.create('zip-encrypted', { zlib: { level: 9 }, encryptionMethod: 'aes256', password: parent.config.settings.autobackup.zippassword }); + archive = archiver.create('zip-encrypted', { zlib: { level: zipLevel }, encryptionMethod: 'aes256', password: parent.config.settings.autobackup.zippassword }); if (func) { func('Creating encrypted ZIP'); } } catch (ex) { // registering encryption failed, do not fall back to non-encrypted, fail backup and skip old backup removal as a precaution to not lose any backups obj.backupStatus |= BACKUPFAIL_ZIPMODULE; @@ -3550,7 +3542,7 @@ module.exports.CreateDB = function (parent, func) { } } else { if (func) { func('Creating a NON-ENCRYPTED ZIP'); } - archive = archiver('zip', { zlib: { level: 9 } }); + archive = archiver('zip', { zlib: { level: zipLevel } }); } //original behavior, just a filebackup if dbdump fails : (obj.backupStatus == 0 || obj.backupStatus == BACKUPFAIL_DBDUMP) @@ -3628,14 +3620,14 @@ module.exports.CreateDB = function (parent, func) { let globIgnoreFiles; //slice in case exclusion gets pushed - globIgnoreFiles = parent.config.settings.autobackup.backupignorefilesglob.slice(); + globIgnoreFiles = parent.config.settings.autobackup.backupignorefilesglob ? parent.config.settings.autobackup.backupignorefilesglob.slice() : []; if (parent.config.settings.sqlite3) { globIgnoreFiles.push (datapathFoldername + '/' + databaseName + '.sqlite*'); }; //skip sqlite database file, and temp files with ext -journal, -wal & -shm //archiver.glob doesn't seem to use the third param, archivesubdir. Bug? //workaround: go up a dir and add data dir explicitly to keep the zip tidy archive.glob((datapathFoldername + '/**'), { cwd: datapathParentPath, ignore: globIgnoreFiles, - skip: parent.config.settings.autobackup.backupskipfoldersglob + skip: (parent.config.settings.autobackup.backupskipfoldersglob ? parent.config.settings.autobackup.backupskipfoldersglob : []) }); if (parent.config.settings.autobackup.backupwebfolders) { diff --git a/meshcentral-config-schema.json b/meshcentral-config-schema.json index 7172ea57..c20bc0c3 100644 --- a/meshcentral-config-schema.json +++ b/meshcentral-config-schema.json @@ -851,6 +851,11 @@ "default": 10, "description": "How many days of backups should the autobackup keep? Default is 10 Days worth" }, + "zipCompression" : { + "type": "integer", + "default": "5", + "description": "Set the zip compression level, 1=fast/less small file to 9=slow/smallest file." + }, "zipPassword": { "type": "string", "default": "", diff --git a/meshcentral.js b/meshcentral.js index 3ee0622b..cb00e1ab 100644 --- a/meshcentral.js +++ b/meshcentral.js @@ -2089,7 +2089,7 @@ function CreateMeshCentralServer(config, args) { obj.updateServerState('state', "running"); // Setup auto-backup defaults - if (obj.config.settings.autobackup == null || obj.config.settings.autobackup == false || obj.config.settings.autobackup == 'false') { delete obj.config.settings.autobackup; } + if (obj.config.settings.autobackup == null || obj.config.settings.autobackup == false || obj.config.settings.autobackup == 'false') { obj.config.settings.autobackup = {backupintervalhours: 0}; } //no schedule, but able to console autobackup else { if (obj.config.settings.autobackup === true) {obj.config.settings.autobackup = {backupintervalhours: 24, keeplastdaysbackup: 10}; }; if (typeof obj.config.settings.autobackup.backupintervalhours != 'number') { obj.config.settings.autobackup.backupintervalhours = 24; }; @@ -2104,7 +2104,7 @@ function CreateMeshCentralServer(config, args) { // Check that autobackup path is not within the "meshcentral-data" folder. if ((typeof obj.config.settings.autobackup == 'object') && (typeof obj.config.settings.autobackup.backuppath == 'string') && (obj.path.normalize(obj.config.settings.autobackup.backuppath).startsWith(obj.path.normalize(obj.datapath)))) { addServerWarning("Backup path can't be set within meshcentral-data folder, backup settings ignored.", 21); - delete obj.config.settings.autobackup; + obj.config.settings.autobackup = {backupintervalhours: -1}; //block console autobackup } // Load Intel AMT passwords from the "amtactivation.log" file @@ -2267,7 +2267,7 @@ function CreateMeshCentralServer(config, args) { // Check if we need to perform an automatic backup function checkAutobackup() { - if (obj.config.settings.autobackup && (typeof obj.config.settings.autobackup.backupintervalhours == 'number')) { + if (obj.config.settings.autobackup.backupintervalhours >= 1) { obj.db.Get('LastAutoBackupTime', function (err, docs) { if (err != null) return; var lastBackup = 0; diff --git a/meshuser.js b/meshuser.js index 862d4706..81ebbac6 100644 --- a/meshuser.js +++ b/meshuser.js @@ -7627,10 +7627,9 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use } function serverUserCommandAutoBackup(cmdData) { - var backupResult = parent.db.performBackup(function (msg) { + cmdData.result = parent.db.performBackup(function (msg) { try { ws.send(JSON.stringify({ action: 'serverconsole', value: msg, tag: cmdData.command.tag })); } catch (ex) { } }); - if (backupResult == 0) { cmdData.result = 'Starting auto-backup...'; } else { cmdData.result = 'Backup alreay in progress.'; } } function serverUserCommandBackupConfig(cmdData) {