From 8ac8953298b774a44d2efb03e7503c1bea3336c8 Mon Sep 17 00:00:00 2001 From: Ylian Saint-Hilaire Date: Mon, 4 Jan 2021 16:26:25 -0800 Subject: [PATCH] Added new --nedbtodb to transfer all NeDB records into the database. --- db.js | 99 ++++++++++++++++++++++++++++++++++++++++++++++++-- meshcentral.js | 12 +++++- 2 files changed, 105 insertions(+), 6 deletions(-) diff --git a/db.js b/db.js index 43917239..fe440403 100644 --- a/db.js +++ b/db.js @@ -927,10 +927,10 @@ module.exports.CreateDB = function (parent, func) { // Database actions on the events collection obj.GetAllEvents = function (func) { sqlDbQuery('SELECT doc FROM meshcentral.events', null, func); }; - obj.StoreEvent = function (event) { + obj.StoreEvent = function (event, func) { var batchQuery = [['INSERT INTO meshcentral.events VALUE (?, ?, ?, ?, ?, ?, ?)', [null, event.time, ((typeof event.domain == 'string') ? event.domain : null), event.action, event.nodeid ? event.nodeid : null, event.userid ? event.userid : null, JSON.stringify(event)]]]; for (var i in event.ids) { if (event.ids[i] != '*') { batchQuery.push(['INSERT INTO meshcentral.eventids VALUE (LAST_INSERT_ID(), ?)', [event.ids[i]]]); } } - sqlDbBatchExec(batchQuery, function (err, docs) { }); + sqlDbBatchExec(batchQuery, function (err, docs) { if (func != null) { func(err, docs); } }); }; obj.GetEvents = function (ids, domain, func) { if (ids.indexOf('*') >= 0) { @@ -1101,7 +1101,7 @@ module.exports.CreateDB = function (parent, func) { // Database actions on the events collection obj.GetAllEvents = function (func) { obj.eventsfile.find({}).toArray(func); }; - obj.StoreEvent = function (event) { obj.eventsfile.insertOne(event); }; + obj.StoreEvent = function (event, func) { obj.eventsfile.insertOne(event, func); }; obj.GetEvents = function (ids, domain, func) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }).project({ type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).toArray(func); }; obj.GetEventsWithLimit = function (ids, domain, limit, func) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }).project({ type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).toArray(func); }; obj.GetUserEvents = function (ids, domain, username, func) { obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }).project({ type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).toArray(func); }; @@ -1249,7 +1249,7 @@ module.exports.CreateDB = function (parent, func) { // Database actions on the events collection obj.GetAllEvents = function (func) { obj.eventsfile.find({}, func); }; - obj.StoreEvent = function (event) { obj.eventsfile.insert(event); }; + obj.StoreEvent = function (event, func) { obj.eventsfile.insert(event, func); }; obj.GetEvents = function (ids, domain, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).exec(func); } else { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }, func); } }; obj.GetEventsWithLimit = function (ids, domain, limit, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).exec(func); } else { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit, func); } }; obj.GetUserEvents = function (ids, domain, username, func) { @@ -1657,6 +1657,97 @@ module.exports.CreateDB = function (parent, func) { } } + // Transfer NeDB data into the current database + obj.nedbtodb = function (func) { + var nedbDatastore = require('nedb'); + var datastoreOptions = { filename: parent.getConfigFilePath('meshcentral.db'), autoload: true }; + + // If a DB encryption key is provided, perform database encryption + if ((typeof parent.args.dbencryptkey == 'string') && (parent.args.dbencryptkey.length != 0)) { + // Hash the database password into a AES256 key and setup encryption and decryption. + var nedbKey = parent.crypto.createHash('sha384').update(parent.args.dbencryptkey).digest('raw').slice(0, 32); + datastoreOptions.afterSerialization = function (plaintext) { + const iv = parent.crypto.randomBytes(16); + const aes = parent.crypto.createCipheriv('aes-256-cbc', nedbKey, iv); + var ciphertext = aes.update(plaintext); + ciphertext = Buffer.concat([iv, ciphertext, aes.final()]); + return ciphertext.toString('base64'); + } + datastoreOptions.beforeDeserialization = function (ciphertext) { + const ciphertextBytes = Buffer.from(ciphertext, 'base64'); + const iv = ciphertextBytes.slice(0, 16); + const data = ciphertextBytes.slice(16); + const aes = parent.crypto.createDecipheriv('aes-256-cbc', nedbKey, iv); + var plaintextBytes = Buffer.from(aes.update(data)); + plaintextBytes = Buffer.concat([plaintextBytes, aes.final()]); + return plaintextBytes.toString(); + } + } + + // Setup all NeDB collections + var nedbfile = new nedbDatastore(datastoreOptions); + var nedbeventsfile = new nedbDatastore({ filename: parent.getConfigFilePath('meshcentral-events.db'), autoload: true, corruptAlertThreshold: 1 }); + var nedbpowerfile = new nedbDatastore({ filename: parent.getConfigFilePath('meshcentral-power.db'), autoload: true, corruptAlertThreshold: 1 }); + var nedbserverstatsfile = new nedbDatastore({ filename: parent.getConfigFilePath('meshcentral-stats.db'), autoload: true, corruptAlertThreshold: 1 }); + + // Transfered record counts + var normalRecordsTransferCount = 0; + var eventRecordsTransferCount = 0; + var powerRecordsTransferCount = 0; + var statsRecordsTransferCount = 0; + var pendingTransfer = 0; + + // Transfer the data from main database + nedbfile.find({}, function (err, docs) { + if ((err == null) && (docs.length > 0)) { + performTypedRecordDecrypt(docs) + for (var i in docs) { + pendingTransfer++; + normalRecordsTransferCount++; + obj.Set(common.unEscapeLinksFieldName(docs[i]), function () { pendingTransfer--; }); + } + } + + // Transfer events + nedbeventsfile.find({}, function (err, docs) { + if ((err == null) && (docs.length > 0)) { + for (var i in docs) { + pendingTransfer++; + eventRecordsTransferCount++; + obj.StoreEvent(docs[i], function () { pendingTransfer--; }); + } + } + + // Transfer power events + nedbpowerfile.find({}, function (err, docs) { + if ((err == null) && (docs.length > 0)) { + for (var i in docs) { + pendingTransfer++; + powerRecordsTransferCount++; + obj.storePowerEvent(docs[i], null, function () { pendingTransfer--; }); + } + } + + // Transfer server stats + nedbserverstatsfile.find({}, function (err, docs) { + if ((err == null) && (docs.length > 0)) { + for (var i in docs) { + pendingTransfer++; + statsRecordsTransferCount++; + obj.SetServerStats(docs[i], function () { pendingTransfer--; }); + } + } + + // Only exit when all the records are stored. + setInterval(function () { + if (pendingTransfer == 0) { func("Done. " + normalRecordsTransferCount + " record(s), " + eventRecordsTransferCount + " event(s), " + powerRecordsTransferCount + " power change(s), " + statsRecordsTransferCount + " stat(s)."); } + }, 200) + }); + }); + }); + }); + } + function padNumber(number, digits) { return Array(Math.max(digits - String(number).length + 1, 0)).join(0) + number; } // Called when a node has changed diff --git a/meshcentral.js b/meshcentral.js index 4c7f9135..44f3d3f3 100644 --- a/meshcentral.js +++ b/meshcentral.js @@ -138,7 +138,7 @@ function CreateMeshCentralServer(config, args) { try { require('./pass').hash('test', function () { }, 0); } catch (e) { console.log('Old version of node, must upgrade.'); return; } // TODO: Not sure if this test works or not. // Check for invalid arguments - var validArguments = ['_', 'user', 'port', 'aliasport', 'mpsport', 'mpsaliasport', 'redirport', 'rediraliasport', 'cert', 'mpscert', 'deletedomain', 'deletedefaultdomain', 'showall', 'showusers', 'showitem', 'listuserids', 'showusergroups', 'shownodes', 'showallmeshes', 'showmeshes', 'showevents', 'showsmbios', 'showpower', 'clearpower', 'showiplocations', 'help', 'exactports', 'xinstall', 'xuninstall', 'install', 'uninstall', 'start', 'stop', 'restart', 'debug', 'filespath', 'datapath', 'noagentupdate', 'launch', 'noserverbackup', 'mongodb', 'mongodbcol', 'wanonly', 'lanonly', 'nousers', 'mpspass', 'ciralocalfqdn', 'dbexport', 'dbexportmin', 'dbimport', 'dbmerge', 'dbfix', 'dbencryptkey', 'selfupdate', 'tlsoffload', 'userallowedip', 'userblockedip', 'swarmallowedip', 'agentallowedip', 'agentblockedip', 'fastcert', 'swarmport', 'logintoken', 'logintokenkey', 'logintokengen', 'mailtokengen', 'admin', 'unadmin', 'sessionkey', 'sessiontime', 'minify', 'minifycore', 'dblistconfigfiles', 'dbshowconfigfile', 'dbpushconfigfiles', 'dbpullconfigfiles', 'dbdeleteconfigfiles', 'vaultpushconfigfiles', 'vaultpullconfigfiles', 'vaultdeleteconfigfiles', 'configkey', 'loadconfigfromdb', 'npmpath', 'serverid', 'recordencryptionrecode', 'vault', 'token', 'unsealkey', 'name', 'log', 'dbstats', 'translate', 'createaccount', 'resetaccount', 'pass', 'adminaccount', 'removeaccount', 'domain', 'email', 'configfile', 'maintenancemode']; + var validArguments = ['_', 'user', 'port', 'aliasport', 'mpsport', 'mpsaliasport', 'redirport', 'rediraliasport', 'cert', 'mpscert', 'deletedomain', 'deletedefaultdomain', 'showall', 'showusers', 'showitem', 'listuserids', 'showusergroups', 'shownodes', 'showallmeshes', 'showmeshes', 'showevents', 'showsmbios', 'showpower', 'clearpower', 'showiplocations', 'help', 'exactports', 'xinstall', 'xuninstall', 'install', 'uninstall', 'start', 'stop', 'restart', 'debug', 'filespath', 'datapath', 'noagentupdate', 'launch', 'noserverbackup', 'mongodb', 'mongodbcol', 'wanonly', 'lanonly', 'nousers', 'mpspass', 'ciralocalfqdn', 'dbexport', 'dbexportmin', 'dbimport', 'dbmerge', 'dbfix', 'dbencryptkey', 'selfupdate', 'tlsoffload', 'userallowedip', 'userblockedip', 'swarmallowedip', 'agentallowedip', 'agentblockedip', 'fastcert', 'swarmport', 'logintoken', 'logintokenkey', 'logintokengen', 'mailtokengen', 'admin', 'unadmin', 'sessionkey', 'sessiontime', 'minify', 'minifycore', 'dblistconfigfiles', 'dbshowconfigfile', 'dbpushconfigfiles', 'dbpullconfigfiles', 'dbdeleteconfigfiles', 'vaultpushconfigfiles', 'vaultpullconfigfiles', 'vaultdeleteconfigfiles', 'configkey', 'loadconfigfromdb', 'npmpath', 'serverid', 'recordencryptionrecode', 'vault', 'token', 'unsealkey', 'name', 'log', 'dbstats', 'translate', 'createaccount', 'resetaccount', 'pass', 'adminaccount', 'removeaccount', 'domain', 'email', 'configfile', 'maintenancemode', 'nedbtodb']; for (var arg in obj.args) { obj.args[arg.toLocaleLowerCase()] = obj.args[arg]; if (validArguments.indexOf(arg.toLocaleLowerCase()) == -1) { console.log('Invalid argument "' + arg + '", use --help.'); return; } } if (obj.args.mongodb == true) { console.log('Must specify: --mongodb [connectionstring] \r\nSee https://docs.mongodb.com/manual/reference/connection-string/ for MongoDB connection string.'); return; } for (i in obj.config.settings) { obj.args[i] = obj.config.settings[i]; } // Place all settings into arguments, arguments have already been placed into settings so arguments take precedence. @@ -783,6 +783,14 @@ function CreateMeshCentralServer(config, args) { return; } + // Import NeDB data into database + if (obj.args.nedbtodb) { + if (db.databaseType == 1) { console.log("NeDB is current database, can't perform transfer."); process.exit(); return; } + console.log("Transfering NeDB data into database..."); + db.nedbtodb(function (msg) { console.log(msg); process.exit(); }) + return; + } + // Show a list of all configuration files in the database if (obj.args.dblistconfigfiles) { obj.db.GetAllType('cfile', function (err, docs) { if (err == null) { if (docs.length == 0) { console.log("No files found."); } else { for (var i in docs) { console.log(docs[i]._id.split('/')[1] + ', ' + Buffer.from(docs[i].data, 'base64').length + ' bytes.'); } } } else { console.log('Unable to read from database.'); } process.exit(); }); return; @@ -2891,7 +2899,7 @@ function mainStart() { } // Build the list of required modules - var modules = ['ws', 'cbor@~5.2.0', 'nedb', 'https', 'yauzl', 'xmldom', 'ipcheck', 'express', 'archiver@4.0.2', 'multiparty', 'node-forge', 'express-ws', 'compression', 'body-parser', 'connect-redis', 'cookie-session', 'express-handlebars']; + var modules = ['ws', 'cbor@5.2.0', 'nedb', 'https', 'yauzl', 'xmldom', 'ipcheck', 'express', 'archiver@4.0.2', 'multiparty', 'node-forge', 'express-ws', 'compression', 'body-parser', 'connect-redis', 'cookie-session', 'express-handlebars']; if (require('os').platform() == 'win32') { modules.push('node-windows@0.1.14'); if (sspi == true) { modules.push('node-sspi'); } } // Add Windows modules if (ldap == true) { modules.push('ldapauth-fork'); } if (mstsc == true) { modules.push('node-rdpjs-2'); }