MyFavMirrors/MeshCentral
1
0
Fork 0
mirror of https://github.com/Ylianst/MeshCentral.git synced 2024-12-25 14:45:52 -05:00
Code Issues Packages Projects Releases Wiki Activity

Added new --nedbtodb to transfer all NeDB records into the database.

Browse Source
This commit is contained in:
Ylian Saint-Hilaire 2021-01-04 16:26:25 -08:00
parent f6870cd4b3
commit 8ac8953298
2 changed files with 105 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
db.js
View File

@ -927,10 +927,10 @@ module.exports.CreateDB = function (parent, func) {
// Database actions on the events collection
obj.GetAllEvents = function (func) { sqlDbQuery('SELECT doc FROM meshcentral.events', null, func); };
obj.StoreEvent = function (event) {
obj.StoreEvent = function (event, func) {
var batchQuery = [['INSERT INTO meshcentral.events VALUE (?, ?, ?, ?, ?, ?, ?)', [null, event.time, ((typeof event.domain == 'string') ? event.domain : null), event.action, event.nodeid ? event.nodeid : null, event.userid ? event.userid : null, JSON.stringify(event)]]];
for (var i in event.ids) { if (event.ids[i] != '*') { batchQuery.push(['INSERT INTO meshcentral.eventids VALUE (LAST_INSERT_ID(), ?)', [event.ids[i]]]); } }
sqlDbBatchExec(batchQuery, function (err, docs) { });
sqlDbBatchExec(batchQuery, function (err, docs) { if (func != null) { func(err, docs); } });
};
obj.GetEvents = function (ids, domain, func) {
if (ids.indexOf('*') >= 0) {
@ -1101,7 +1101,7 @@ module.exports.CreateDB = function (parent, func) {
// Database actions on the events collection
obj.GetAllEvents = function (func) { obj.eventsfile.find({}).toArray(func); };
obj.StoreEvent = function (event) { obj.eventsfile.insertOne(event); };
obj.StoreEvent = function (event, func) { obj.eventsfile.insertOne(event, func); };
obj.GetEvents = function (ids, domain, func) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }).project({ type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).toArray(func); };
obj.GetEventsWithLimit = function (ids, domain, limit, func) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }).project({ type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).toArray(func); };
obj.GetUserEvents = function (ids, domain, username, func) { obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }).project({ type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).toArray(func); };
@ -1249,7 +1249,7 @@ module.exports.CreateDB = function (parent, func) {
// Database actions on the events collection
obj.GetAllEvents = function (func) { obj.eventsfile.find({}, func); };
obj.StoreEvent = function (event) { obj.eventsfile.insert(event); };
obj.StoreEvent = function (event, func) { obj.eventsfile.insert(event, func); };
obj.GetEvents = function (ids, domain, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).exec(func); } else { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }, func); } };
obj.GetEventsWithLimit = function (ids, domain, limit, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).exec(func); } else { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit, func); } };
obj.GetUserEvents = function (ids, domain, username, func) {
@ -1657,6 +1657,97 @@ module.exports.CreateDB = function (parent, func) {
}
}
// Transfer NeDB data into the current database
obj.nedbtodb = function (func) {
var nedbDatastore = require('nedb');
var datastoreOptions = { filename: parent.getConfigFilePath('meshcentral.db'), autoload: true };
// If a DB encryption key is provided, perform database encryption
if ((typeof parent.args.dbencryptkey == 'string') && (parent.args.dbencryptkey.length != 0)) {
// Hash the database password into a AES256 key and setup encryption and decryption.
var nedbKey = parent.crypto.createHash('sha384').update(parent.args.dbencryptkey).digest('raw').slice(0, 32);
datastoreOptions.afterSerialization = function (plaintext) {
const iv = parent.crypto.randomBytes(16);
const aes = parent.crypto.createCipheriv('aes-256-cbc', nedbKey, iv);
var ciphertext = aes.update(plaintext);
ciphertext = Buffer.concat([iv, ciphertext, aes.final()]);
return ciphertext.toString('base64');
}
datastoreOptions.beforeDeserialization = function (ciphertext) {
const ciphertextBytes = Buffer.from(ciphertext, 'base64');
const iv = ciphertextBytes.slice(0, 16);
const data = ciphertextBytes.slice(16);
const aes = parent.crypto.createDecipheriv('aes-256-cbc', nedbKey, iv);
var plaintextBytes = Buffer.from(aes.update(data));
plaintextBytes = Buffer.concat([plaintextBytes, aes.final()]);
return plaintextBytes.toString();
}
}
// Setup all NeDB collections
var nedbfile = new nedbDatastore(datastoreOptions);
var nedbeventsfile = new nedbDatastore({ filename: parent.getConfigFilePath('meshcentral-events.db'), autoload: true, corruptAlertThreshold: 1 });
var nedbpowerfile = new nedbDatastore({ filename: parent.getConfigFilePath('meshcentral-power.db'), autoload: true, corruptAlertThreshold: 1 });
var nedbserverstatsfile = new nedbDatastore({ filename: parent.getConfigFilePath('meshcentral-stats.db'), autoload: true, corruptAlertThreshold: 1 });
// Transfered record counts
var normalRecordsTransferCount = 0;
var eventRecordsTransferCount = 0;
var powerRecordsTransferCount = 0;
var statsRecordsTransferCount = 0;
var pendingTransfer = 0;
// Transfer the data from main database
nedbfile.find({}, function (err, docs) {
if ((err == null) && (docs.length > 0)) {
performTypedRecordDecrypt(docs)
for (var i in docs) {
pendingTransfer++;
normalRecordsTransferCount++;
obj.Set(common.unEscapeLinksFieldName(docs[i]), function () { pendingTransfer--; });
}
}
// Transfer events
nedbeventsfile.find({}, function (err, docs) {
if ((err == null) && (docs.length > 0)) {
for (var i in docs) {
pendingTransfer++;
eventRecordsTransferCount++;
obj.StoreEvent(docs[i], function () { pendingTransfer--; });
}
}
// Transfer power events
nedbpowerfile.find({}, function (err, docs) {
if ((err == null) && (docs.length > 0)) {
for (var i in docs) {
pendingTransfer++;
powerRecordsTransferCount++;
obj.storePowerEvent(docs[i], null, function () { pendingTransfer--; });
}
}
// Transfer server stats
nedbserverstatsfile.find({}, function (err, docs) {
if ((err == null) && (docs.length > 0)) {
for (var i in docs) {
pendingTransfer++;
statsRecordsTransferCount++;
obj.SetServerStats(docs[i], function () { pendingTransfer--; });
}
}
// Only exit when all the records are stored.
setInterval(function () {
if (pendingTransfer == 0) { func("Done. " + normalRecordsTransferCount + " record(s), " + eventRecordsTransferCount + " event(s), " + powerRecordsTransferCount + " power change(s), " + statsRecordsTransferCount + " stat(s)."); }
}, 200)
});
});
});
});
}
function padNumber(number, digits) { return Array(Math.max(digits - String(number).length + 1, 0)).join(0) + number; }
// Called when a node has changed

12
meshcentral.js
View File

@ -138,7 +138,7 @@ function CreateMeshCentralServer(config, args) {
try { require('./pass').hash('test', function () { }, 0); } catch (e) { console.log('Old version of node, must upgrade.'); return; } // TODO: Not sure if this test works or not.
// Check for invalid arguments
var validArguments = ['_', 'user', 'port', 'aliasport', 'mpsport', 'mpsaliasport', 'redirport', 'rediraliasport', 'cert', 'mpscert', 'deletedomain', 'deletedefaultdomain', 'showall', 'showusers', 'showitem', 'listuserids', 'showusergroups', 'shownodes', 'showallmeshes', 'showmeshes', 'showevents', 'showsmbios', 'showpower', 'clearpower', 'showiplocations', 'help', 'exactports', 'xinstall', 'xuninstall', 'install', 'uninstall', 'start', 'stop', 'restart', 'debug', 'filespath', 'datapath', 'noagentupdate', 'launch', 'noserverbackup', 'mongodb', 'mongodbcol', 'wanonly', 'lanonly', 'nousers', 'mpspass', 'ciralocalfqdn', 'dbexport', 'dbexportmin', 'dbimport', 'dbmerge', 'dbfix', 'dbencryptkey', 'selfupdate', 'tlsoffload', 'userallowedip', 'userblockedip', 'swarmallowedip', 'agentallowedip', 'agentblockedip', 'fastcert', 'swarmport', 'logintoken', 'logintokenkey', 'logintokengen', 'mailtokengen', 'admin', 'unadmin', 'sessionkey', 'sessiontime', 'minify', 'minifycore', 'dblistconfigfiles', 'dbshowconfigfile', 'dbpushconfigfiles', 'dbpullconfigfiles', 'dbdeleteconfigfiles', 'vaultpushconfigfiles', 'vaultpullconfigfiles', 'vaultdeleteconfigfiles', 'configkey', 'loadconfigfromdb', 'npmpath', 'serverid', 'recordencryptionrecode', 'vault', 'token', 'unsealkey', 'name', 'log', 'dbstats', 'translate', 'createaccount', 'resetaccount', 'pass', 'adminaccount', 'removeaccount', 'domain', 'email', 'configfile', 'maintenancemode'];
var validArguments = ['_', 'user', 'port', 'aliasport', 'mpsport', 'mpsaliasport', 'redirport', 'rediraliasport', 'cert', 'mpscert', 'deletedomain', 'deletedefaultdomain', 'showall', 'showusers', 'showitem', 'listuserids', 'showusergroups', 'shownodes', 'showallmeshes', 'showmeshes', 'showevents', 'showsmbios', 'showpower', 'clearpower', 'showiplocations', 'help', 'exactports', 'xinstall', 'xuninstall', 'install', 'uninstall', 'start', 'stop', 'restart', 'debug', 'filespath', 'datapath', 'noagentupdate', 'launch', 'noserverbackup', 'mongodb', 'mongodbcol', 'wanonly', 'lanonly', 'nousers', 'mpspass', 'ciralocalfqdn', 'dbexport', 'dbexportmin', 'dbimport', 'dbmerge', 'dbfix', 'dbencryptkey', 'selfupdate', 'tlsoffload', 'userallowedip', 'userblockedip', 'swarmallowedip', 'agentallowedip', 'agentblockedip', 'fastcert', 'swarmport', 'logintoken', 'logintokenkey', 'logintokengen', 'mailtokengen', 'admin', 'unadmin', 'sessionkey', 'sessiontime', 'minify', 'minifycore', 'dblistconfigfiles', 'dbshowconfigfile', 'dbpushconfigfiles', 'dbpullconfigfiles', 'dbdeleteconfigfiles', 'vaultpushconfigfiles', 'vaultpullconfigfiles', 'vaultdeleteconfigfiles', 'configkey', 'loadconfigfromdb', 'npmpath', 'serverid', 'recordencryptionrecode', 'vault', 'token', 'unsealkey', 'name', 'log', 'dbstats', 'translate', 'createaccount', 'resetaccount', 'pass', 'adminaccount', 'removeaccount', 'domain', 'email', 'configfile', 'maintenancemode', 'nedbtodb'];
for (var arg in obj.args) { obj.args[arg.toLocaleLowerCase()] = obj.args[arg]; if (validArguments.indexOf(arg.toLocaleLowerCase()) == -1) { console.log('Invalid argument "' + arg + '", use --help.'); return; } }
if (obj.args.mongodb == true) { console.log('Must specify: --mongodb [connectionstring] \r\nSee https://docs.mongodb.com/manual/reference/connection-string/ for MongoDB connection string.'); return; }
for (i in obj.config.settings) { obj.args[i] = obj.config.settings[i]; } // Place all settings into arguments, arguments have already been placed into settings so arguments take precedence.
@ -783,6 +783,14 @@ function CreateMeshCentralServer(config, args) {
return;
}
// Import NeDB data into database
if (obj.args.nedbtodb) {
if (db.databaseType == 1) { console.log("NeDB is current database, can't perform transfer."); process.exit(); return; }
console.log("Transfering NeDB data into database...");
db.nedbtodb(function (msg) { console.log(msg); process.exit(); })
return;
}
// Show a list of all configuration files in the database
if (obj.args.dblistconfigfiles) {
obj.db.GetAllType('cfile', function (err, docs) { if (err == null) { if (docs.length == 0) { console.log("No files found."); } else { for (var i in docs) { console.log(docs[i]._id.split('/')[1] + ', ' + Buffer.from(docs[i].data, 'base64').length + ' bytes.'); } } } else { console.log('Unable to read from database.'); } process.exit(); }); return;
@ -2891,7 +2899,7 @@ function mainStart() {
}
// Build the list of required modules
var modules = ['ws', 'cbor@~5.2.0', 'nedb', 'https', 'yauzl', 'xmldom', 'ipcheck', 'express', 'archiver@4.0.2', 'multiparty', 'node-forge', 'express-ws', 'compression', 'body-parser', 'connect-redis', 'cookie-session', 'express-handlebars'];
var modules = ['ws', 'cbor@5.2.0', 'nedb', 'https', 'yauzl', 'xmldom', 'ipcheck', 'express', 'archiver@4.0.2', 'multiparty', 'node-forge', 'express-ws', 'compression', 'body-parser', 'connect-redis', 'cookie-session', 'express-handlebars'];
if (require('os').platform() == 'win32') { modules.push('node-windows@0.1.14'); if (sspi == true) { modules.push('node-sspi'); } } // Add Windows modules
if (ldap == true) { modules.push('ldapauth-fork'); }
if (mstsc == true) { modules.push('node-rdpjs-2'); }