diff --git a/MeshCentralServer.njsproj b/MeshCentralServer.njsproj index 0fbff49c..458e9bff 100644 --- a/MeshCentralServer.njsproj +++ b/MeshCentralServer.njsproj @@ -91,6 +91,7 @@ + diff --git a/db-test.js b/db-test.js deleted file mode 100644 index 040ba310..00000000 --- a/db-test.js +++ /dev/null @@ -1,487 +0,0 @@ -/** -* @description MeshCentral database module -* @author Ylian Saint-Hilaire -* @copyright Intel Corporation 2018-2019 -* @license Apache-2.0 -* @version v0.0.2 -*/ - -/*xjslint node: true */ -/*xjslint plusplus: true */ -/*xjslint maxlen: 256 */ -/*jshint node: true */ -/*jshint strict: false */ -/*jshint esversion: 6 */ -"use strict"; - -// -// Construct Meshcentral database object -// -// The default database is NeDB -// https://github.com/louischatriot/nedb -// -// Alternativety, MongoDB can be used -// https://www.mongodb.com/ -// Just run with --mongodb [connectionstring], where the connection string is documented here: https://docs.mongodb.com/manual/reference/connection-string/ -// The default collection is "meshcentral", but you can override it using --mongodbcol [collection] -// -module.exports.CreateDB = function (parent, func) { - var obj = {}; - var Datastore = null; - var expireEventsSeconds = (60 * 60 * 24 * 20); // By default, expire events after 20 days. (Seconds * Minutes * Hours * Days) - var expirePowerEventsSeconds = (60 * 60 * 24 * 10); // By default, expire power events after 10 days. (Seconds * Minutes * Hours * Days) - var expireServerStatsSeconds = (60 * 60 * 24 * 30); // By default, expire power events after 30 days. (Seconds * Minutes * Hours * Days) - obj.path = require('path'); - obj.parent = parent; - obj.identifier = null; - obj.dbKey = null; - - // Read expiration time from configuration file - if (typeof obj.parent.args.dbexpire == 'object') { - if (typeof obj.parent.args.dbexpire.events == 'number') { expireEventsSeconds = obj.parent.args.dbexpire.events; } - if (typeof obj.parent.args.dbexpire.powerevents == 'number') { expirePowerEventsSeconds = obj.parent.args.dbexpire.powerevents; } - if (typeof obj.parent.args.dbexpire.statsevents == 'number') { expireServerStatsSeconds = obj.parent.args.dbexpire.statsevents; } - } - - if (obj.parent.args.mongodb) { - // Use MongoDB - obj.databaseType = 2; - Datastore = require('mongodb').MongoClient; - Datastore.connect(obj.parent.args.mongodb, function (err, client) { - if (err != null) { console.log("Unable to connect to database: " + err); process.exit(); return; } - const db = client.db('meshcentral'); - - var dbcollection = 'meshcentral'; - if (obj.parent.args.mongodbcol) { dbcollection = obj.parent.args.mongodbcol; } - - // Setup MongoDB main collection and indexes - obj.file = db.collection(dbcollection); - - obj.file.find({ type: 'mesh' }, function (err, cursor) { - cursor.each(function (err, item) { - console.log(err, item); - }); - }); - - - /* - obj.file.getIndexes(function (err, indexes) { - // Check if we need to reset indexes - var indexesByName = {}, indexCount = 0; - for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; } - if ((indexCount != 4) || (indexesByName['TypeDomainMesh1'] == null) || (indexesByName['Email1'] == null) || (indexesByName['Mesh1'] == null)) { - console.log('Resetting main indexes...'); - obj.file.dropIndexes(function (err) { - obj.file.createIndex({ type: 1, domain: 1, meshid: 1 }, { sparse: 1, name: 'TypeDomainMesh1' }); // Speeds up GetAllTypeNoTypeField() and GetAllTypeNoTypeFieldMeshFiltered() - obj.file.createIndex({ email: 1 }, { sparse: 1, name: 'Email1' }); // Speeds up GetUserWithEmail() and GetUserWithVerifiedEmail() - obj.file.createIndex({ meshid: 1 }, { sparse: 1, name: 'Mesh1' }); // Speeds up RemoveMesh() - }); - } - }); - */ - - /* - // Setup the changeStream on the MongoDB main collection - obj.fileChangeStream = obj.file.watch(); - obj.fileChangeStream.on('change', function (next) { - // Process next document - console.log('change', next); - }); - */ - - // Setup MongoDB events collection and indexes - obj.eventsfile = db.collection('events'); // Collection containing all events - /* - obj.eventsfile.getIndexes(function (err, indexes) { - // Check if we need to reset indexes - var indexesByName = {}, indexCount = 0; - for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; } - if ((indexCount != 5) || (indexesByName['Username1'] == null) || (indexesByName['DomainNodeTime1'] == null) || (indexesByName['IdsAndTime1'] == null) || (indexesByName['ExpireTime1'] == null)) { - // Reset all indexes - console.log('Resetting events indexes...'); - obj.eventsfile.dropIndexes(function (err) { - obj.eventsfile.createIndex({ username: 1 }, { sparse: 1, name: 'Username1' }); - obj.eventsfile.createIndex({ domain: 1, nodeid: 1, time: -1 }, { sparse: 1, name: 'DomainNodeTime1' }); - obj.eventsfile.createIndex({ ids: 1, time: -1 }, { sparse: 1, name: 'IdsAndTime1' }); - obj.eventsfile.createIndex({ "time": 1 }, { expireAfterSeconds: expireEventsSeconds, name: 'ExpireTime1' }); - }); - } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expireEventsSeconds) { - // Reset the timeout index - console.log('Resetting events expire index...'); - obj.eventsfile.dropIndex("ExpireTime1", function (err) { - obj.eventsfile.createIndex({ "time": 1 }, { expireAfterSeconds: expireEventsSeconds, name: 'ExpireTime1' }); - }); - } - }); - */ - - // Setup MongoDB power events collection and indexes - obj.powerfile = db.collection('power'); // Collection containing all power events - /* - obj.powerfile.getIndexes(function (err, indexes) { - // Check if we need to reset indexes - var indexesByName = {}, indexCount = 0; - for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; } - if ((indexCount != 3) || (indexesByName['NodeIdAndTime1'] == null) || (indexesByName['ExpireTime1'] == null)) { - // Reset all indexes - console.log('Resetting power events indexes...'); - obj.powerfile.dropIndexes(function (err) { - // Create all indexes - obj.powerfile.createIndex({ nodeid: 1, time: 1 }, { sparse: 1, name: 'NodeIdAndTime1' }); - obj.powerfile.createIndex({ "time": 1 }, { expireAfterSeconds: expirePowerEventsSeconds, name: 'ExpireTime1' }); - }); - } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expirePowerEventsSeconds) { - // Reset the timeout index - console.log('Resetting power events expire index...'); - obj.powerfile.dropIndex("ExpireTime1", function (err) { - // Reset the expire power events index - obj.powerfile.createIndex({ "time": 1 }, { expireAfterSeconds: expirePowerEventsSeconds, name: 'ExpireTime1' }); - }); - } - }); - */ - - // Setup MongoDB smbios collection, no indexes needed - obj.smbiosfile = db.collection('smbios'); // Collection containing all smbios information - - // Setup MongoDB server stats collection - obj.serverstatsfile = db.collection('serverstats'); // Collection of server stats - /* - obj.serverstatsfile.getIndexes(function (err, indexes) { - // Check if we need to reset indexes - var indexesByName = {}, indexCount = 0; - for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; } - if ((indexCount != 3) || (indexesByName['ExpireTime1'] == null)) { - // Reset all indexes - console.log('Resetting server stats indexes...'); - obj.serverstatsfile.dropIndexes(function (err) { - // Create all indexes - obj.serverstatsfile.createIndex({ "time": 1 }, { expireAfterSeconds: expireServerStatsSeconds, name: 'ExpireTime1' }); - obj.serverstatsfile.createIndex({ "expire": 1 }, { expireAfterSeconds: 0, name: 'ExpireTime2' }); // Auto-expire events - }); - } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expireServerStatsSeconds) { - // Reset the timeout index - console.log('Resetting server stats expire index...'); - obj.serverstatsfile.dropIndex("ExpireTime1", function (err) { - // Reset the expire server stats index - obj.serverstatsfile.createIndex({ "time": 1 }, { expireAfterSeconds: expireServerStatsSeconds, name: 'ExpireTime1' }); - }); - } - }); - */ - func(); // Completed MongoDB setup - }); - } else { - // Use NeDB (The default) - obj.databaseType = 1; - Datastore = require('nedb'); - var datastoreOptions = { filename: obj.parent.getConfigFilePath('meshcentral.db'), autoload: true }; - - // If a DB encryption key is provided, perform database encryption - if ((typeof obj.parent.args.dbencryptkey == 'string') && (obj.parent.args.dbencryptkey.length != 0)) { - // Hash the database password into a AES256 key and setup encryption and decryption. - obj.dbKey = obj.parent.crypto.createHash('sha384').update(obj.parent.args.dbencryptkey).digest("raw").slice(0, 32); - datastoreOptions.afterSerialization = function (plaintext) { - const iv = obj.parent.crypto.randomBytes(16); - const aes = obj.parent.crypto.createCipheriv('aes-256-cbc', obj.dbKey, iv); - var ciphertext = aes.update(plaintext); - ciphertext = Buffer.concat([iv, ciphertext, aes.final()]); - return ciphertext.toString('base64'); - } - datastoreOptions.beforeDeserialization = function (ciphertext) { - const ciphertextBytes = Buffer.from(ciphertext, 'base64'); - const iv = ciphertextBytes.slice(0, 16); - const data = ciphertextBytes.slice(16); - const aes = obj.parent.crypto.createDecipheriv('aes-256-cbc', obj.dbKey, iv); - var plaintextBytes = Buffer.from(aes.update(data)); - plaintextBytes = Buffer.concat([plaintextBytes, aes.final()]); - return plaintextBytes.toString(); - } - } - - // Start NeDB main collection and setup indexes - obj.file = new Datastore(datastoreOptions); - obj.file.persistence.setAutocompactionInterval(36000); - obj.file.ensureIndex({ fieldName: 'type' }); - obj.file.ensureIndex({ fieldName: 'domain' }); - obj.file.ensureIndex({ fieldName: 'meshid', sparse: true }); - obj.file.ensureIndex({ fieldName: 'nodeid', sparse: true }); - obj.file.ensureIndex({ fieldName: 'email', sparse: true }); - - // Setup the events collection and setup indexes - obj.eventsfile = new Datastore({ filename: obj.parent.getConfigFilePath('meshcentral-events.db'), autoload: true }); - obj.eventsfile.persistence.setAutocompactionInterval(36000); - obj.eventsfile.ensureIndex({ fieldName: 'ids' }); // TODO: Not sure if this is a good index, this is a array field. - obj.eventsfile.ensureIndex({ fieldName: 'nodeid', sparse: true }); - obj.eventsfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: 60 * 60 * 24 * 20 }); // Limit the power event log to 20 days (Seconds * Minutes * Hours * Days) - - // Setup the power collection and setup indexes - obj.powerfile = new Datastore({ filename: obj.parent.getConfigFilePath('meshcentral-power.db'), autoload: true }); - obj.powerfile.persistence.setAutocompactionInterval(36000); - obj.powerfile.ensureIndex({ fieldName: 'nodeid' }); - obj.powerfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: 60 * 60 * 24 * 10 }); // Limit the power event log to 10 days (Seconds * Minutes * Hours * Days) - - // Setup the SMBIOS collection - obj.smbiosfile = new Datastore({ filename: obj.parent.getConfigFilePath('meshcentral-smbios.db'), autoload: true }); - - // Setup the server stats collection and setup indexes - obj.serverstatsfile = new Datastore({ filename: obj.parent.getConfigFilePath('meshcentral-stats.db'), autoload: true }); - obj.serverstatsfile.persistence.setAutocompactionInterval(36000); - obj.serverstatsfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: 60 * 60 * 24 * 30 }); // Limit the server stats log to 30 days (Seconds * Minutes * Hours * Days) - obj.serverstatsfile.ensureIndex({ fieldName: 'expire', expireAfterSeconds: 0 }); // Auto-expire events - - func(); // Completed NeDB setup - } - - obj.SetupDatabase = function (func) { - // Check if the database unique identifier is present - // This is used to check that in server peering mode, everyone is using the same database. - obj.Get('DatabaseIdentifier', function (err, docs) { - if ((docs.length == 1) && (docs[0].value != null)) { - obj.identifier = docs[0].value; - } else { - obj.identifier = Buffer.from(require('crypto').randomBytes(48), 'binary').toString('hex'); - obj.Set({ _id: 'DatabaseIdentifier', value: obj.identifier }); - } - }); - - // Load database schema version and check if we need to update - obj.Get('SchemaVersion', function (err, docs) { - var ver = 0; - if (docs && docs.length == 1) { ver = docs[0].value; } - if (ver == 1) { console.log('This is an unsupported beta 1 database, delete it to create a new one.'); process.exit(0); } - - // TODO: Any schema upgrades here... - obj.Set({ _id: 'SchemaVersion', value: 2 }); - - func(ver); - }); - }; - - obj.cleanup = function (func) { - // TODO: Remove all mesh links to invalid users - // TODO: Remove all meshes that dont have any links - - // Remove all events, power events and SMBIOS data from the main collection. They are all in seperate collections now. - obj.file.remove({ type: 'event' }, { multi: true }); - obj.file.remove({ type: 'power' }, { multi: true }); - obj.file.remove({ type: 'smbios' }, { multi: true }); - - // Remove all objects that have a "meshid" that no longer points to a valid mesh. - obj.GetAllType('mesh', function (err, docs) { - var meshlist = []; - if ((err == null) && (docs.length > 0)) { for (var i in docs) { meshlist.push(docs[i]._id); } } - obj.file.remove({ meshid: { $exists: true, $nin: meshlist } }, { multi: true }); - - // Fix all of the creating & login to ticks by seconds, not milliseconds. - obj.GetAllType('user', function (err, docs) { - if (err == null && docs.length > 0) { - for (var i in docs) { - var fixed = false; - - // Fix account creation - if (docs[i].creation) { - if (docs[i].creation > 1300000000000) { docs[i].creation = Math.floor(docs[i].creation / 1000); fixed = true; } - if ((docs[i].creation % 1) != 0) { docs[i].creation = Math.floor(docs[i].creation); fixed = true; } - } - - // Fix last account login - if (docs[i].login) { - if (docs[i].login > 1300000000000) { docs[i].login = Math.floor(docs[i].login / 1000); fixed = true; } - if ((docs[i].login % 1) != 0) { docs[i].login = Math.floor(docs[i].login); fixed = true; } - } - - // Fix last password change - if (docs[i].passchange) { - if (docs[i].passchange > 1300000000000) { docs[i].passchange = Math.floor(docs[i].passchange / 1000); fixed = true; } - if ((docs[i].passchange % 1) != 0) { docs[i].passchange = Math.floor(docs[i].passchange); fixed = true; } - } - - // Fix subscriptions - if (docs[i].subscriptions != null) { delete docs[i].subscriptions; fixed = true; } - - // Save the user if needed - if (fixed) { obj.Set(docs[i]); } - - // We are done - if (func) { func(); } - } - } - }); - }); - }; - - // Database actions on the main collection - obj.Set = function (data, func) { obj.file.update({ _id: data._id }, data, { upsert: true }, func); }; - obj.Get = function (id, func) - { - if (arguments.length > 2) - { - var parms = [func]; - for (var parmx = 2; parmx < arguments.length; ++parmx) { parms.push(arguments[parmx]); } - var func2 = function _func2(arg1, arg2) - { - var userCallback = _func2.userArgs.shift(); - _func2.userArgs.unshift(arg2); - _func2.userArgs.unshift(arg1); - userCallback.apply(obj, _func2.userArgs); - }; - func2.userArgs = parms; - obj.file.find({ _id: id }, func2); - } - else - { - obj.file.find({ _id: id }, func); - } - }; - obj.GetAll = function (func) { obj.file.find({}, func); }; - obj.GetAllTypeNoTypeField = function (type, domain, func) { obj.file.find({ type: type, domain: domain }, { type: 0 }, func); }; - obj.GetAllTypeNoTypeFieldMeshFiltered = function (meshes, domain, type, id, func) { var x = { type: type, domain: domain, meshid: { $in: meshes } }; if (id) { x._id = id; } obj.file.find(x, { type: 0 }, func); }; - //obj.GetAllType = function (type, func) { obj.file.find({ type: type }, func); }; - - obj.GetAllType = function (type, func) { obj.file.find({ type: type }, function (err, cursor) { if (err) { func(err); } else { var r = []; cursor.each(function (err, item) { if (err) { func(err); } else { if (item) { r.push(item); } else { func(null, r); } } }); } }); }; - - - - obj.GetAllIdsOfType = function (ids, domain, type, func) { obj.file.find({ type: type, domain: domain, _id: { $in: ids } }, func); }; - obj.GetUserWithEmail = function (domain, email, func) { obj.file.find({ type: 'user', domain: domain, email: email }, { type: 0 }, func); }; - obj.GetUserWithVerifiedEmail = function (domain, email, func) { obj.file.find({ type: 'user', domain: domain, email: email, emailVerified: true }, { type: 0 }, func); }; - obj.Remove = function (id) { obj.file.remove({ _id: id }); }; - obj.RemoveAll = function (func) { obj.file.remove({}, { multi: true }, func); }; - obj.RemoveAllOfType = function (type, func) { obj.file.remove({ type: type }, { multi: true }, func); }; - obj.InsertMany = function (data, func) { obj.file.insert(data, func); }; - obj.RemoveMeshDocuments = function (id) { obj.file.remove({ meshid: id }, { multi: true }); obj.file.remove({ _id: 'nt' + id }); }; - obj.MakeSiteAdmin = function (username, domain) { obj.Get('user/' + domain + '/' + username, function (err, docs) { if (docs.length == 1) { docs[0].siteadmin = 0xFFFFFFFF; obj.Set(docs[0]); } }); }; - obj.DeleteDomain = function (domain, func) { obj.file.remove({ domain: domain }, { multi: true }, func); }; - obj.SetUser = function (user) { var u = Clone(user); if (u.subscriptions) { delete u.subscriptions; } obj.Set(u); }; - obj.dispose = function () { for (var x in obj) { if (obj[x].close) { obj[x].close(); } delete obj[x]; } }; - obj.getLocalAmtNodes = function (func) { obj.file.find({ type: 'node', host: { $exists: true, $ne: null }, intelamt: { $exists: true } }, func); }; - obj.getAmtUuidNode = function (meshid, uuid, func) { obj.file.find({ type: 'node', meshid: meshid, 'intelamt.uuid': uuid }, func); }; - obj.isMaxType = function (max, type, domainid, func) { if (max == null) { func(false); } else { obj.file.count({ type: type, domain: domainid }, function (err, count) { func((err != null) || (count > max)); }); } } - - // Database actions on the events collection - obj.GetAllEvents = function (func) { obj.eventsfile.find({}, func); }; - obj.StoreEvent = function (event) { obj.eventsfile.insert(event); }; - obj.GetEvents = function (ids, domain, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).exec(func); } else { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }, func); } }; - obj.GetEventsWithLimit = function (ids, domain, limit, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).exec(func); } else { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit, func); } }; - obj.GetUserEvents = function (ids, domain, username, func) { - if (obj.databaseType == 1) { - obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).exec(func); - } else { - obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }, func); - } - }; - obj.GetUserEventsWithLimit = function (ids, domain, username, limit, func) { - if (obj.databaseType == 1) { - obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).exec(func); - } else { - obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit, func); - } - }; - obj.GetNodeEventsWithLimit = function (nodeid, domain, limit, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, nodeid: nodeid }, { type: 0, etype: 0, _id: 0, domain: 0, ids: 0, node: 0, nodeid: 0 }).sort({ time: -1 }).limit(limit).exec(func); } else { obj.eventsfile.find({ domain: domain, nodeid: nodeid }, { type: 0, etype: 0, _id: 0, domain: 0, ids: 0, node: 0, nodeid: 0 }).sort({ time: -1 }).limit(limit, func); } }; - obj.RemoveAllEvents = function (domain) { obj.eventsfile.remove({ domain: domain }, { multi: true }); }; - obj.RemoveAllNodeEvents = function (domain, nodeid) { obj.eventsfile.remove({ domain: domain, nodeid: nodeid }, { multi: true }); }; - - // Database actions on the power collection - obj.getAllPower = function (func) { obj.powerfile.find({}, func); }; - obj.storePowerEvent = function (event, multiServer, func) { if (multiServer != null) { event.server = multiServer.serverid; } obj.powerfile.insert(event, func); }; - obj.getPowerTimeline = function (nodeid, func) { if (obj.databaseType == 1) { obj.powerfile.find({ nodeid: { $in: ['*', nodeid] } }, { _id: 0, nodeid: 0, s: 0 }).sort({ time: 1 }).exec(func); } else { obj.powerfile.find({ nodeid: { $in: ['*', nodeid] } }, { _id: 0, nodeid: 0, s: 0 }).sort({ time: 1 }, func); } }; - obj.removeAllPowerEvents = function () { obj.powerfile.remove({}, { multi: true }); }; - obj.removeAllPowerEventsForNode = function (nodeid) { obj.powerfile.remove({ nodeid: nodeid }, { multi: true }); }; - - // Database actions on the SMBIOS collection - obj.SetSMBIOS = function (smbios, func) { obj.smbiosfile.update({ _id: smbios._id }, smbios, { upsert: true }, func); }; - obj.RemoveSMBIOS = function (id) { obj.smbiosfile.remove({ _id: id }); }; - obj.GetSMBIOS = function (id, func) { obj.smbiosfile.find({ _id: id }, func); }; - - // Database actions on the Server Stats collection - obj.SetServerStats = function (data, func) { obj.serverstatsfile.insert(data, func); }; - obj.GetServerStats = function (hours, func) { var t = new Date(); t.setTime(t.getTime() - (60 * 60 * 1000 * hours)); obj.serverstatsfile.find({ time: { $gt: t } }, { _id: 0, cpu: 0 }, func); }; - - // Read a configuration file from the database - obj.getConfigFile = function (path, func) { obj.Get('cfile/' + path, func); } - - // Write a configuration file to the database - obj.setConfigFile = function (path, data, func) { obj.Set({ _id: 'cfile/' + path, type: 'cfile', data: data.toString('base64') }, func); } - - // List all configuration files - obj.listConfigFiles = function (func) { obj.file.find({ type: 'cfile' }).sort({ _id: 1 }).exec(func); } - - // Get all configuration files - obj.getAllConfigFiles = function (password, func) { - obj.file.find({ type: 'cfile' }, function (err, docs) { - if (err != null) { func(null); return; } - var r = null; - for (var i = 0; i < docs.length; i++) { - var name = docs[i]._id.split('/')[1]; - var data = obj.decryptData(password, docs[i].data); - if (data != null) { if (r == null) { r = {}; } r[name] = data; } - } - func(r); - }); - } - - // Get encryption key - obj.getEncryptDataKey = function (password) { - if (typeof password != 'string') return null; - return obj.parent.crypto.createHash('sha384').update(password).digest("raw").slice(0, 32); - } - - // Encrypt data - obj.encryptData = function (password, plaintext) { - var key = obj.getEncryptDataKey(password); - if (key == null) return null; - const iv = obj.parent.crypto.randomBytes(16); - const aes = obj.parent.crypto.createCipheriv('aes-256-cbc', key, iv); - var ciphertext = aes.update(plaintext); - ciphertext = Buffer.concat([iv, ciphertext, aes.final()]); - return ciphertext.toString('base64'); - } - - // Decrypt data - obj.decryptData = function (password, ciphertext) { - try { - var key = obj.getEncryptDataKey(password); - if (key == null) return null; - const ciphertextBytes = Buffer.from(ciphertext, 'base64'); - const iv = ciphertextBytes.slice(0, 16); - const data = ciphertextBytes.slice(16); - const aes = obj.parent.crypto.createDecipheriv('aes-256-cbc', key, iv); - var plaintextBytes = Buffer.from(aes.update(data)); - plaintextBytes = Buffer.concat([plaintextBytes, aes.final()]); - return plaintextBytes; - } catch (ex) { return null; } - } - - // Get the number of records in the database for various types, this is the slow NeDB way. - // WARNING: This is a terrible query for database performance. Only do this when needed. This query will look at almost every document in the database. - obj.getStats = function (func) { - if (obj.databaseType == 2) { - // MongoDB version - obj.file.aggregate([{ "$group": { _id: "$type", count: { $sum: 1 } } }], function (err, docs) { - var counters = {}, totalCount = 0; - for (var i in docs) { if (docs[i]._id != null) { counters[docs[i]._id] = docs[i].count; totalCount += docs[i].count; } } - func({ nodes: counters['node'], meshes: counters['mesh'], users: counters['user'], total: totalCount }); - }) - } else { - // NeDB version - obj.file.count({ type: 'node' }, function (err, nodeCount) { - obj.file.count({ type: 'mesh' }, function (err, meshCount) { - obj.file.count({ type: 'user' }, function (err, userCount) { - obj.file.count({}, function (err, totalCount) { - func({ nodes: nodeCount, meshes: meshCount, users: userCount, total: totalCount }); - }); - }); - }); - }); - } - } - - // This is used to rate limit a number of operation per day. Returns a startValue each new days, but you can substract it and save the value in the db. - obj.getValueOfTheDay = function (id, startValue, func) { obj.Get(id, function (err, docs) { var date = new Date(), t = date.toLocaleDateString(); if (docs.length == 1) { var r = docs[0]; if (r.day == t) { func({ _id: id, value: r.value, day: t }); return; } } func({ _id: id, value: startValue, day: t }); }); }; - obj.escapeBase64 = function escapeBase64(val) { return (val.replace(/\+/g, '@').replace(/\//g, '$')); } - - function Clone(v) { return JSON.parse(JSON.stringify(v)); } - - return obj; -}; \ No newline at end of file diff --git a/db.js b/db.js index 62e1954d..07f6eae9 100644 --- a/db.js +++ b/db.js @@ -668,5 +668,88 @@ module.exports.CreateDB = function (parent, func) { func(obj); // Completed function setup } + obj.performBackup = function () { + console.log('Performing backup...'); + try { obj.parent.fs.mkdirSync(obj.parent.backuppath); } catch (e) { } + const dbname = (obj.parent.args.mongodbname) ? (obj.parent.args.mongodbname) : 'meshcentral'; + const currentDate = new Date(); + const fileSuffix = currentDate.getFullYear() + '-' + padNumber(currentDate.getMonth() + 1, 2) + '-' + padNumber(currentDate.getDate(), 2) + '-' + padNumber(currentDate.getHours(), 2) + '-' + padNumber(currentDate.getMinutes(), 2); + const newAutoBackupFile = 'meshcentral-autobackup-' + fileSuffix; + const newAutoBackupPath = obj.parent.path.join(obj.parent.backuppath, newAutoBackupFile); + + if ((obj.databaseType == 2) || (obj.databaseType == 3)) { + // Perform a MongoDump backup + const newBackupFile = 'mongodump-' + fileSuffix; + const newBackupPath = obj.parent.path.join(obj.parent.backuppath, newBackupFile); + var mongoDumpPath = 'mongodump'; + if (obj.parent.config.settings.autobackup && obj.parent.config.settings.autobackup.mongodumppath) { mongoDumpPath = obj.parent.config.settings.autobackup.mongodumppath; } + const child_process = require('child_process'); + const cmd = mongoDumpPath + ' --db \"' + dbname + '\" --archive=\"' + newBackupPath + '.archive\"'; + var backupProcess = child_process.exec(cmd, { cwd: obj.parent.backuppath }, function (error, stdout, stderr) { + backupProcess = null; + if ((error != null) && (error != '')) { console.log('ERROR: Unable to perform database backup.\r\n'); return; } + + // Perform archive compression + var archiver = require('archiver'); + var output = obj.parent.fs.createWriteStream(newAutoBackupPath + '.zip'); + var archive = null; + if (obj.parent.config.settings.autobackup && (typeof obj.parent.config.settings.autobackup.zippassword == 'string')) { + try { archiver.registerFormat('zip-encrypted', require("archiver-zip-encrypted")); } catch (ex) { } + archive = archiver.create('zip-encrypted', { zlib: { level: 9 }, encryptionMethod: 'aes256', password: obj.parent.config.settings.autobackup.zippassword }); + } else { + archive = archiver('zip', { zlib: { level: 9 } }); + } + output.on('close', function () { setTimeout(function () { try { obj.parent.fs.unlink(newBackupPath + '.archive'); } catch (ex) { } }, 5000); }); + output.on('end', function () { }); + archive.on('warning', function (err) { console.log('Backup warning: ' + err); }); + archive.on('error', function (err) { console.log('Backup error: ' + err); }); + archive.pipe(output); + archive.file(newBackupPath + '.archive', { name: newBackupFile + '.archive' }); + archive.directory(obj.parent.datapath, 'meshcentral-data'); + archive.finalize(); + }); + } else { + // Perform a NeDB backup + var archiver = require('archiver'); + var output = obj.parent.fs.createWriteStream(newAutoBackupPath + '.zip'); + var archive = null; + if (obj.parent.config.settings.autobackup && (typeof obj.parent.config.settings.autobackup.zippassword == 'string')) { + try { archiver.registerFormat('zip-encrypted', require("archiver-zip-encrypted")); } catch (ex) { } + archive = archiver.create('zip-encrypted', { zlib: { level: 9 }, encryptionMethod: 'aes256', password: obj.parent.config.settings.autobackup.zippassword }); + } else { + archive = archiver('zip', { zlib: { level: 9 } }); + } + output.on('close', function () { }); + output.on('end', function () { }); + archive.on('warning', function (err) { console.log('Backup warning: ' + err); }); + archive.on('error', function (err) { console.log('Backup error: ' + err); }); + archive.pipe(output); + archive.directory(obj.parent.datapath, 'meshcentral-data'); + archive.finalize(); + } + + // Remove old backups + if (obj.parent.config.settings.autobackup && (typeof obj.parent.config.settings.autobackup.keeplastdaysbackup == 'number')) { + var cutoffDate = new Date(); + cutoffDate.setDate(cutoffDate.getDate() - obj.parent.config.settings.autobackup.keeplastdaysbackup); + obj.parent.fs.readdir(obj.parent.backuppath, function (err, dir) { + if ((err == null) && (dir.length > 0)) { + for (var i in dir) { + var name = dir[i]; + if (name.startsWith('meshcentral-autobackup-') && name.endsWith('.zip')) { + var timex = name.substring(23, name.length - 4).split('-'); + if (timex.length == 5) { + var fileDate = new Date(parseInt(timex[0]), parseInt(timex[1]) - 1, parseInt(timex[2]), parseInt(timex[3]), parseInt(timex[4])); + if (fileDate && (cutoffDate > fileDate)) { try { obj.parent.fs.unlink(obj.parent.path.join(obj.parent.backuppath, name)); } catch (ex) { } } + } + } + } + } + }); + } + } + + function padNumber(number, digits) { return Array(Math.max(digits - String(number).length + 1, 0)).join(0) + number; } + return obj; }; \ No newline at end of file diff --git a/meshcentral.js b/meshcentral.js index 830e3603..4f4237c5 100644 --- a/meshcentral.js +++ b/meshcentral.js @@ -69,12 +69,14 @@ function CreateMeshCentralServer(config, args) { obj.parentpath = obj.path.join(__dirname, '../..'); obj.datapath = obj.path.join(__dirname, '../../meshcentral-data'); obj.filespath = obj.path.join(__dirname, '../../meshcentral-files'); + obj.backuppath = obj.path.join(__dirname, '../../meshcentral-backup'); if (obj.fs.existsSync(obj.path.join(__dirname, '../../meshcentral-web/views'))) { obj.webViewsPath = obj.path.join(__dirname, '../../meshcentral-web/views'); } else { obj.webViewsPath = obj.path.join(__dirname, 'views'); } if (obj.fs.existsSync(obj.path.join(__dirname, '../../meshcentral-web/public'))) { obj.webPublicPath = obj.path.join(__dirname, '../../meshcentral-web/public'); } else { obj.webPublicPath = obj.path.join(__dirname, 'public'); } } else { obj.parentpath = __dirname; obj.datapath = obj.path.join(__dirname, '../meshcentral-data'); obj.filespath = obj.path.join(__dirname, '../meshcentral-files'); + obj.backuppath = obj.path.join(__dirname, '../meshcentral-backups'); if (obj.fs.existsSync(obj.path.join(__dirname, '../meshcentral-web/views'))) { obj.webViewsPath = obj.path.join(__dirname, '../meshcentral-web/views'); } else { obj.webViewsPath = obj.path.join(__dirname, 'views'); } if (obj.fs.existsSync(obj.path.join(__dirname, '../meshcentral-web/public'))) { obj.webPublicPath = obj.path.join(__dirname, '../meshcentral-web/public'); } else { obj.webPublicPath = obj.path.join(__dirname, 'public'); } } @@ -506,7 +508,7 @@ function CreateMeshCentralServer(config, args) { ); }; - // Time to start the serverf or real. + // Time to start the server or real. obj.StartEx1b = function () { var i; @@ -865,6 +867,11 @@ function CreateMeshCentralServer(config, args) { //obj.debug(1, 'Server started'); if (obj.args.nousers == true) { obj.updateServerState('nousers', '1'); } obj.updateServerState('state', 'running'); + + // Setup database backup + if (obj.config.settings.autobackup && (typeof obj.config.settings.autobackup.backupinvervalhours == 'number')) { + setInterval(obj.db.performBackup, obj.config.settings.autobackup.backupinvervalhours * 60 * 60 * 1000); + } }); }); }; @@ -1615,10 +1622,7 @@ function CreateMeshCentralServer(config, args) { // Return the server configuration function getConfig(createSampleConfig) { // Figure out the datapath location - var i; - var fs = require('fs'); - var path = require('path'); - var datapath = null; + var i, fs = require('fs'), path = require('path'), datapath = null; var args = require('minimist')(process.argv.slice(2)); if ((__dirname.endsWith('/node_modules/meshcentral')) || (__dirname.endsWith('\\node_modules\\meshcentral')) || (__dirname.endsWith('/node_modules/meshcentral/')) || (__dirname.endsWith('\\node_modules\\meshcentral\\'))) { datapath = path.join(__dirname, '../../meshcentral-data'); @@ -1686,7 +1690,7 @@ function InstallModule(modulename, func, tag1, tag2) { // Looks like we need to keep a global reference to the child process object for this to work correctly. InstallModuleChildProcess = child_process.exec('npm install --no-optional --save ' + modulename, { maxBuffer: 512000, timeout: 10000, cwd: parentpath }, function (error, stdout, stderr) { InstallModuleChildProcess = null; - if (error != null) { + if ((error != null) && (error != '')) { console.log('ERROR: Unable to install required module "' + modulename + '". MeshCentral may not have access to npm, or npm may not have suffisent rights to load the new module. Try "npm install ' + modulename + '" to manualy install this module.\r\n'); process.exit(); return; @@ -1735,7 +1739,7 @@ function mainStart(args) { if (ldap == true) { modules.push('ldapauth-fork'); } if (config.letsencrypt != null) { modules.push('greenlock'); modules.push('le-store-certbot'); modules.push('le-challenge-fs'); modules.push('le-acme-core'); } // Add Greenlock Modules if (config.settings.mongodb != null) { modules.push('mongojs'); } // Add MongoJS - else if (config.settings.mongo != null) { modules.push('mongodb'); } // Add MongoDB + else if (config.settings.xmongodb != null) { modules.push('mongodb'); } // Add MongoDB if (config.smtp != null) { modules.push('nodemailer'); } // Add SMTP support // Get the current node version @@ -1744,6 +1748,9 @@ function mainStart(args) { // If running NodeJS < 8, install "util.promisify" if (nodeVersion < 8) { modules.push('util.promisify'); } + // Setup encrypted zip support if needed + if (config.settings.autobackup && config.settings.autobackup.zippassword) { modules.push('archiver-zip-encrypted'); } + // Setup 2nd factor authentication if (config.settings.no2factorauth !== true) { // Setup YubiKey OTP if configured diff --git a/package.json b/package.json index b410c477..84198f1a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "meshcentral", - "version": "0.3.4-p", + "version": "0.3.4-q", "keywords": [ "Remote Management", "Intel AMT", @@ -28,6 +28,7 @@ ], "dependencies": { "archiver": "^3.0.0", + "archiver-zip-encrypted": "^1.0.3", "body-parser": "^1.19.0", "cbor": "^4.1.5", "compression": "^1.7.4", diff --git a/sample-config.json b/sample-config.json index d1a6522a..9e4c457a 100644 --- a/sample-config.json +++ b/sample-config.json @@ -36,6 +36,11 @@ { "urls": "stun:stun.services.mozilla.com" }, { "urls": "stun:stun.l.google.com:19302" } ] + }, + "_AutoBackup": { + "backupInvervalHours": 24, + "keepLastDaysBackup": 10, + "zippassword": "MyReallySecretPassword3" } }, "_domains": {