diff --git a/db-test.js b/db-test.js new file mode 100644 index 00000000..040ba310 --- /dev/null +++ b/db-test.js @@ -0,0 +1,487 @@ +/** +* @description MeshCentral database module +* @author Ylian Saint-Hilaire +* @copyright Intel Corporation 2018-2019 +* @license Apache-2.0 +* @version v0.0.2 +*/ + +/*xjslint node: true */ +/*xjslint plusplus: true */ +/*xjslint maxlen: 256 */ +/*jshint node: true */ +/*jshint strict: false */ +/*jshint esversion: 6 */ +"use strict"; + +// +// Construct Meshcentral database object +// +// The default database is NeDB +// https://github.com/louischatriot/nedb +// +// Alternativety, MongoDB can be used +// https://www.mongodb.com/ +// Just run with --mongodb [connectionstring], where the connection string is documented here: https://docs.mongodb.com/manual/reference/connection-string/ +// The default collection is "meshcentral", but you can override it using --mongodbcol [collection] +// +module.exports.CreateDB = function (parent, func) { + var obj = {}; + var Datastore = null; + var expireEventsSeconds = (60 * 60 * 24 * 20); // By default, expire events after 20 days. (Seconds * Minutes * Hours * Days) + var expirePowerEventsSeconds = (60 * 60 * 24 * 10); // By default, expire power events after 10 days. (Seconds * Minutes * Hours * Days) + var expireServerStatsSeconds = (60 * 60 * 24 * 30); // By default, expire power events after 30 days. (Seconds * Minutes * Hours * Days) + obj.path = require('path'); + obj.parent = parent; + obj.identifier = null; + obj.dbKey = null; + + // Read expiration time from configuration file + if (typeof obj.parent.args.dbexpire == 'object') { + if (typeof obj.parent.args.dbexpire.events == 'number') { expireEventsSeconds = obj.parent.args.dbexpire.events; } + if (typeof obj.parent.args.dbexpire.powerevents == 'number') { expirePowerEventsSeconds = obj.parent.args.dbexpire.powerevents; } + if (typeof obj.parent.args.dbexpire.statsevents == 'number') { expireServerStatsSeconds = obj.parent.args.dbexpire.statsevents; } + } + + if (obj.parent.args.mongodb) { + // Use MongoDB + obj.databaseType = 2; + Datastore = require('mongodb').MongoClient; + Datastore.connect(obj.parent.args.mongodb, function (err, client) { + if (err != null) { console.log("Unable to connect to database: " + err); process.exit(); return; } + const db = client.db('meshcentral'); + + var dbcollection = 'meshcentral'; + if (obj.parent.args.mongodbcol) { dbcollection = obj.parent.args.mongodbcol; } + + // Setup MongoDB main collection and indexes + obj.file = db.collection(dbcollection); + + obj.file.find({ type: 'mesh' }, function (err, cursor) { + cursor.each(function (err, item) { + console.log(err, item); + }); + }); + + + /* + obj.file.getIndexes(function (err, indexes) { + // Check if we need to reset indexes + var indexesByName = {}, indexCount = 0; + for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; } + if ((indexCount != 4) || (indexesByName['TypeDomainMesh1'] == null) || (indexesByName['Email1'] == null) || (indexesByName['Mesh1'] == null)) { + console.log('Resetting main indexes...'); + obj.file.dropIndexes(function (err) { + obj.file.createIndex({ type: 1, domain: 1, meshid: 1 }, { sparse: 1, name: 'TypeDomainMesh1' }); // Speeds up GetAllTypeNoTypeField() and GetAllTypeNoTypeFieldMeshFiltered() + obj.file.createIndex({ email: 1 }, { sparse: 1, name: 'Email1' }); // Speeds up GetUserWithEmail() and GetUserWithVerifiedEmail() + obj.file.createIndex({ meshid: 1 }, { sparse: 1, name: 'Mesh1' }); // Speeds up RemoveMesh() + }); + } + }); + */ + + /* + // Setup the changeStream on the MongoDB main collection + obj.fileChangeStream = obj.file.watch(); + obj.fileChangeStream.on('change', function (next) { + // Process next document + console.log('change', next); + }); + */ + + // Setup MongoDB events collection and indexes + obj.eventsfile = db.collection('events'); // Collection containing all events + /* + obj.eventsfile.getIndexes(function (err, indexes) { + // Check if we need to reset indexes + var indexesByName = {}, indexCount = 0; + for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; } + if ((indexCount != 5) || (indexesByName['Username1'] == null) || (indexesByName['DomainNodeTime1'] == null) || (indexesByName['IdsAndTime1'] == null) || (indexesByName['ExpireTime1'] == null)) { + // Reset all indexes + console.log('Resetting events indexes...'); + obj.eventsfile.dropIndexes(function (err) { + obj.eventsfile.createIndex({ username: 1 }, { sparse: 1, name: 'Username1' }); + obj.eventsfile.createIndex({ domain: 1, nodeid: 1, time: -1 }, { sparse: 1, name: 'DomainNodeTime1' }); + obj.eventsfile.createIndex({ ids: 1, time: -1 }, { sparse: 1, name: 'IdsAndTime1' }); + obj.eventsfile.createIndex({ "time": 1 }, { expireAfterSeconds: expireEventsSeconds, name: 'ExpireTime1' }); + }); + } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expireEventsSeconds) { + // Reset the timeout index + console.log('Resetting events expire index...'); + obj.eventsfile.dropIndex("ExpireTime1", function (err) { + obj.eventsfile.createIndex({ "time": 1 }, { expireAfterSeconds: expireEventsSeconds, name: 'ExpireTime1' }); + }); + } + }); + */ + + // Setup MongoDB power events collection and indexes + obj.powerfile = db.collection('power'); // Collection containing all power events + /* + obj.powerfile.getIndexes(function (err, indexes) { + // Check if we need to reset indexes + var indexesByName = {}, indexCount = 0; + for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; } + if ((indexCount != 3) || (indexesByName['NodeIdAndTime1'] == null) || (indexesByName['ExpireTime1'] == null)) { + // Reset all indexes + console.log('Resetting power events indexes...'); + obj.powerfile.dropIndexes(function (err) { + // Create all indexes + obj.powerfile.createIndex({ nodeid: 1, time: 1 }, { sparse: 1, name: 'NodeIdAndTime1' }); + obj.powerfile.createIndex({ "time": 1 }, { expireAfterSeconds: expirePowerEventsSeconds, name: 'ExpireTime1' }); + }); + } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expirePowerEventsSeconds) { + // Reset the timeout index + console.log('Resetting power events expire index...'); + obj.powerfile.dropIndex("ExpireTime1", function (err) { + // Reset the expire power events index + obj.powerfile.createIndex({ "time": 1 }, { expireAfterSeconds: expirePowerEventsSeconds, name: 'ExpireTime1' }); + }); + } + }); + */ + + // Setup MongoDB smbios collection, no indexes needed + obj.smbiosfile = db.collection('smbios'); // Collection containing all smbios information + + // Setup MongoDB server stats collection + obj.serverstatsfile = db.collection('serverstats'); // Collection of server stats + /* + obj.serverstatsfile.getIndexes(function (err, indexes) { + // Check if we need to reset indexes + var indexesByName = {}, indexCount = 0; + for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; } + if ((indexCount != 3) || (indexesByName['ExpireTime1'] == null)) { + // Reset all indexes + console.log('Resetting server stats indexes...'); + obj.serverstatsfile.dropIndexes(function (err) { + // Create all indexes + obj.serverstatsfile.createIndex({ "time": 1 }, { expireAfterSeconds: expireServerStatsSeconds, name: 'ExpireTime1' }); + obj.serverstatsfile.createIndex({ "expire": 1 }, { expireAfterSeconds: 0, name: 'ExpireTime2' }); // Auto-expire events + }); + } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expireServerStatsSeconds) { + // Reset the timeout index + console.log('Resetting server stats expire index...'); + obj.serverstatsfile.dropIndex("ExpireTime1", function (err) { + // Reset the expire server stats index + obj.serverstatsfile.createIndex({ "time": 1 }, { expireAfterSeconds: expireServerStatsSeconds, name: 'ExpireTime1' }); + }); + } + }); + */ + func(); // Completed MongoDB setup + }); + } else { + // Use NeDB (The default) + obj.databaseType = 1; + Datastore = require('nedb'); + var datastoreOptions = { filename: obj.parent.getConfigFilePath('meshcentral.db'), autoload: true }; + + // If a DB encryption key is provided, perform database encryption + if ((typeof obj.parent.args.dbencryptkey == 'string') && (obj.parent.args.dbencryptkey.length != 0)) { + // Hash the database password into a AES256 key and setup encryption and decryption. + obj.dbKey = obj.parent.crypto.createHash('sha384').update(obj.parent.args.dbencryptkey).digest("raw").slice(0, 32); + datastoreOptions.afterSerialization = function (plaintext) { + const iv = obj.parent.crypto.randomBytes(16); + const aes = obj.parent.crypto.createCipheriv('aes-256-cbc', obj.dbKey, iv); + var ciphertext = aes.update(plaintext); + ciphertext = Buffer.concat([iv, ciphertext, aes.final()]); + return ciphertext.toString('base64'); + } + datastoreOptions.beforeDeserialization = function (ciphertext) { + const ciphertextBytes = Buffer.from(ciphertext, 'base64'); + const iv = ciphertextBytes.slice(0, 16); + const data = ciphertextBytes.slice(16); + const aes = obj.parent.crypto.createDecipheriv('aes-256-cbc', obj.dbKey, iv); + var plaintextBytes = Buffer.from(aes.update(data)); + plaintextBytes = Buffer.concat([plaintextBytes, aes.final()]); + return plaintextBytes.toString(); + } + } + + // Start NeDB main collection and setup indexes + obj.file = new Datastore(datastoreOptions); + obj.file.persistence.setAutocompactionInterval(36000); + obj.file.ensureIndex({ fieldName: 'type' }); + obj.file.ensureIndex({ fieldName: 'domain' }); + obj.file.ensureIndex({ fieldName: 'meshid', sparse: true }); + obj.file.ensureIndex({ fieldName: 'nodeid', sparse: true }); + obj.file.ensureIndex({ fieldName: 'email', sparse: true }); + + // Setup the events collection and setup indexes + obj.eventsfile = new Datastore({ filename: obj.parent.getConfigFilePath('meshcentral-events.db'), autoload: true }); + obj.eventsfile.persistence.setAutocompactionInterval(36000); + obj.eventsfile.ensureIndex({ fieldName: 'ids' }); // TODO: Not sure if this is a good index, this is a array field. + obj.eventsfile.ensureIndex({ fieldName: 'nodeid', sparse: true }); + obj.eventsfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: 60 * 60 * 24 * 20 }); // Limit the power event log to 20 days (Seconds * Minutes * Hours * Days) + + // Setup the power collection and setup indexes + obj.powerfile = new Datastore({ filename: obj.parent.getConfigFilePath('meshcentral-power.db'), autoload: true }); + obj.powerfile.persistence.setAutocompactionInterval(36000); + obj.powerfile.ensureIndex({ fieldName: 'nodeid' }); + obj.powerfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: 60 * 60 * 24 * 10 }); // Limit the power event log to 10 days (Seconds * Minutes * Hours * Days) + + // Setup the SMBIOS collection + obj.smbiosfile = new Datastore({ filename: obj.parent.getConfigFilePath('meshcentral-smbios.db'), autoload: true }); + + // Setup the server stats collection and setup indexes + obj.serverstatsfile = new Datastore({ filename: obj.parent.getConfigFilePath('meshcentral-stats.db'), autoload: true }); + obj.serverstatsfile.persistence.setAutocompactionInterval(36000); + obj.serverstatsfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: 60 * 60 * 24 * 30 }); // Limit the server stats log to 30 days (Seconds * Minutes * Hours * Days) + obj.serverstatsfile.ensureIndex({ fieldName: 'expire', expireAfterSeconds: 0 }); // Auto-expire events + + func(); // Completed NeDB setup + } + + obj.SetupDatabase = function (func) { + // Check if the database unique identifier is present + // This is used to check that in server peering mode, everyone is using the same database. + obj.Get('DatabaseIdentifier', function (err, docs) { + if ((docs.length == 1) && (docs[0].value != null)) { + obj.identifier = docs[0].value; + } else { + obj.identifier = Buffer.from(require('crypto').randomBytes(48), 'binary').toString('hex'); + obj.Set({ _id: 'DatabaseIdentifier', value: obj.identifier }); + } + }); + + // Load database schema version and check if we need to update + obj.Get('SchemaVersion', function (err, docs) { + var ver = 0; + if (docs && docs.length == 1) { ver = docs[0].value; } + if (ver == 1) { console.log('This is an unsupported beta 1 database, delete it to create a new one.'); process.exit(0); } + + // TODO: Any schema upgrades here... + obj.Set({ _id: 'SchemaVersion', value: 2 }); + + func(ver); + }); + }; + + obj.cleanup = function (func) { + // TODO: Remove all mesh links to invalid users + // TODO: Remove all meshes that dont have any links + + // Remove all events, power events and SMBIOS data from the main collection. They are all in seperate collections now. + obj.file.remove({ type: 'event' }, { multi: true }); + obj.file.remove({ type: 'power' }, { multi: true }); + obj.file.remove({ type: 'smbios' }, { multi: true }); + + // Remove all objects that have a "meshid" that no longer points to a valid mesh. + obj.GetAllType('mesh', function (err, docs) { + var meshlist = []; + if ((err == null) && (docs.length > 0)) { for (var i in docs) { meshlist.push(docs[i]._id); } } + obj.file.remove({ meshid: { $exists: true, $nin: meshlist } }, { multi: true }); + + // Fix all of the creating & login to ticks by seconds, not milliseconds. + obj.GetAllType('user', function (err, docs) { + if (err == null && docs.length > 0) { + for (var i in docs) { + var fixed = false; + + // Fix account creation + if (docs[i].creation) { + if (docs[i].creation > 1300000000000) { docs[i].creation = Math.floor(docs[i].creation / 1000); fixed = true; } + if ((docs[i].creation % 1) != 0) { docs[i].creation = Math.floor(docs[i].creation); fixed = true; } + } + + // Fix last account login + if (docs[i].login) { + if (docs[i].login > 1300000000000) { docs[i].login = Math.floor(docs[i].login / 1000); fixed = true; } + if ((docs[i].login % 1) != 0) { docs[i].login = Math.floor(docs[i].login); fixed = true; } + } + + // Fix last password change + if (docs[i].passchange) { + if (docs[i].passchange > 1300000000000) { docs[i].passchange = Math.floor(docs[i].passchange / 1000); fixed = true; } + if ((docs[i].passchange % 1) != 0) { docs[i].passchange = Math.floor(docs[i].passchange); fixed = true; } + } + + // Fix subscriptions + if (docs[i].subscriptions != null) { delete docs[i].subscriptions; fixed = true; } + + // Save the user if needed + if (fixed) { obj.Set(docs[i]); } + + // We are done + if (func) { func(); } + } + } + }); + }); + }; + + // Database actions on the main collection + obj.Set = function (data, func) { obj.file.update({ _id: data._id }, data, { upsert: true }, func); }; + obj.Get = function (id, func) + { + if (arguments.length > 2) + { + var parms = [func]; + for (var parmx = 2; parmx < arguments.length; ++parmx) { parms.push(arguments[parmx]); } + var func2 = function _func2(arg1, arg2) + { + var userCallback = _func2.userArgs.shift(); + _func2.userArgs.unshift(arg2); + _func2.userArgs.unshift(arg1); + userCallback.apply(obj, _func2.userArgs); + }; + func2.userArgs = parms; + obj.file.find({ _id: id }, func2); + } + else + { + obj.file.find({ _id: id }, func); + } + }; + obj.GetAll = function (func) { obj.file.find({}, func); }; + obj.GetAllTypeNoTypeField = function (type, domain, func) { obj.file.find({ type: type, domain: domain }, { type: 0 }, func); }; + obj.GetAllTypeNoTypeFieldMeshFiltered = function (meshes, domain, type, id, func) { var x = { type: type, domain: domain, meshid: { $in: meshes } }; if (id) { x._id = id; } obj.file.find(x, { type: 0 }, func); }; + //obj.GetAllType = function (type, func) { obj.file.find({ type: type }, func); }; + + obj.GetAllType = function (type, func) { obj.file.find({ type: type }, function (err, cursor) { if (err) { func(err); } else { var r = []; cursor.each(function (err, item) { if (err) { func(err); } else { if (item) { r.push(item); } else { func(null, r); } } }); } }); }; + + + + obj.GetAllIdsOfType = function (ids, domain, type, func) { obj.file.find({ type: type, domain: domain, _id: { $in: ids } }, func); }; + obj.GetUserWithEmail = function (domain, email, func) { obj.file.find({ type: 'user', domain: domain, email: email }, { type: 0 }, func); }; + obj.GetUserWithVerifiedEmail = function (domain, email, func) { obj.file.find({ type: 'user', domain: domain, email: email, emailVerified: true }, { type: 0 }, func); }; + obj.Remove = function (id) { obj.file.remove({ _id: id }); }; + obj.RemoveAll = function (func) { obj.file.remove({}, { multi: true }, func); }; + obj.RemoveAllOfType = function (type, func) { obj.file.remove({ type: type }, { multi: true }, func); }; + obj.InsertMany = function (data, func) { obj.file.insert(data, func); }; + obj.RemoveMeshDocuments = function (id) { obj.file.remove({ meshid: id }, { multi: true }); obj.file.remove({ _id: 'nt' + id }); }; + obj.MakeSiteAdmin = function (username, domain) { obj.Get('user/' + domain + '/' + username, function (err, docs) { if (docs.length == 1) { docs[0].siteadmin = 0xFFFFFFFF; obj.Set(docs[0]); } }); }; + obj.DeleteDomain = function (domain, func) { obj.file.remove({ domain: domain }, { multi: true }, func); }; + obj.SetUser = function (user) { var u = Clone(user); if (u.subscriptions) { delete u.subscriptions; } obj.Set(u); }; + obj.dispose = function () { for (var x in obj) { if (obj[x].close) { obj[x].close(); } delete obj[x]; } }; + obj.getLocalAmtNodes = function (func) { obj.file.find({ type: 'node', host: { $exists: true, $ne: null }, intelamt: { $exists: true } }, func); }; + obj.getAmtUuidNode = function (meshid, uuid, func) { obj.file.find({ type: 'node', meshid: meshid, 'intelamt.uuid': uuid }, func); }; + obj.isMaxType = function (max, type, domainid, func) { if (max == null) { func(false); } else { obj.file.count({ type: type, domain: domainid }, function (err, count) { func((err != null) || (count > max)); }); } } + + // Database actions on the events collection + obj.GetAllEvents = function (func) { obj.eventsfile.find({}, func); }; + obj.StoreEvent = function (event) { obj.eventsfile.insert(event); }; + obj.GetEvents = function (ids, domain, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).exec(func); } else { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }, func); } }; + obj.GetEventsWithLimit = function (ids, domain, limit, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).exec(func); } else { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit, func); } }; + obj.GetUserEvents = function (ids, domain, username, func) { + if (obj.databaseType == 1) { + obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).exec(func); + } else { + obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }, func); + } + }; + obj.GetUserEventsWithLimit = function (ids, domain, username, limit, func) { + if (obj.databaseType == 1) { + obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).exec(func); + } else { + obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit, func); + } + }; + obj.GetNodeEventsWithLimit = function (nodeid, domain, limit, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, nodeid: nodeid }, { type: 0, etype: 0, _id: 0, domain: 0, ids: 0, node: 0, nodeid: 0 }).sort({ time: -1 }).limit(limit).exec(func); } else { obj.eventsfile.find({ domain: domain, nodeid: nodeid }, { type: 0, etype: 0, _id: 0, domain: 0, ids: 0, node: 0, nodeid: 0 }).sort({ time: -1 }).limit(limit, func); } }; + obj.RemoveAllEvents = function (domain) { obj.eventsfile.remove({ domain: domain }, { multi: true }); }; + obj.RemoveAllNodeEvents = function (domain, nodeid) { obj.eventsfile.remove({ domain: domain, nodeid: nodeid }, { multi: true }); }; + + // Database actions on the power collection + obj.getAllPower = function (func) { obj.powerfile.find({}, func); }; + obj.storePowerEvent = function (event, multiServer, func) { if (multiServer != null) { event.server = multiServer.serverid; } obj.powerfile.insert(event, func); }; + obj.getPowerTimeline = function (nodeid, func) { if (obj.databaseType == 1) { obj.powerfile.find({ nodeid: { $in: ['*', nodeid] } }, { _id: 0, nodeid: 0, s: 0 }).sort({ time: 1 }).exec(func); } else { obj.powerfile.find({ nodeid: { $in: ['*', nodeid] } }, { _id: 0, nodeid: 0, s: 0 }).sort({ time: 1 }, func); } }; + obj.removeAllPowerEvents = function () { obj.powerfile.remove({}, { multi: true }); }; + obj.removeAllPowerEventsForNode = function (nodeid) { obj.powerfile.remove({ nodeid: nodeid }, { multi: true }); }; + + // Database actions on the SMBIOS collection + obj.SetSMBIOS = function (smbios, func) { obj.smbiosfile.update({ _id: smbios._id }, smbios, { upsert: true }, func); }; + obj.RemoveSMBIOS = function (id) { obj.smbiosfile.remove({ _id: id }); }; + obj.GetSMBIOS = function (id, func) { obj.smbiosfile.find({ _id: id }, func); }; + + // Database actions on the Server Stats collection + obj.SetServerStats = function (data, func) { obj.serverstatsfile.insert(data, func); }; + obj.GetServerStats = function (hours, func) { var t = new Date(); t.setTime(t.getTime() - (60 * 60 * 1000 * hours)); obj.serverstatsfile.find({ time: { $gt: t } }, { _id: 0, cpu: 0 }, func); }; + + // Read a configuration file from the database + obj.getConfigFile = function (path, func) { obj.Get('cfile/' + path, func); } + + // Write a configuration file to the database + obj.setConfigFile = function (path, data, func) { obj.Set({ _id: 'cfile/' + path, type: 'cfile', data: data.toString('base64') }, func); } + + // List all configuration files + obj.listConfigFiles = function (func) { obj.file.find({ type: 'cfile' }).sort({ _id: 1 }).exec(func); } + + // Get all configuration files + obj.getAllConfigFiles = function (password, func) { + obj.file.find({ type: 'cfile' }, function (err, docs) { + if (err != null) { func(null); return; } + var r = null; + for (var i = 0; i < docs.length; i++) { + var name = docs[i]._id.split('/')[1]; + var data = obj.decryptData(password, docs[i].data); + if (data != null) { if (r == null) { r = {}; } r[name] = data; } + } + func(r); + }); + } + + // Get encryption key + obj.getEncryptDataKey = function (password) { + if (typeof password != 'string') return null; + return obj.parent.crypto.createHash('sha384').update(password).digest("raw").slice(0, 32); + } + + // Encrypt data + obj.encryptData = function (password, plaintext) { + var key = obj.getEncryptDataKey(password); + if (key == null) return null; + const iv = obj.parent.crypto.randomBytes(16); + const aes = obj.parent.crypto.createCipheriv('aes-256-cbc', key, iv); + var ciphertext = aes.update(plaintext); + ciphertext = Buffer.concat([iv, ciphertext, aes.final()]); + return ciphertext.toString('base64'); + } + + // Decrypt data + obj.decryptData = function (password, ciphertext) { + try { + var key = obj.getEncryptDataKey(password); + if (key == null) return null; + const ciphertextBytes = Buffer.from(ciphertext, 'base64'); + const iv = ciphertextBytes.slice(0, 16); + const data = ciphertextBytes.slice(16); + const aes = obj.parent.crypto.createDecipheriv('aes-256-cbc', key, iv); + var plaintextBytes = Buffer.from(aes.update(data)); + plaintextBytes = Buffer.concat([plaintextBytes, aes.final()]); + return plaintextBytes; + } catch (ex) { return null; } + } + + // Get the number of records in the database for various types, this is the slow NeDB way. + // WARNING: This is a terrible query for database performance. Only do this when needed. This query will look at almost every document in the database. + obj.getStats = function (func) { + if (obj.databaseType == 2) { + // MongoDB version + obj.file.aggregate([{ "$group": { _id: "$type", count: { $sum: 1 } } }], function (err, docs) { + var counters = {}, totalCount = 0; + for (var i in docs) { if (docs[i]._id != null) { counters[docs[i]._id] = docs[i].count; totalCount += docs[i].count; } } + func({ nodes: counters['node'], meshes: counters['mesh'], users: counters['user'], total: totalCount }); + }) + } else { + // NeDB version + obj.file.count({ type: 'node' }, function (err, nodeCount) { + obj.file.count({ type: 'mesh' }, function (err, meshCount) { + obj.file.count({ type: 'user' }, function (err, userCount) { + obj.file.count({}, function (err, totalCount) { + func({ nodes: nodeCount, meshes: meshCount, users: userCount, total: totalCount }); + }); + }); + }); + }); + } + } + + // This is used to rate limit a number of operation per day. Returns a startValue each new days, but you can substract it and save the value in the db. + obj.getValueOfTheDay = function (id, startValue, func) { obj.Get(id, function (err, docs) { var date = new Date(), t = date.toLocaleDateString(); if (docs.length == 1) { var r = docs[0]; if (r.day == t) { func({ _id: id, value: r.value, day: t }); return; } } func({ _id: id, value: startValue, day: t }); }); }; + obj.escapeBase64 = function escapeBase64(val) { return (val.replace(/\+/g, '@').replace(/\//g, '$')); } + + function Clone(v) { return JSON.parse(JSON.stringify(v)); } + + return obj; +}; \ No newline at end of file diff --git a/db.js b/db.js index ae81441e..e70f8465 100644 --- a/db.js +++ b/db.js @@ -25,7 +25,7 @@ // Just run with --mongodb [connectionstring], where the connection string is documented here: https://docs.mongodb.com/manual/reference/connection-string/ // The default collection is "meshcentral", but you can override it using --mongodbcol [collection] // -module.exports.CreateDB = function (parent) { +module.exports.CreateDB = function (parent, func) { var obj = {}; var Datastore = null; var expireEventsSeconds = (60 * 60 * 24 * 20); // By default, expire events after 20 days. (Seconds * Minutes * Hours * Days) @@ -36,6 +36,347 @@ module.exports.CreateDB = function (parent) { obj.identifier = null; obj.dbKey = null; + obj.SetupDatabase = function (func) { + // Check if the database unique identifier is present + // This is used to check that in server peering mode, everyone is using the same database. + obj.Get('DatabaseIdentifier', function (err, docs) { + if ((docs.length == 1) && (docs[0].value != null)) { + obj.identifier = docs[0].value; + } else { + obj.identifier = Buffer.from(require('crypto').randomBytes(48), 'binary').toString('hex'); + obj.Set({ _id: 'DatabaseIdentifier', value: obj.identifier }); + } + }); + + // Load database schema version and check if we need to update + obj.Get('SchemaVersion', function (err, docs) { + var ver = 0; + if (docs && docs.length == 1) { ver = docs[0].value; } + if (ver == 1) { console.log('This is an unsupported beta 1 database, delete it to create a new one.'); process.exit(0); } + + // TODO: Any schema upgrades here... + obj.Set({ _id: 'SchemaVersion', value: 2 }); + + func(ver); + }); + }; + + obj.cleanup = function (func) { + // TODO: Remove all mesh links to invalid users + // TODO: Remove all meshes that dont have any links + + // Remove all events, power events and SMBIOS data from the main collection. They are all in seperate collections now. + obj.file.remove({ type: 'event' }, { multi: true }); + obj.file.remove({ type: 'power' }, { multi: true }); + obj.file.remove({ type: 'smbios' }, { multi: true }); + + // Remove all objects that have a "meshid" that no longer points to a valid mesh. + obj.GetAllType('mesh', function (err, docs) { + var meshlist = []; + if ((err == null) && (docs.length > 0)) { for (var i in docs) { meshlist.push(docs[i]._id); } } + obj.file.remove({ meshid: { $exists: true, $nin: meshlist } }, { multi: true }); + + // Fix all of the creating & login to ticks by seconds, not milliseconds. + obj.GetAllType('user', function (err, docs) { + if (err == null && docs.length > 0) { + for (var i in docs) { + var fixed = false; + + // Fix account creation + if (docs[i].creation) { + if (docs[i].creation > 1300000000000) { docs[i].creation = Math.floor(docs[i].creation / 1000); fixed = true; } + if ((docs[i].creation % 1) != 0) { docs[i].creation = Math.floor(docs[i].creation); fixed = true; } + } + + // Fix last account login + if (docs[i].login) { + if (docs[i].login > 1300000000000) { docs[i].login = Math.floor(docs[i].login / 1000); fixed = true; } + if ((docs[i].login % 1) != 0) { docs[i].login = Math.floor(docs[i].login); fixed = true; } + } + + // Fix last password change + if (docs[i].passchange) { + if (docs[i].passchange > 1300000000000) { docs[i].passchange = Math.floor(docs[i].passchange / 1000); fixed = true; } + if ((docs[i].passchange % 1) != 0) { docs[i].passchange = Math.floor(docs[i].passchange); fixed = true; } + } + + // Fix subscriptions + if (docs[i].subscriptions != null) { delete docs[i].subscriptions; fixed = true; } + + // Save the user if needed + if (fixed) { obj.Set(docs[i]); } + + // We are done + if (func) { func(); } + } + } + }); + }); + }; + + if (obj.databaseType == 3) { + // Database actions on the main collection (MongoDB) + function xfind(collection, query, projection, func) { + if (projection) { + collection.find(query, projection, function (err, cursor) { if (err) { func(err); } else { var r = []; cursor.each(function (err, item) { if (err) { func(err); } else { if (item) { r.push(item); } else { func(null, r); } } }); } }); + } else { + collection.find(query, function (err, cursor) { if (err) { func(err); } else { var r = []; cursor.each(function (err, item) { if (err) { func(err); } else { if (item) { r.push(item); } else { func(null, r); } } }); } }); + } + }; + + obj.Set = function (data, func) { obj.file.update({ _id: data._id }, data, { upsert: true }, func); }; + obj.Get = function (id, func) { + if (arguments.length > 2) { + var parms = [func]; + for (var parmx = 2; parmx < arguments.length; ++parmx) { parms.push(arguments[parmx]); } + var func2 = function _func2(arg1, arg2) { + var userCallback = _func2.userArgs.shift(); + _func2.userArgs.unshift(arg2); + _func2.userArgs.unshift(arg1); + userCallback.apply(obj, _func2.userArgs); + }; + func2.userArgs = parms; + xfind(obj.file, { _id: id }, null, func2); + } else { + xfind(obj.file, { _id: id }, null, func); + } + }; + obj.GetAll = function (func) { xfind(obj.file, {}, null, func); }; + obj.GetAllTypeNoTypeField = function (type, domain, func) { xfind(obj.file, { type: type, domain: domain }, { type: 0 }, func); }; + obj.GetAllTypeNoTypeFieldMeshFiltered = function (meshes, domain, type, id, func) { var x = { type: type, domain: domain, meshid: { $in: meshes } }; if (id) { x._id = id; } xfind(obj.file, x, { type: 0 }, func); }; + obj.GetAllType = function (type, func) { xfind(obj.file, { type: type }, null, func); }; + obj.GetAllIdsOfType = function (ids, domain, type, func) { xfind(obj.file, { type: type, domain: domain, _id: { $in: ids } }, func); }; + obj.GetUserWithEmail = function (domain, email, func) { xfind(obj.file, { type: 'user', domain: domain, email: email }, { type: 0 }, func); }; + obj.GetUserWithVerifiedEmail = function (domain, email, func) { xfind(obj.file, { type: 'user', domain: domain, email: email, emailVerified: true }, { type: 0 }, func); }; + obj.Remove = function (id) { obj.file.remove({ _id: id }); }; + obj.RemoveAll = function (func) { obj.file.remove({}, { multi: true }, func); }; + obj.RemoveAllOfType = function (type, func) { obj.file.remove({ type: type }, { multi: true }, func); }; + obj.InsertMany = function (data, func) { obj.file.insert(data, func); }; + obj.RemoveMeshDocuments = function (id) { obj.file.remove({ meshid: id }, { multi: true }); obj.file.remove({ _id: 'nt' + id }); }; + obj.MakeSiteAdmin = function (username, domain) { obj.Get('user/' + domain + '/' + username, function (err, docs) { if (docs.length == 1) { docs[0].siteadmin = 0xFFFFFFFF; obj.Set(docs[0]); } }); }; + obj.DeleteDomain = function (domain, func) { obj.file.remove({ domain: domain }, { multi: true }, func); }; + obj.SetUser = function (user) { var u = Clone(user); if (u.subscriptions) { delete u.subscriptions; } obj.Set(u); }; + obj.dispose = function () { for (var x in obj) { if (obj[x].close) { obj[x].close(); } delete obj[x]; } }; + obj.getLocalAmtNodes = function (func) { xfind(obj.file, { type: 'node', host: { $exists: true, $ne: null }, intelamt: { $exists: true } }, func); }; + obj.getAmtUuidNode = function (meshid, uuid, func) { xfind(obj.file, { type: 'node', meshid: meshid, 'intelamt.uuid': uuid }, func); }; + obj.isMaxType = function (max, type, domainid, func) { if (max == null) { func(false); } else { obj.file.count({ type: type, domain: domainid }, function (err, count) { func((err != null) || (count > max)); }); } } + + // Database actions on the events collection + obj.GetAllEvents = function (func) { xfind(obj.eventsfile, {}, func); }; + obj.StoreEvent = function (event) { obj.eventsfile.insert(event); }; + obj.GetEvents = function (ids, domain, func) { if (obj.databaseType == 1) { xfind(obj.eventsfile, { domain: domain, ids: { $in: ids } }, { _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).exec(func); } else { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }, func); } }; + obj.GetEventsWithLimit = function (ids, domain, limit, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).exec(func); } else { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit, func); } }; + obj.GetUserEvents = function (ids, domain, username, func) { xfind(obj.eventsfile, { domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).exec(func); }; + obj.GetUserEventsWithLimit = function (ids, domain, username, limit, func) { xfind(obj.eventsfile, { domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).exec(func); }; + obj.GetNodeEventsWithLimit = function (nodeid, domain, limit, func) { xfind(obj.eventsfile, { domain: domain, nodeid: nodeid }, { type: 0, etype: 0, _id: 0, domain: 0, ids: 0, node: 0, nodeid: 0 }).sort({ time: -1 }).limit(limit).exec(func); }; + obj.RemoveAllEvents = function (domain) { obj.eventsfile.remove({ domain: domain }, { multi: true }); }; + obj.RemoveAllNodeEvents = function (domain, nodeid) { obj.eventsfile.remove({ domain: domain, nodeid: nodeid }, { multi: true }); }; + + // Database actions on the power collection + obj.getAllPower = function (func) { xfind(obj.powerfile, {}, func); }; + obj.storePowerEvent = function (event, multiServer, func) { if (multiServer != null) { event.server = multiServer.serverid; } obj.powerfile.insert(event, func); }; + obj.getPowerTimeline = function (nodeid, func) { xfind(obj.powerfile, { nodeid: { $in: ['*', nodeid] } }, { _id: 0, nodeid: 0, s: 0 }).sort({ time: 1 }).exec(func); }; + obj.removeAllPowerEvents = function () { obj.powerfile.remove({}, { multi: true }); }; + obj.removeAllPowerEventsForNode = function (nodeid) { obj.powerfile.remove({ nodeid: nodeid }, { multi: true }); }; + + // Database actions on the SMBIOS collection + obj.SetSMBIOS = function (smbios, func) { obj.smbiosfile.update({ _id: smbios._id }, smbios, { upsert: true }, func); }; + obj.RemoveSMBIOS = function (id) { obj.smbiosfile.remove({ _id: id }); }; + obj.GetSMBIOS = function (id, func) { xfind(obj.smbiosfile, { _id: id }, func); }; + + // Database actions on the Server Stats collection + obj.SetServerStats = function (data, func) { obj.serverstatsfile.insert(data, func); }; + obj.GetServerStats = function (hours, func) { var t = new Date(); t.setTime(t.getTime() - (60 * 60 * 1000 * hours)); xfind(obj.serverstatsfile, { time: { $gt: t } }, { _id: 0, cpu: 0 }, func); }; + + // Read a configuration file from the database + obj.getConfigFile = function (path, func) { obj.Get('cfile/' + path, func); } + + // Write a configuration file to the database + obj.setConfigFile = function (path, data, func) { obj.Set({ _id: 'cfile/' + path, type: 'cfile', data: data.toString('base64') }, func); } + + // List all configuration files + obj.listConfigFiles = function (func) { xfind(obj.file, { type: 'cfile' }).sort({ _id: 1 }).exec(func); } + + // Get all configuration files + obj.getAllConfigFiles = function (password, func) { + xfind(obj.file, { type: 'cfile' }, function (err, docs) { + if (err != null) { func(null); return; } + var r = null; + for (var i = 0; i < docs.length; i++) { + var name = docs[i]._id.split('/')[1]; + var data = obj.decryptData(password, docs[i].data); + if (data != null) { if (r == null) { r = {}; } r[name] = data; } + } + func(r); + }); + } + } else { + // Database actions on the main collection (NeDB and MongoJS) + obj.Set = function (data, func) { obj.file.update({ _id: data._id }, data, { upsert: true }, func); }; + obj.Get = function (id, func) { + if (arguments.length > 2) { + var parms = [func]; + for (var parmx = 2; parmx < arguments.length; ++parmx) { parms.push(arguments[parmx]); } + var func2 = function _func2(arg1, arg2) { + var userCallback = _func2.userArgs.shift(); + _func2.userArgs.unshift(arg2); + _func2.userArgs.unshift(arg1); + userCallback.apply(obj, _func2.userArgs); + }; + func2.userArgs = parms; + obj.file.find({ _id: id }, func2); + } + else { + obj.file.find({ _id: id }, func); + } + }; + obj.GetAll = function (func) { obj.file.find({}, func); }; + obj.GetAllTypeNoTypeField = function (type, domain, func) { obj.file.find({ type: type, domain: domain }, { type: 0 }, func); }; + obj.GetAllTypeNoTypeFieldMeshFiltered = function (meshes, domain, type, id, func) { var x = { type: type, domain: domain, meshid: { $in: meshes } }; if (id) { x._id = id; } obj.file.find(x, { type: 0 }, func); }; + obj.GetAllType = function (type, func) { obj.file.find({ type: type }, func); }; + obj.GetAllIdsOfType = function (ids, domain, type, func) { obj.file.find({ type: type, domain: domain, _id: { $in: ids } }, func); }; + obj.GetUserWithEmail = function (domain, email, func) { obj.file.find({ type: 'user', domain: domain, email: email }, { type: 0 }, func); }; + obj.GetUserWithVerifiedEmail = function (domain, email, func) { obj.file.find({ type: 'user', domain: domain, email: email, emailVerified: true }, { type: 0 }, func); }; + obj.Remove = function (id) { obj.file.remove({ _id: id }); }; + obj.RemoveAll = function (func) { obj.file.remove({}, { multi: true }, func); }; + obj.RemoveAllOfType = function (type, func) { obj.file.remove({ type: type }, { multi: true }, func); }; + obj.InsertMany = function (data, func) { obj.file.insert(data, func); }; + obj.RemoveMeshDocuments = function (id) { obj.file.remove({ meshid: id }, { multi: true }); obj.file.remove({ _id: 'nt' + id }); }; + obj.MakeSiteAdmin = function (username, domain) { obj.Get('user/' + domain + '/' + username, function (err, docs) { if (docs.length == 1) { docs[0].siteadmin = 0xFFFFFFFF; obj.Set(docs[0]); } }); }; + obj.DeleteDomain = function (domain, func) { obj.file.remove({ domain: domain }, { multi: true }, func); }; + obj.SetUser = function (user) { var u = Clone(user); if (u.subscriptions) { delete u.subscriptions; } obj.Set(u); }; + obj.dispose = function () { for (var x in obj) { if (obj[x].close) { obj[x].close(); } delete obj[x]; } }; + obj.getLocalAmtNodes = function (func) { obj.file.find({ type: 'node', host: { $exists: true, $ne: null }, intelamt: { $exists: true } }, func); }; + obj.getAmtUuidNode = function (meshid, uuid, func) { obj.file.find({ type: 'node', meshid: meshid, 'intelamt.uuid': uuid }, func); }; + obj.isMaxType = function (max, type, domainid, func) { if (max == null) { func(false); } else { obj.file.count({ type: type, domain: domainid }, function (err, count) { func((err != null) || (count > max)); }); } } + + // Database actions on the events collection + obj.GetAllEvents = function (func) { obj.eventsfile.find({}, func); }; + obj.StoreEvent = function (event) { obj.eventsfile.insert(event); }; + obj.GetEvents = function (ids, domain, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).exec(func); } else { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }, func); } }; + obj.GetEventsWithLimit = function (ids, domain, limit, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).exec(func); } else { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit, func); } }; + obj.GetUserEvents = function (ids, domain, username, func) { + if (obj.databaseType == 1) { + obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).exec(func); + } else { + obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }, func); + } + }; + obj.GetUserEventsWithLimit = function (ids, domain, username, limit, func) { + if (obj.databaseType == 1) { + obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).exec(func); + } else { + obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit, func); + } + }; + obj.GetNodeEventsWithLimit = function (nodeid, domain, limit, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, nodeid: nodeid }, { type: 0, etype: 0, _id: 0, domain: 0, ids: 0, node: 0, nodeid: 0 }).sort({ time: -1 }).limit(limit).exec(func); } else { obj.eventsfile.find({ domain: domain, nodeid: nodeid }, { type: 0, etype: 0, _id: 0, domain: 0, ids: 0, node: 0, nodeid: 0 }).sort({ time: -1 }).limit(limit, func); } }; + obj.RemoveAllEvents = function (domain) { obj.eventsfile.remove({ domain: domain }, { multi: true }); }; + obj.RemoveAllNodeEvents = function (domain, nodeid) { obj.eventsfile.remove({ domain: domain, nodeid: nodeid }, { multi: true }); }; + + // Database actions on the power collection + obj.getAllPower = function (func) { obj.powerfile.find({}, func); }; + obj.storePowerEvent = function (event, multiServer, func) { if (multiServer != null) { event.server = multiServer.serverid; } obj.powerfile.insert(event, func); }; + obj.getPowerTimeline = function (nodeid, func) { if (obj.databaseType == 1) { obj.powerfile.find({ nodeid: { $in: ['*', nodeid] } }, { _id: 0, nodeid: 0, s: 0 }).sort({ time: 1 }).exec(func); } else { obj.powerfile.find({ nodeid: { $in: ['*', nodeid] } }, { _id: 0, nodeid: 0, s: 0 }).sort({ time: 1 }, func); } }; + obj.removeAllPowerEvents = function () { obj.powerfile.remove({}, { multi: true }); }; + obj.removeAllPowerEventsForNode = function (nodeid) { obj.powerfile.remove({ nodeid: nodeid }, { multi: true }); }; + + // Database actions on the SMBIOS collection + obj.SetSMBIOS = function (smbios, func) { obj.smbiosfile.update({ _id: smbios._id }, smbios, { upsert: true }, func); }; + obj.RemoveSMBIOS = function (id) { obj.smbiosfile.remove({ _id: id }); }; + obj.GetSMBIOS = function (id, func) { obj.smbiosfile.find({ _id: id }, func); }; + + // Database actions on the Server Stats collection + obj.SetServerStats = function (data, func) { obj.serverstatsfile.insert(data, func); }; + obj.GetServerStats = function (hours, func) { var t = new Date(); t.setTime(t.getTime() - (60 * 60 * 1000 * hours)); obj.serverstatsfile.find({ time: { $gt: t } }, { _id: 0, cpu: 0 }, func); }; + + // Read a configuration file from the database + obj.getConfigFile = function (path, func) { obj.Get('cfile/' + path, func); } + + // Write a configuration file to the database + obj.setConfigFile = function (path, data, func) { obj.Set({ _id: 'cfile/' + path, type: 'cfile', data: data.toString('base64') }, func); } + + // List all configuration files + obj.listConfigFiles = function (func) { obj.file.find({ type: 'cfile' }).sort({ _id: 1 }).exec(func); } + + // Get all configuration files + obj.getAllConfigFiles = function (password, func) { + obj.file.find({ type: 'cfile' }, function (err, docs) { + if (err != null) { func(null); return; } + var r = null; + for (var i = 0; i < docs.length; i++) { + var name = docs[i]._id.split('/')[1]; + var data = obj.decryptData(password, docs[i].data); + if (data != null) { if (r == null) { r = {}; } r[name] = data; } + } + func(r); + }); + } + } + + + // Get encryption key + obj.getEncryptDataKey = function (password) { + if (typeof password != 'string') return null; + return obj.parent.crypto.createHash('sha384').update(password).digest("raw").slice(0, 32); + } + + // Encrypt data + obj.encryptData = function (password, plaintext) { + var key = obj.getEncryptDataKey(password); + if (key == null) return null; + const iv = obj.parent.crypto.randomBytes(16); + const aes = obj.parent.crypto.createCipheriv('aes-256-cbc', key, iv); + var ciphertext = aes.update(plaintext); + ciphertext = Buffer.concat([iv, ciphertext, aes.final()]); + return ciphertext.toString('base64'); + } + + // Decrypt data + obj.decryptData = function (password, ciphertext) { + try { + var key = obj.getEncryptDataKey(password); + if (key == null) return null; + const ciphertextBytes = Buffer.from(ciphertext, 'base64'); + const iv = ciphertextBytes.slice(0, 16); + const data = ciphertextBytes.slice(16); + const aes = obj.parent.crypto.createDecipheriv('aes-256-cbc', key, iv); + var plaintextBytes = Buffer.from(aes.update(data)); + plaintextBytes = Buffer.concat([plaintextBytes, aes.final()]); + return plaintextBytes; + } catch (ex) { return null; } + } + + // Get the number of records in the database for various types, this is the slow NeDB way. + // WARNING: This is a terrible query for database performance. Only do this when needed. This query will look at almost every document in the database. + obj.getStats = function (func) { + if (obj.databaseType == 2) { + // MongoDB version + obj.file.aggregate([{ "$group": { _id: "$type", count: { $sum: 1 } } }], function (err, docs) { + var counters = {}, totalCount = 0; + for (var i in docs) { if (docs[i]._id != null) { counters[docs[i]._id] = docs[i].count; totalCount += docs[i].count; } } + func({ nodes: counters['node'], meshes: counters['mesh'], users: counters['user'], total: totalCount }); + }) + } else { + // NeDB version + obj.file.count({ type: 'node' }, function (err, nodeCount) { + obj.file.count({ type: 'mesh' }, function (err, meshCount) { + obj.file.count({ type: 'user' }, function (err, userCount) { + obj.file.count({}, function (err, totalCount) { + func({ nodes: nodeCount, meshes: meshCount, users: userCount, total: totalCount }); + }); + }); + }); + }); + } + } + + // This is used to rate limit a number of operation per day. Returns a startValue each new days, but you can substract it and save the value in the db. + obj.getValueOfTheDay = function (id, startValue, func) { obj.Get(id, function (err, docs) { var date = new Date(), t = date.toLocaleDateString(); if (docs.length == 1) { var r = docs[0]; if (r.day == t) { func({ _id: id, value: r.value, day: t }); return; } } func({ _id: id, value: startValue, day: t }); }); }; + obj.escapeBase64 = function escapeBase64(val) { return (val.replace(/\+/g, '@').replace(/\//g, '$')); } + + function Clone(v) { return JSON.parse(JSON.stringify(v)); } + + // Read expiration time from configuration file if (typeof obj.parent.args.dbexpire == 'object') { if (typeof obj.parent.args.dbexpire.events == 'number') { expireEventsSeconds = obj.parent.args.dbexpire.events; } @@ -43,8 +384,132 @@ module.exports.CreateDB = function (parent) { if (typeof obj.parent.args.dbexpire.statsevents == 'number') { expireServerStatsSeconds = obj.parent.args.dbexpire.statsevents; } } - if (obj.parent.args.mongodb) { + if (obj.parent.args.mongo) { // Use MongoDB + obj.databaseType = 3; + Datastore = require('mongodb').MongoClient; + Datastore.connect(obj.parent.args.mongo, function (err, client) { + if (err != null) { console.log("Unable to connect to database: " + err); process.exit(); return; } + + var dbname = 'meshcentral'; + if (obj.parent.args.mongodbname) { dbname = obj.parent.args.mongodbname; } + const db = client.db(dbname); + + var dbcollection = 'meshcentral'; + if (obj.parent.args.mongodbcol) { dbcollection = obj.parent.args.mongodbcol; } + + // Setup MongoDB main collection and indexes + obj.file = db.collection(dbcollection); + /* + obj.file.getIndexes(function (err, indexes) { + // Check if we need to reset indexes + var indexesByName = {}, indexCount = 0; + for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; } + if ((indexCount != 4) || (indexesByName['TypeDomainMesh1'] == null) || (indexesByName['Email1'] == null) || (indexesByName['Mesh1'] == null)) { + console.log('Resetting main indexes...'); + obj.file.dropIndexes(function (err) { + obj.file.createIndex({ type: 1, domain: 1, meshid: 1 }, { sparse: 1, name: 'TypeDomainMesh1' }); // Speeds up GetAllTypeNoTypeField() and GetAllTypeNoTypeFieldMeshFiltered() + obj.file.createIndex({ email: 1 }, { sparse: 1, name: 'Email1' }); // Speeds up GetUserWithEmail() and GetUserWithVerifiedEmail() + obj.file.createIndex({ meshid: 1 }, { sparse: 1, name: 'Mesh1' }); // Speeds up RemoveMesh() + }); + } + }); + */ + + /* + // Setup the changeStream on the MongoDB main collection + obj.fileChangeStream = obj.file.watch(); + obj.fileChangeStream.on('change', function (next) { + // Process next document + console.log('change', next); + }); + */ + + // Setup MongoDB events collection and indexes + obj.eventsfile = db.collection('events'); // Collection containing all events + /* + obj.eventsfile.getIndexes(function (err, indexes) { + // Check if we need to reset indexes + var indexesByName = {}, indexCount = 0; + for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; } + if ((indexCount != 5) || (indexesByName['Username1'] == null) || (indexesByName['DomainNodeTime1'] == null) || (indexesByName['IdsAndTime1'] == null) || (indexesByName['ExpireTime1'] == null)) { + // Reset all indexes + console.log('Resetting events indexes...'); + obj.eventsfile.dropIndexes(function (err) { + obj.eventsfile.createIndex({ username: 1 }, { sparse: 1, name: 'Username1' }); + obj.eventsfile.createIndex({ domain: 1, nodeid: 1, time: -1 }, { sparse: 1, name: 'DomainNodeTime1' }); + obj.eventsfile.createIndex({ ids: 1, time: -1 }, { sparse: 1, name: 'IdsAndTime1' }); + obj.eventsfile.createIndex({ "time": 1 }, { expireAfterSeconds: expireEventsSeconds, name: 'ExpireTime1' }); + }); + } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expireEventsSeconds) { + // Reset the timeout index + console.log('Resetting events expire index...'); + obj.eventsfile.dropIndex("ExpireTime1", function (err) { + obj.eventsfile.createIndex({ "time": 1 }, { expireAfterSeconds: expireEventsSeconds, name: 'ExpireTime1' }); + }); + } + }); + */ + + // Setup MongoDB power events collection and indexes + obj.powerfile = db.collection('power'); // Collection containing all power events + /* + obj.powerfile.getIndexes(function (err, indexes) { + // Check if we need to reset indexes + var indexesByName = {}, indexCount = 0; + for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; } + if ((indexCount != 3) || (indexesByName['NodeIdAndTime1'] == null) || (indexesByName['ExpireTime1'] == null)) { + // Reset all indexes + console.log('Resetting power events indexes...'); + obj.powerfile.dropIndexes(function (err) { + // Create all indexes + obj.powerfile.createIndex({ nodeid: 1, time: 1 }, { sparse: 1, name: 'NodeIdAndTime1' }); + obj.powerfile.createIndex({ "time": 1 }, { expireAfterSeconds: expirePowerEventsSeconds, name: 'ExpireTime1' }); + }); + } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expirePowerEventsSeconds) { + // Reset the timeout index + console.log('Resetting power events expire index...'); + obj.powerfile.dropIndex("ExpireTime1", function (err) { + // Reset the expire power events index + obj.powerfile.createIndex({ "time": 1 }, { expireAfterSeconds: expirePowerEventsSeconds, name: 'ExpireTime1' }); + }); + } + }); + */ + + // Setup MongoDB smbios collection, no indexes needed + obj.smbiosfile = db.collection('smbios'); // Collection containing all smbios information + + // Setup MongoDB server stats collection + obj.serverstatsfile = db.collection('serverstats'); // Collection of server stats + /* + obj.serverstatsfile.getIndexes(function (err, indexes) { + // Check if we need to reset indexes + var indexesByName = {}, indexCount = 0; + for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; } + if ((indexCount != 3) || (indexesByName['ExpireTime1'] == null)) { + // Reset all indexes + console.log('Resetting server stats indexes...'); + obj.serverstatsfile.dropIndexes(function (err) { + // Create all indexes + obj.serverstatsfile.createIndex({ "time": 1 }, { expireAfterSeconds: expireServerStatsSeconds, name: 'ExpireTime1' }); + obj.serverstatsfile.createIndex({ "expire": 1 }, { expireAfterSeconds: 0, name: 'ExpireTime2' }); // Auto-expire events + }); + } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expireServerStatsSeconds) { + // Reset the timeout index + console.log('Resetting server stats expire index...'); + obj.serverstatsfile.dropIndex("ExpireTime1", function (err) { + // Reset the expire server stats index + obj.serverstatsfile.createIndex({ "time": 1 }, { expireAfterSeconds: expireServerStatsSeconds, name: 'ExpireTime1' }); + }); + } + }); + */ + + func(obj); // Completed setup of MongoDB + }); + } else if (obj.parent.args.mongodb) { + // Use MongoJS obj.databaseType = 2; Datastore = require('mongojs'); var db = Datastore(obj.parent.args.mongodb); @@ -141,6 +606,8 @@ module.exports.CreateDB = function (parent) { }); } }); + + func(obj); // Completed setup of MongoJS } else { // Use NeDB (The default) obj.databaseType = 1; @@ -199,252 +666,9 @@ module.exports.CreateDB = function (parent) { obj.serverstatsfile.persistence.setAutocompactionInterval(36000); obj.serverstatsfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: 60 * 60 * 24 * 30 }); // Limit the server stats log to 30 days (Seconds * Minutes * Hours * Days) obj.serverstatsfile.ensureIndex({ fieldName: 'expire', expireAfterSeconds: 0 }); // Auto-expire events + + func(obj); // Completed setup of NeDB } - obj.SetupDatabase = function (func) { - // Check if the database unique identifier is present - // This is used to check that in server peering mode, everyone is using the same database. - obj.Get('DatabaseIdentifier', function (err, docs) { - if ((docs.length == 1) && (docs[0].value != null)) { - obj.identifier = docs[0].value; - } else { - obj.identifier = Buffer.from(require('crypto').randomBytes(48), 'binary').toString('hex'); - obj.Set({ _id: 'DatabaseIdentifier', value: obj.identifier }); - } - }); - - // Load database schema version and check if we need to update - obj.Get('SchemaVersion', function (err, docs) { - var ver = 0; - if (docs && docs.length == 1) { ver = docs[0].value; } - if (ver == 1) { console.log('This is an unsupported beta 1 database, delete it to create a new one.'); process.exit(0); } - - // TODO: Any schema upgrades here... - obj.Set({ _id: 'SchemaVersion', value: 2 }); - - func(ver); - }); - }; - - obj.cleanup = function (func) { - // TODO: Remove all mesh links to invalid users - // TODO: Remove all meshes that dont have any links - - // Remove all events, power events and SMBIOS data from the main collection. They are all in seperate collections now. - obj.file.remove({ type: 'event' }, { multi: true }); - obj.file.remove({ type: 'power' }, { multi: true }); - obj.file.remove({ type: 'smbios' }, { multi: true }); - - // Remove all objects that have a "meshid" that no longer points to a valid mesh. - obj.GetAllType('mesh', function (err, docs) { - var meshlist = []; - if ((err == null) && (docs.length > 0)) { for (var i in docs) { meshlist.push(docs[i]._id); } } - obj.file.remove({ meshid: { $exists: true, $nin: meshlist } }, { multi: true }); - - // Fix all of the creating & login to ticks by seconds, not milliseconds. - obj.GetAllType('user', function (err, docs) { - if (err == null && docs.length > 0) { - for (var i in docs) { - var fixed = false; - - // Fix account creation - if (docs[i].creation) { - if (docs[i].creation > 1300000000000) { docs[i].creation = Math.floor(docs[i].creation / 1000); fixed = true; } - if ((docs[i].creation % 1) != 0) { docs[i].creation = Math.floor(docs[i].creation); fixed = true; } - } - - // Fix last account login - if (docs[i].login) { - if (docs[i].login > 1300000000000) { docs[i].login = Math.floor(docs[i].login / 1000); fixed = true; } - if ((docs[i].login % 1) != 0) { docs[i].login = Math.floor(docs[i].login); fixed = true; } - } - - // Fix last password change - if (docs[i].passchange) { - if (docs[i].passchange > 1300000000000) { docs[i].passchange = Math.floor(docs[i].passchange / 1000); fixed = true; } - if ((docs[i].passchange % 1) != 0) { docs[i].passchange = Math.floor(docs[i].passchange); fixed = true; } - } - - // Fix subscriptions - if (docs[i].subscriptions != null) { delete docs[i].subscriptions; fixed = true; } - - // Save the user if needed - if (fixed) { obj.Set(docs[i]); } - - // We are done - if (func) { func(); } - } - } - }); - }); - }; - - // Database actions on the main collection - obj.Set = function (data, func) { obj.file.update({ _id: data._id }, data, { upsert: true }, func); }; - obj.Get = function (id, func) - { - if (arguments.length > 2) - { - var parms = [func]; - for (var parmx = 2; parmx < arguments.length; ++parmx) { parms.push(arguments[parmx]); } - var func2 = function _func2(arg1, arg2) - { - var userCallback = _func2.userArgs.shift(); - _func2.userArgs.unshift(arg2); - _func2.userArgs.unshift(arg1); - userCallback.apply(obj, _func2.userArgs); - }; - func2.userArgs = parms; - obj.file.find({ _id: id }, func2); - } - else - { - obj.file.find({ _id: id }, func); - } - }; - obj.GetAll = function (func) { obj.file.find({}, func); }; - obj.GetAllTypeNoTypeField = function (type, domain, func) { obj.file.find({ type: type, domain: domain }, { type: 0 }, func); }; - obj.GetAllTypeNoTypeFieldMeshFiltered = function (meshes, domain, type, id, func) { var x = { type: type, domain: domain, meshid: { $in: meshes } }; if (id) { x._id = id; } obj.file.find(x, { type: 0 }, func); }; - obj.GetAllType = function (type, func) { obj.file.find({ type: type }, func); }; - obj.GetAllIdsOfType = function (ids, domain, type, func) { obj.file.find({ type: type, domain: domain, _id: { $in: ids } }, func); }; - obj.GetUserWithEmail = function (domain, email, func) { obj.file.find({ type: 'user', domain: domain, email: email }, { type: 0 }, func); }; - obj.GetUserWithVerifiedEmail = function (domain, email, func) { obj.file.find({ type: 'user', domain: domain, email: email, emailVerified: true }, { type: 0 }, func); }; - obj.Remove = function (id) { obj.file.remove({ _id: id }); }; - obj.RemoveAll = function (func) { obj.file.remove({}, { multi: true }, func); }; - obj.RemoveAllOfType = function (type, func) { obj.file.remove({ type: type }, { multi: true }, func); }; - obj.InsertMany = function (data, func) { obj.file.insert(data, func); }; - obj.RemoveMeshDocuments = function (id) { obj.file.remove({ meshid: id }, { multi: true }); obj.file.remove({ _id: 'nt' + id }); }; - obj.MakeSiteAdmin = function (username, domain) { obj.Get('user/' + domain + '/' + username, function (err, docs) { if (docs.length == 1) { docs[0].siteadmin = 0xFFFFFFFF; obj.Set(docs[0]); } }); }; - obj.DeleteDomain = function (domain, func) { obj.file.remove({ domain: domain }, { multi: true }, func); }; - obj.SetUser = function (user) { var u = Clone(user); if (u.subscriptions) { delete u.subscriptions; } obj.Set(u); }; - obj.dispose = function () { for (var x in obj) { if (obj[x].close) { obj[x].close(); } delete obj[x]; } }; - obj.getLocalAmtNodes = function (func) { obj.file.find({ type: 'node', host: { $exists: true, $ne: null }, intelamt: { $exists: true } }, func); }; - obj.getAmtUuidNode = function (meshid, uuid, func) { obj.file.find({ type: 'node', meshid: meshid, 'intelamt.uuid': uuid }, func); }; - obj.isMaxType = function (max, type, domainid, func) { if (max == null) { func(false); } else { obj.file.count({ type: type, domain: domainid }, function (err, count) { func((err != null) || (count > max)); }); } } - - // Database actions on the events collection - obj.GetAllEvents = function (func) { obj.eventsfile.find({}, func); }; - obj.StoreEvent = function (event) { obj.eventsfile.insert(event); }; - obj.GetEvents = function (ids, domain, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).exec(func); } else { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }, func); } }; - obj.GetEventsWithLimit = function (ids, domain, limit, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).exec(func); } else { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit, func); } }; - obj.GetUserEvents = function (ids, domain, username, func) { - if (obj.databaseType == 1) { - obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).exec(func); - } else { - obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }, func); - } - }; - obj.GetUserEventsWithLimit = function (ids, domain, username, limit, func) { - if (obj.databaseType == 1) { - obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).exec(func); - } else { - obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit, func); - } - }; - obj.GetNodeEventsWithLimit = function (nodeid, domain, limit, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, nodeid: nodeid }, { type: 0, etype: 0, _id: 0, domain: 0, ids: 0, node: 0, nodeid: 0 }).sort({ time: -1 }).limit(limit).exec(func); } else { obj.eventsfile.find({ domain: domain, nodeid: nodeid }, { type: 0, etype: 0, _id: 0, domain: 0, ids: 0, node: 0, nodeid: 0 }).sort({ time: -1 }).limit(limit, func); } }; - obj.RemoveAllEvents = function (domain) { obj.eventsfile.remove({ domain: domain }, { multi: true }); }; - obj.RemoveAllNodeEvents = function (domain, nodeid) { obj.eventsfile.remove({ domain: domain, nodeid: nodeid }, { multi: true }); }; - - // Database actions on the power collection - obj.getAllPower = function (func) { obj.powerfile.find({}, func); }; - obj.storePowerEvent = function (event, multiServer, func) { if (multiServer != null) { event.server = multiServer.serverid; } obj.powerfile.insert(event, func); }; - obj.getPowerTimeline = function (nodeid, func) { if (obj.databaseType == 1) { obj.powerfile.find({ nodeid: { $in: ['*', nodeid] } }, { _id: 0, nodeid: 0, s: 0 }).sort({ time: 1 }).exec(func); } else { obj.powerfile.find({ nodeid: { $in: ['*', nodeid] } }, { _id: 0, nodeid: 0, s: 0 }).sort({ time: 1 }, func); } }; - obj.removeAllPowerEvents = function () { obj.powerfile.remove({}, { multi: true }); }; - obj.removeAllPowerEventsForNode = function (nodeid) { obj.powerfile.remove({ nodeid: nodeid }, { multi: true }); }; - - // Database actions on the SMBIOS collection - obj.SetSMBIOS = function (smbios, func) { obj.smbiosfile.update({ _id: smbios._id }, smbios, { upsert: true }, func); }; - obj.RemoveSMBIOS = function (id) { obj.smbiosfile.remove({ _id: id }); }; - obj.GetSMBIOS = function (id, func) { obj.smbiosfile.find({ _id: id }, func); }; - - // Database actions on the Server Stats collection - obj.SetServerStats = function (data, func) { obj.serverstatsfile.insert(data, func); }; - obj.GetServerStats = function (hours, func) { var t = new Date(); t.setTime(t.getTime() - (60 * 60 * 1000 * hours)); obj.serverstatsfile.find({ time: { $gt: t } }, { _id: 0, cpu: 0 }, func); }; - - // Read a configuration file from the database - obj.getConfigFile = function (path, func) { obj.Get('cfile/' + path, func); } - - // Write a configuration file to the database - obj.setConfigFile = function (path, data, func) { obj.Set({ _id: 'cfile/' + path, type: 'cfile', data: data.toString('base64') }, func); } - - // List all configuration files - obj.listConfigFiles = function (func) { obj.file.find({ type: 'cfile' }).sort({ _id: 1 }).exec(func); } - - // Get all configuration files - obj.getAllConfigFiles = function (password, func) { - obj.file.find({ type: 'cfile' }, function (err, docs) { - if (err != null) { func(null); return; } - var r = null; - for (var i = 0; i < docs.length; i++) { - var name = docs[i]._id.split('/')[1]; - var data = obj.decryptData(password, docs[i].data); - if (data != null) { if (r == null) { r = {}; } r[name] = data; } - } - func(r); - }); - } - - // Get encryption key - obj.getEncryptDataKey = function (password) { - if (typeof password != 'string') return null; - return obj.parent.crypto.createHash('sha384').update(password).digest("raw").slice(0, 32); - } - - // Encrypt data - obj.encryptData = function (password, plaintext) { - var key = obj.getEncryptDataKey(password); - if (key == null) return null; - const iv = obj.parent.crypto.randomBytes(16); - const aes = obj.parent.crypto.createCipheriv('aes-256-cbc', key, iv); - var ciphertext = aes.update(plaintext); - ciphertext = Buffer.concat([iv, ciphertext, aes.final()]); - return ciphertext.toString('base64'); - } - - // Decrypt data - obj.decryptData = function (password, ciphertext) { - try { - var key = obj.getEncryptDataKey(password); - if (key == null) return null; - const ciphertextBytes = Buffer.from(ciphertext, 'base64'); - const iv = ciphertextBytes.slice(0, 16); - const data = ciphertextBytes.slice(16); - const aes = obj.parent.crypto.createDecipheriv('aes-256-cbc', key, iv); - var plaintextBytes = Buffer.from(aes.update(data)); - plaintextBytes = Buffer.concat([plaintextBytes, aes.final()]); - return plaintextBytes; - } catch (ex) { return null; } - } - - // Get the number of records in the database for various types, this is the slow NeDB way. - // WARNING: This is a terrible query for database performance. Only do this when needed. This query will look at almost every document in the database. - obj.getStats = function (func) { - if (obj.databaseType == 2) { - // MongoDB version - obj.file.aggregate([{ "$group": { _id: "$type", count: { $sum: 1 } } }], function (err, docs) { - var counters = {}, totalCount = 0; - for (var i in docs) { if (docs[i]._id != null) { counters[docs[i]._id] = docs[i].count; totalCount += docs[i].count; } } - func({ nodes: counters['node'], meshes: counters['mesh'], users: counters['user'], total: totalCount }); - }) - } else { - // NeDB version - obj.file.count({ type: 'node' }, function (err, nodeCount) { - obj.file.count({ type: 'mesh' }, function (err, meshCount) { - obj.file.count({ type: 'user' }, function (err, userCount) { - obj.file.count({}, function (err, totalCount) { - func({ nodes: nodeCount, meshes: meshCount, users: userCount, total: totalCount }); - }); - }); - }); - }); - } - } - - // This is used to rate limit a number of operation per day. Returns a startValue each new days, but you can substract it and save the value in the db. - obj.getValueOfTheDay = function (id, startValue, func) { obj.Get(id, function (err, docs) { var date = new Date(), t = date.toLocaleDateString(); if (docs.length == 1) { var r = docs[0]; if (r.day == t) { func({ _id: id, value: r.value, day: t }); return; } } func({ _id: id, value: startValue, day: t }); }); }; - obj.escapeBase64 = function escapeBase64(val) { return (val.replace(/\+/g, '@').replace(/\//g, '$')); } - - function Clone(v) { return JSON.parse(JSON.stringify(v)); } - return obj; }; \ No newline at end of file diff --git a/meshcentral.js b/meshcentral.js index 613a1498..685d66cb 100644 --- a/meshcentral.js +++ b/meshcentral.js @@ -238,268 +238,272 @@ function CreateMeshCentralServer(config, args) { if (typeof obj.args.swarmallowedip == 'string') { if (obj.args.swarmallowedip == '') { obj.args.swarmallowedip = null; } else { obj.args.swarmallowedip = obj.args.swarmallowedip.split(','); } } if (typeof obj.args.debug == 'number') obj.debugLevel = obj.args.debug; if (obj.args.debug == true) obj.debugLevel = 1; - obj.db = require('./db.js').CreateDB(obj); - obj.db.SetupDatabase(function (dbversion) { - // See if any database operations needs to be completed - if (obj.args.deletedomain) { obj.db.DeleteDomain(obj.args.deletedomain, function () { console.log('Deleted domain ' + obj.args.deletedomain + '.'); process.exit(); }); return; } - if (obj.args.deletedefaultdomain) { obj.db.DeleteDomain('', function () { console.log('Deleted default domain.'); process.exit(); }); return; } - if (obj.args.showall) { obj.db.GetAll(function (err, docs) { console.log(docs); process.exit(); }); return; } - if (obj.args.showusers) { obj.db.GetAllType('user', function (err, docs) { console.log(docs); process.exit(); }); return; } - if (obj.args.shownodes) { obj.db.GetAllType('node', function (err, docs) { console.log(docs); process.exit(); }); return; } - if (obj.args.showmeshes) { obj.db.GetAllType('mesh', function (err, docs) { console.log(docs); process.exit(); }); return; } - if (obj.args.showevents) { obj.db.GetAllEvents(function (err, docs) { console.log(docs); process.exit(); }); return; } - if (obj.args.showpower) { obj.db.getAllPower(function (err, docs) { console.log(docs); process.exit(); }); return; } - if (obj.args.clearpower) { obj.db.removeAllPowerEvents(function () { process.exit(); }); return; } - if (obj.args.showiplocations) { obj.db.GetAllType('iploc', function (err, docs) { console.log(docs); process.exit(); }); return; } - if (obj.args.logintoken) { obj.getLoginToken(obj.args.logintoken, function (r) { console.log(r); process.exit(); }); return; } - if (obj.args.logintokenkey) { obj.showLoginTokenKey(function (r) { console.log(r); process.exit(); }); return; } + require('./db.js').CreateDB(obj, + function (db) { + obj.db = db; + obj.db.SetupDatabase(function (dbversion) { + // See if any database operations needs to be completed + if (obj.args.deletedomain) { obj.db.DeleteDomain(obj.args.deletedomain, function () { console.log('Deleted domain ' + obj.args.deletedomain + '.'); process.exit(); }); return; } + if (obj.args.deletedefaultdomain) { obj.db.DeleteDomain('', function () { console.log('Deleted default domain.'); process.exit(); }); return; } + if (obj.args.showall) { obj.db.GetAll(function (err, docs) { console.log(docs); process.exit(); }); return; } + if (obj.args.showusers) { obj.db.GetAllType('user', function (err, docs) { console.log(docs); process.exit(); }); return; } + if (obj.args.shownodes) { obj.db.GetAllType('node', function (err, docs) { console.log(docs); process.exit(); }); return; } + if (obj.args.showmeshes) { obj.db.GetAllType('mesh', function (err, docs) { console.log(docs); process.exit(); }); return; } + if (obj.args.showevents) { obj.db.GetAllEvents(function (err, docs) { console.log(docs); process.exit(); }); return; } + if (obj.args.showpower) { obj.db.getAllPower(function (err, docs) { console.log(docs); process.exit(); }); return; } + if (obj.args.clearpower) { obj.db.removeAllPowerEvents(function () { process.exit(); }); return; } + if (obj.args.showiplocations) { obj.db.GetAllType('iploc', function (err, docs) { console.log(docs); process.exit(); }); return; } + if (obj.args.logintoken) { obj.getLoginToken(obj.args.logintoken, function (r) { console.log(r); process.exit(); }); return; } + if (obj.args.logintokenkey) { obj.showLoginTokenKey(function (r) { console.log(r); process.exit(); }); return; } - // Show a list of all configuration files in the database - if (obj.args.dblistconfigfiles) { - obj.db.GetAllType('cfile', function (err, docs) { if (err == null) { if (docs.length == 0) { console.log('No files found.'); } else { for (var i in docs) { console.log(docs[i]._id.split('/')[1] + ', ' + Buffer.from(docs[i].data, 'base64').length + ' bytes.'); } } } else { console.log('Unable to read from database.'); } process.exit(); }); return; - } + // Show a list of all configuration files in the database + if (obj.args.dblistconfigfiles) { + obj.db.GetAllType('cfile', function (err, docs) { if (err == null) { if (docs.length == 0) { console.log('No files found.'); } else { for (var i in docs) { console.log(docs[i]._id.split('/')[1] + ', ' + Buffer.from(docs[i].data, 'base64').length + ' bytes.'); } } } else { console.log('Unable to read from database.'); } process.exit(); }); return; + } - // Display the content of a configuration file in the database - if (obj.args.dbshowconfigfile) { - if (typeof obj.args.configkey != 'string') { console.log('Error, --configkey is required.'); process.exit(); return; } - obj.db.getConfigFile(obj.args.dbshowconfigfile, function (err, docs) { - if (err == null) { - if (docs.length == 0) { console.log('File not found.'); } else { - var data = obj.db.decryptData(obj.args.configkey, docs[0].data); - if (data == null) { console.log('Invalid config key.'); } else { console.log(data); } - } - } else { console.log('Unable to read from database.'); } - process.exit(); - }); return; - } - - // Delete all configuration files from database - if (obj.args.dbdeleteconfigfiles) { - console.log('Deleting all configuration files from the database...'); obj.db.RemoveAllOfType('cfile', function () { console.log('Done.'); process.exit(); }); - } - - // Push all relevent files from meshcentral-data into the database - if (obj.args.dbpushconfigfiles) { - if (typeof obj.args.configkey != 'string') { console.log('Error, --configkey is required.'); process.exit(); return; } - if ((obj.args.dbpushconfigfiles !== true) && (typeof obj.args.dbpushconfigfiles != 'string')) { - console.log('Usage: --dbpulldatafiles (path) This will import files from folder into the database'); - console.log(' --dbpulldatafiles This will import files from meshcentral-data into the db.'); - process.exit(); - } else { - if ((obj.args.dbpushconfigfiles == '*') || (obj.args.dbpushconfigfiles === true)) { obj.args.dbpushconfigfiles = obj.datapath; } - obj.fs.readdir(obj.args.dbpushconfigfiles, function (err, files) { - if (err != null) { console.log('ERROR: Unable to read from folder ' + obj.args.dbpushconfigfiles); process.exit(); return; } - var configFound = false; - for (var i in files) { if (files[i] == 'config.json') { configFound = true; } } - if (configFound == false) { console.log('ERROR: No config.json in folder ' + obj.args.dbpushconfigfiles); process.exit(); return; } - obj.db.RemoveAllOfType('cfile', function () { - obj.fs.readdir(obj.args.dbpushconfigfiles, function (err, files) { - var lockCount = 1 - for (var i in files) { - const file = files[i]; - if ((file == 'config.json') || file.endsWith('.key') || file.endsWith('.crt') || (file == 'terms.txt') || file.endsWith('.jpg') || file.endsWith('.png')) { - const path = obj.path.join(obj.args.dbpushconfigfiles, files[i]), binary = Buffer.from(obj.fs.readFileSync(path, { encoding: 'binary' }), 'binary'); - console.log('Pushing ' + file + ', ' + binary.length + ' bytes.'); - lockCount++; - obj.db.setConfigFile(file, obj.db.encryptData(obj.args.configkey, binary), function () { if ((--lockCount) == 0) { console.log('Done.'); process.exit(); } }); - } + // Display the content of a configuration file in the database + if (obj.args.dbshowconfigfile) { + if (typeof obj.args.configkey != 'string') { console.log('Error, --configkey is required.'); process.exit(); return; } + obj.db.getConfigFile(obj.args.dbshowconfigfile, function (err, docs) { + if (err == null) { + if (docs.length == 0) { console.log('File not found.'); } else { + var data = obj.db.decryptData(obj.args.configkey, docs[0].data); + if (data == null) { console.log('Invalid config key.'); } else { console.log(data); } } - if (--lockCount == 0) { process.exit(); } - }); - }); - }); - } - return; - } + } else { console.log('Unable to read from database.'); } + process.exit(); + }); return; + } - // Pull all database files into meshcentral-data - if (obj.args.dbpullconfigfiles) { - if (typeof obj.args.configkey != 'string') { console.log('Error, --configkey is required.'); process.exit(); return; } - if (typeof obj.args.dbpullconfigfiles != 'string') { - console.log('Usage: --dbpulldatafiles (path)'); - process.exit(); - } else { - obj.db.GetAllType('cfile', function (err, docs) { - if (err == null) { - if (docs.length == 0) { - console.log('File not found.'); - } else { - for (var i in docs) { - const file = docs[i]._id.split('/')[1], binary = obj.db.decryptData(obj.args.configkey, docs[i].data); - if (binary == null) { - console.log('Invalid config key.'); - } else { - var fullFileName = obj.path.join(obj.args.dbpullconfigfiles, file); - try { obj.fs.writeFileSync(fullFileName, binary); } catch (ex) { console.log('Unable to write to ' + fullFileName); process.exit(); return; } - console.log('Pulling ' + file + ', ' + binary.length + ' bytes.'); - } - } - } + // Delete all configuration files from database + if (obj.args.dbdeleteconfigfiles) { + console.log('Deleting all configuration files from the database...'); obj.db.RemoveAllOfType('cfile', function () { console.log('Done.'); process.exit(); }); + } + + // Push all relevent files from meshcentral-data into the database + if (obj.args.dbpushconfigfiles) { + if (typeof obj.args.configkey != 'string') { console.log('Error, --configkey is required.'); process.exit(); return; } + if ((obj.args.dbpushconfigfiles !== true) && (typeof obj.args.dbpushconfigfiles != 'string')) { + console.log('Usage: --dbpulldatafiles (path) This will import files from folder into the database'); + console.log(' --dbpulldatafiles This will import files from meshcentral-data into the db.'); + process.exit(); } else { - console.log('Unable to read from database.'); - } - process.exit(); - }); - } - return; - } - - if (obj.args.dbexport) { - // Export the entire database to a JSON file - if (obj.args.dbexport == true) { obj.args.dbexport = obj.getConfigFilePath('meshcentral.db.json'); } - obj.db.GetAll(function (err, docs) { - obj.fs.writeFileSync(obj.args.dbexport, JSON.stringify(docs)); - console.log('Exported ' + docs.length + ' objects(s) to ' + obj.args.dbexport + '.'); process.exit(); - }); - return; - } - if (obj.args.dbexportmin) { - // Export a minimal database to a JSON file. Export only users, meshes and nodes. - // This is a useful command to look at the database. - if (obj.args.dbexportmin == true) { obj.args.dbexportmin = obj.getConfigFilePath('meshcentral.db.json'); } - obj.db.GetAllType({ $in: ['user', 'node', 'mesh'] }, function (err, docs) { - obj.fs.writeFileSync(obj.args.dbexportmin, JSON.stringify(docs)); - console.log('Exported ' + docs.length + ' objects(s) to ' + obj.args.dbexportmin + '.'); process.exit(); - }); - return; - } - if (obj.args.dbimport) { - // Import the entire database from a JSON file - if (obj.args.dbimport == true) { obj.args.dbimport = obj.getConfigFilePath('meshcentral.db.json'); } - var json = null, json2 = "", badCharCount = 0; - try { json = obj.fs.readFileSync(obj.args.dbimport, { encoding: 'utf8' }); } catch (e) { console.log('Invalid JSON file: ' + obj.args.dbimport + ': ' + e); process.exit(); } - for (i = 0; i < json.length; i++) { if (json.charCodeAt(i) >= 32) { json2 += json[i]; } else { var tt = json.charCodeAt(i); if (tt != 10 && tt != 13) { badCharCount++; } } } // Remove all bad chars - if (badCharCount > 0) { console.log(badCharCount + ' invalid character(s) where removed.'); } - try { json = JSON.parse(json2); } catch (e) { console.log('Invalid JSON format: ' + obj.args.dbimport + ': ' + e); process.exit(); } - if ((json == null) || (typeof json.length != 'number') || (json.length < 1)) { console.log('Invalid JSON format: ' + obj.args.dbimport + '.'); } - for (i in json) { if ((json[i].type == "mesh") && (json[i].links != null)) { for (var j in json[i].links) { var esc = obj.common.escapeFieldName(j); if (esc !== j) { json[i].links[esc] = json[i].links[j]; delete json[i].links[j]; } } } } // Escape MongoDB invalid field chars - //for (i in json) { if ((json[i].type == "node") && (json[i].host != null)) { json[i].rname = json[i].host; delete json[i].host; } } // DEBUG: Change host to rname - setTimeout(function () { // If the Mongo database is being created for the first time, there is a race condition here. This will get around it. - obj.db.RemoveAll(function () { - obj.db.InsertMany(json, function (err) { - if (err != null) { console.log(err); } else { console.log('Imported ' + json.length + ' objects(s) from ' + obj.args.dbimport + '.'); } process.exit(); - }); - }); - }, 100); - return; - } - /* - if (obj.args.dbimport) { - // Import the entire database from a very large JSON file - obj.db.RemoveAll(function () { - if (obj.args.dbimport == true) { obj.args.dbimport = obj.getConfigFilePath('meshcentral.db.json'); } - var json = null, json2 = "", badCharCount = 0; - const StreamArray = require('stream-json/streamers/StreamArray'); - const jsonStream = StreamArray.withParser(); - jsonStream.on('data', function (data) { obj.db.Set(data.value); }); - jsonStream.on('end', () => { console.log('Done.'); process.exit(); }); - obj.fs.createReadStream(obj.args.dbimport).pipe(jsonStream.input); - }); - return; - } - */ - if (obj.args.dbmerge) { - // Import the entire database from a JSON file - if (obj.args.dbmerge == true) { obj.args.dbmerge = obj.getConfigFilePath('meshcentral.db.json'); } - var json = null, json2 = "", badCharCount = 0; - try { json = obj.fs.readFileSync(obj.args.dbmerge, { encoding: 'utf8' }); } catch (e) { console.log('Invalid JSON file: ' + obj.args.dbmerge + ': ' + e); process.exit(); } - for (i = 0; i < json.length; i++) { if (json.charCodeAt(i) >= 32) { json2 += json[i]; } else { var tt = json.charCodeAt(i); if (tt != 10 && tt != 13) { badCharCount++; } } } // Remove all bad chars - if (badCharCount > 0) { console.log(badCharCount + ' invalid character(s) where removed.'); } - try { json = JSON.parse(json2); } catch (e) { console.log('Invalid JSON format: ' + obj.args.dbmerge + ': ' + e); process.exit(); } - if ((json == null) || (typeof json.length != 'number') || (json.length < 1)) { console.log('Invalid JSON format: ' + obj.args.dbimport + '.'); } - - // Get all users from current database - obj.db.GetAllType('user', function (err, docs) { - var users = {}, usersCount = 0; - for (var i in docs) { users[docs[i]._id] = docs[i]; usersCount++; } - - // Fetch all meshes from the database - obj.db.GetAllType('mesh', function (err, docs) { - obj.common.unEscapeAllLinksFieldName(docs); - var meshes = {}, meshesCount = 0; - for (var i in docs) { meshes[docs[i]._id] = docs[i]; meshesCount++; } - console.log('Loaded ' + usersCount + ' users and ' + meshesCount + ' meshes.'); - // Look at each object in the import file - var objectToAdd = []; - for (var i in json) { - var newobj = json[i]; - if (newobj.type == 'user') { - // Check if the user already exists - var existingUser = users[newobj._id]; - if (existingUser) { - // Merge the links - if (typeof newobj.links == 'object') { - for (var j in newobj.links) { - if ((existingUser.links == null) || (existingUser.links[j] == null)) { - if (existingUser.links == null) { existingUser.links = {}; } - existingUser.links[j] = newobj.links[j]; + if ((obj.args.dbpushconfigfiles == '*') || (obj.args.dbpushconfigfiles === true)) { obj.args.dbpushconfigfiles = obj.datapath; } + obj.fs.readdir(obj.args.dbpushconfigfiles, function (err, files) { + if (err != null) { console.log('ERROR: Unable to read from folder ' + obj.args.dbpushconfigfiles); process.exit(); return; } + var configFound = false; + for (var i in files) { if (files[i] == 'config.json') { configFound = true; } } + if (configFound == false) { console.log('ERROR: No config.json in folder ' + obj.args.dbpushconfigfiles); process.exit(); return; } + obj.db.RemoveAllOfType('cfile', function () { + obj.fs.readdir(obj.args.dbpushconfigfiles, function (err, files) { + var lockCount = 1 + for (var i in files) { + const file = files[i]; + if ((file == 'config.json') || file.endsWith('.key') || file.endsWith('.crt') || (file == 'terms.txt') || file.endsWith('.jpg') || file.endsWith('.png')) { + const path = obj.path.join(obj.args.dbpushconfigfiles, files[i]), binary = Buffer.from(obj.fs.readFileSync(path, { encoding: 'binary' }), 'binary'); + console.log('Pushing ' + file + ', ' + binary.length + ' bytes.'); + lockCount++; + obj.db.setConfigFile(file, obj.db.encryptData(obj.args.configkey, binary), function () { if ((--lockCount) == 0) { console.log('Done.'); process.exit(); } }); } } - } - if (existingUser.name == 'admin') { existingUser.links = {}; } - objectToAdd.push(existingUser); // Add this user - } else { - objectToAdd.push(newobj); // Add this user - } - } else if (newobj.type == 'mesh') { - // Add this object after escaping - objectToAdd.push(obj.common.escapeLinksFieldName(newobj)); - } // Don't add nodes. + if (--lockCount == 0) { process.exit(); } + }); + }); + }); } - console.log('Importing ' + objectToAdd.length + ' object(s)...'); - var pendingCalls = 1; - for (var i in objectToAdd) { - pendingCalls++; - obj.db.Set(objectToAdd[i], function (err) { if (err != null) { console.log(err); } else { if (--pendingCalls == 0) { process.exit(); } } }); - } - if (--pendingCalls == 0) { process.exit(); } - }); - }); - return; - } - - // Load configuration for database if needed - if (obj.args.loadconfigfromdb) { - var key = null; - if (typeof obj.args.configkey == 'string') { key = obj.args.configkey; } - else if (typeof obj.args.loadconfigfromdb == 'string') { key = obj.args.loadconfigfromdb; } - if (key == null) { console.log('Error, --configkey is required.'); process.exit(); return; } - obj.db.getAllConfigFiles(key, function (configFiles) { - if (configFiles == null) { console.log('Error, no configuration files found or invalid configkey.'); process.exit(); return; } - if (!configFiles['config.json']) { console.log('Error, could not file config.json from database.'); process.exit(); return; } - obj.configurationFiles = configFiles; - - // Parse the new configuration file - var config2 = null; - try { config2 = JSON.parse(configFiles['config.json']); } catch (ex) { console.log('Error, unable to parse config.json from database.'); process.exit(); return; } - - // Set the command line arguments to the config file if they are not present - if (!config2.settings) { config2.settings = {}; } - for (i in args) { config2.settings[i] = args[i]; } - - // Lower case all keys in the config file - try { - require('./common.js').objKeysToLower(config2, ["ldapoptions"]); - } catch (ex) { - console.log('CRITICAL ERROR: Unable to access the file \"./common.js\".\r\nCheck folder & file permissions.'); - process.exit(); return; } - // Grad some of the values from the original config.json file if present. - config2['mongodb'] = config['mongodb']; - config2['mongodbcol'] = config['mongodbcol']; - config2['dbencryptkey'] = config['dbencryptkey']; + // Pull all database files into meshcentral-data + if (obj.args.dbpullconfigfiles) { + if (typeof obj.args.configkey != 'string') { console.log('Error, --configkey is required.'); process.exit(); return; } + if (typeof obj.args.dbpullconfigfiles != 'string') { + console.log('Usage: --dbpulldatafiles (path)'); + process.exit(); + } else { + obj.db.GetAllType('cfile', function (err, docs) { + if (err == null) { + if (docs.length == 0) { + console.log('File not found.'); + } else { + for (var i in docs) { + const file = docs[i]._id.split('/')[1], binary = obj.db.decryptData(obj.args.configkey, docs[i].data); + if (binary == null) { + console.log('Invalid config key.'); + } else { + var fullFileName = obj.path.join(obj.args.dbpullconfigfiles, file); + try { obj.fs.writeFileSync(fullFileName, binary); } catch (ex) { console.log('Unable to write to ' + fullFileName); process.exit(); return; } + console.log('Pulling ' + file + ', ' + binary.length + ' bytes.'); + } + } + } + } else { + console.log('Unable to read from database.'); + } + process.exit(); + }); + } + return; + } - // We got a new config.json from the database, let's use it. - config = obj.config = config2; - obj.StartEx1b(); + if (obj.args.dbexport) { + // Export the entire database to a JSON file + if (obj.args.dbexport == true) { obj.args.dbexport = obj.getConfigFilePath('meshcentral.db.json'); } + obj.db.GetAll(function (err, docs) { + obj.fs.writeFileSync(obj.args.dbexport, JSON.stringify(docs)); + console.log('Exported ' + docs.length + ' objects(s) to ' + obj.args.dbexport + '.'); process.exit(); + }); + return; + } + if (obj.args.dbexportmin) { + // Export a minimal database to a JSON file. Export only users, meshes and nodes. + // This is a useful command to look at the database. + if (obj.args.dbexportmin == true) { obj.args.dbexportmin = obj.getConfigFilePath('meshcentral.db.json'); } + obj.db.GetAllType({ $in: ['user', 'node', 'mesh'] }, function (err, docs) { + obj.fs.writeFileSync(obj.args.dbexportmin, JSON.stringify(docs)); + console.log('Exported ' + docs.length + ' objects(s) to ' + obj.args.dbexportmin + '.'); process.exit(); + }); + return; + } + if (obj.args.dbimport) { + // Import the entire database from a JSON file + if (obj.args.dbimport == true) { obj.args.dbimport = obj.getConfigFilePath('meshcentral.db.json'); } + var json = null, json2 = "", badCharCount = 0; + try { json = obj.fs.readFileSync(obj.args.dbimport, { encoding: 'utf8' }); } catch (e) { console.log('Invalid JSON file: ' + obj.args.dbimport + ': ' + e); process.exit(); } + for (i = 0; i < json.length; i++) { if (json.charCodeAt(i) >= 32) { json2 += json[i]; } else { var tt = json.charCodeAt(i); if (tt != 10 && tt != 13) { badCharCount++; } } } // Remove all bad chars + if (badCharCount > 0) { console.log(badCharCount + ' invalid character(s) where removed.'); } + try { json = JSON.parse(json2); } catch (e) { console.log('Invalid JSON format: ' + obj.args.dbimport + ': ' + e); process.exit(); } + if ((json == null) || (typeof json.length != 'number') || (json.length < 1)) { console.log('Invalid JSON format: ' + obj.args.dbimport + '.'); } + for (i in json) { if ((json[i].type == "mesh") && (json[i].links != null)) { for (var j in json[i].links) { var esc = obj.common.escapeFieldName(j); if (esc !== j) { json[i].links[esc] = json[i].links[j]; delete json[i].links[j]; } } } } // Escape MongoDB invalid field chars + //for (i in json) { if ((json[i].type == "node") && (json[i].host != null)) { json[i].rname = json[i].host; delete json[i].host; } } // DEBUG: Change host to rname + setTimeout(function () { // If the Mongo database is being created for the first time, there is a race condition here. This will get around it. + obj.db.RemoveAll(function () { + obj.db.InsertMany(json, function (err) { + if (err != null) { console.log(err); } else { console.log('Imported ' + json.length + ' objects(s) from ' + obj.args.dbimport + '.'); } process.exit(); + }); + }); + }, 100); + return; + } + /* + if (obj.args.dbimport) { + // Import the entire database from a very large JSON file + obj.db.RemoveAll(function () { + if (obj.args.dbimport == true) { obj.args.dbimport = obj.getConfigFilePath('meshcentral.db.json'); } + var json = null, json2 = "", badCharCount = 0; + const StreamArray = require('stream-json/streamers/StreamArray'); + const jsonStream = StreamArray.withParser(); + jsonStream.on('data', function (data) { obj.db.Set(data.value); }); + jsonStream.on('end', () => { console.log('Done.'); process.exit(); }); + obj.fs.createReadStream(obj.args.dbimport).pipe(jsonStream.input); + }); + return; + } + */ + if (obj.args.dbmerge) { + // Import the entire database from a JSON file + if (obj.args.dbmerge == true) { obj.args.dbmerge = obj.getConfigFilePath('meshcentral.db.json'); } + var json = null, json2 = "", badCharCount = 0; + try { json = obj.fs.readFileSync(obj.args.dbmerge, { encoding: 'utf8' }); } catch (e) { console.log('Invalid JSON file: ' + obj.args.dbmerge + ': ' + e); process.exit(); } + for (i = 0; i < json.length; i++) { if (json.charCodeAt(i) >= 32) { json2 += json[i]; } else { var tt = json.charCodeAt(i); if (tt != 10 && tt != 13) { badCharCount++; } } } // Remove all bad chars + if (badCharCount > 0) { console.log(badCharCount + ' invalid character(s) where removed.'); } + try { json = JSON.parse(json2); } catch (e) { console.log('Invalid JSON format: ' + obj.args.dbmerge + ': ' + e); process.exit(); } + if ((json == null) || (typeof json.length != 'number') || (json.length < 1)) { console.log('Invalid JSON format: ' + obj.args.dbimport + '.'); } + + // Get all users from current database + obj.db.GetAllType('user', function (err, docs) { + var users = {}, usersCount = 0; + for (var i in docs) { users[docs[i]._id] = docs[i]; usersCount++; } + + // Fetch all meshes from the database + obj.db.GetAllType('mesh', function (err, docs) { + obj.common.unEscapeAllLinksFieldName(docs); + var meshes = {}, meshesCount = 0; + for (var i in docs) { meshes[docs[i]._id] = docs[i]; meshesCount++; } + console.log('Loaded ' + usersCount + ' users and ' + meshesCount + ' meshes.'); + // Look at each object in the import file + var objectToAdd = []; + for (var i in json) { + var newobj = json[i]; + if (newobj.type == 'user') { + // Check if the user already exists + var existingUser = users[newobj._id]; + if (existingUser) { + // Merge the links + if (typeof newobj.links == 'object') { + for (var j in newobj.links) { + if ((existingUser.links == null) || (existingUser.links[j] == null)) { + if (existingUser.links == null) { existingUser.links = {}; } + existingUser.links[j] = newobj.links[j]; + } + } + } + if (existingUser.name == 'admin') { existingUser.links = {}; } + objectToAdd.push(existingUser); // Add this user + } else { + objectToAdd.push(newobj); // Add this user + } + } else if (newobj.type == 'mesh') { + // Add this object after escaping + objectToAdd.push(obj.common.escapeLinksFieldName(newobj)); + } // Don't add nodes. + } + console.log('Importing ' + objectToAdd.length + ' object(s)...'); + var pendingCalls = 1; + for (var i in objectToAdd) { + pendingCalls++; + obj.db.Set(objectToAdd[i], function (err) { if (err != null) { console.log(err); } else { if (--pendingCalls == 0) { process.exit(); } } }); + } + if (--pendingCalls == 0) { process.exit(); } + }); + }); + return; + } + + // Load configuration for database if needed + if (obj.args.loadconfigfromdb) { + var key = null; + if (typeof obj.args.configkey == 'string') { key = obj.args.configkey; } + else if (typeof obj.args.loadconfigfromdb == 'string') { key = obj.args.loadconfigfromdb; } + if (key == null) { console.log('Error, --configkey is required.'); process.exit(); return; } + obj.db.getAllConfigFiles(key, function (configFiles) { + if (configFiles == null) { console.log('Error, no configuration files found or invalid configkey.'); process.exit(); return; } + if (!configFiles['config.json']) { console.log('Error, could not file config.json from database.'); process.exit(); return; } + obj.configurationFiles = configFiles; + + // Parse the new configuration file + var config2 = null; + try { config2 = JSON.parse(configFiles['config.json']); } catch (ex) { console.log('Error, unable to parse config.json from database.'); process.exit(); return; } + + // Set the command line arguments to the config file if they are not present + if (!config2.settings) { config2.settings = {}; } + for (i in args) { config2.settings[i] = args[i]; } + + // Lower case all keys in the config file + try { + require('./common.js').objKeysToLower(config2, ["ldapoptions"]); + } catch (ex) { + console.log('CRITICAL ERROR: Unable to access the file \"./common.js\".\r\nCheck folder & file permissions.'); + process.exit(); + return; + } + + // Grad some of the values from the original config.json file if present. + config2['mongodb'] = config['mongodb']; + config2['mongodbcol'] = config['mongodbcol']; + config2['dbencryptkey'] = config['dbencryptkey']; + + // We got a new config.json from the database, let's use it. + config = obj.config = config2; + obj.StartEx1b(); + }); + } else { + config = obj.config = getConfig(true); + obj.StartEx1b(); + } }); - } else { - config = obj.config = getConfig(true); - obj.StartEx1b(); } - }); + ); }; // Time to start the serverf or real. @@ -1711,7 +1715,8 @@ function mainStart(args) { if (require('os').platform() == 'win32') { modules.push('node-windows'); if (sspi == true) { modules.push('node-sspi'); } } // Add Windows modules if (ldap == true) { modules.push('ldapauth-fork'); } if (config.letsencrypt != null) { modules.push('greenlock'); modules.push('le-store-certbot'); modules.push('le-challenge-fs'); modules.push('le-acme-core'); } // Add Greenlock Modules - if (config.settings.mongodb != null) { modules.push('mongojs'); } // Add MongoDB + if (config.settings.mongodb != null) { modules.push('mongojs'); } // Add MongoJS + else if (config.settings.mongo != null) { modules.push('mongodb'); } // Add MongoDB if (config.smtp != null) { modules.push('nodemailer'); } // Add SMTP support // Get the current node version