From a14522548f456a044122105bea94762c5fbcbdb5 Mon Sep 17 00:00:00 2001 From: Ylian Saint-Hilaire Date: Mon, 18 Feb 2019 14:32:55 -0800 Subject: [PATCH] Database performance + mesh undelete. --- db.js | 151 ++++++++++++++++++++++++++++------- meshagent.js | 27 ++++++- meshcentral.js | 54 +++++-------- meshuser.js | 49 +++++++++--- package.json | 2 +- views/default-min.handlebars | 2 +- views/default.handlebars | 3 +- webserver.js | 40 +++++----- 8 files changed, 228 insertions(+), 100 deletions(-) diff --git a/db.js b/db.js index 2ec75a2d..0682cf95 100644 --- a/db.js +++ b/db.js @@ -28,6 +28,8 @@ module.exports.CreateDB = function (parent) { var obj = {}; var Datastore = null; + var expireEventsSeconds = (60 * 60 * 24 * 20); // By default, expire events after 20 days. (Seconds * Minutes * Hours * Days) + var expirePowerEventsSeconds = (60 * 60 * 24 * 10); // By default, expire power events after 10 days. (Seconds * Minutes * Hours * Days) obj.path = require('path'); obj.parent = parent; obj.identifier = null; @@ -40,14 +42,69 @@ module.exports.CreateDB = function (parent) { var db = Datastore(obj.parent.args.mongodb); var dbcollection = 'meshcentral'; if (obj.parent.args.mongodbcol) { dbcollection = obj.parent.args.mongodbcol; } - obj.file = db.collection(dbcollection); - // Setup MongoDB indexes - obj.file.createIndex({ type: 1, domain: 1, meshid: 1 }, { sparse: 1 }); // Speeds up GetAllTypeNoTypeField() and GetAllTypeNoTypeFieldMeshFiltered() - obj.file.createIndex({ email: 1 }, { sparse: 1 }); // Speeds up GetUserWithEmail() and GetUserWithVerifiedEmail() - obj.file.createIndex({ ids: 1, time: -1 }, { sparse: 1 }); // Speeds up GetEvents() and GetEventsWithLimit() - obj.file.createIndex({ type: 1, nodeid: 1, time: 1 }, { sparse: 1 }); // Speeds up getPowerTimeline() - obj.file.createIndex({ mesh: 1 }, { sparse: 1 }); // Speeds up RemoveMesh() + // Setup MongoDB main collection and indexes + obj.file = db.collection(dbcollection); + obj.file.getIndexes(function (err, indexes) { + // Check if we need to reset indexes + var indexesByName = {}, indexCount = 0; + for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; } + if ((indexCount != 4) || (indexesByName['TypeDomainMesh1'] == null) || (indexesByName['Email1'] == null) || (indexesByName['Mesh1'] == null)) { + console.log('Resetting main indexes...'); + obj.file.dropIndexes(function (err) { + obj.file.createIndex({ type: 1, domain: 1, meshid: 1 }, { sparse: 1, name: 'TypeDomainMesh1' }); // Speeds up GetAllTypeNoTypeField() and GetAllTypeNoTypeFieldMeshFiltered() + obj.file.createIndex({ email: 1 }, { sparse: 1, name: 'Email1' }); // Speeds up GetUserWithEmail() and GetUserWithVerifiedEmail() + obj.file.createIndex({ meshid: 1 }, { sparse: 1, name: 'Mesh1' }); // Speeds up RemoveMesh() + }); + } + }); + + // Setup MongoDB events collection and indexes + obj.eventsfile = db.collection(dbcollection + '-events'); // Collection containing all events + obj.eventsfile.getIndexes(function (err, indexes) { + // Check if we need to reset indexes + var indexesByName = {}, indexCount = 0; + for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; } + if ((indexCount != 3) || (indexesByName['IdsAndTime1'] == null) || (indexesByName['ExpireTime1'] == null)) { + // Reset all indexes + console.log('Resetting events indexes...'); + obj.eventsfile.dropIndexes(function (err) { + obj.eventsfile.createIndex({ ids: 1, time: -1 }, { sparse: 1, name: 'IdsAndTime1' }); + obj.eventsfile.createIndex({ "time": 1 }, { expireAfterSeconds: expireEventsSeconds, name: 'ExpireTime1' }); + }); + } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expireEventsSeconds) { + // Reset the timeout index + console.log('Resetting events expire index...'); + obj.eventsfile.dropIndex("ExpireTime1", function (err) { + obj.eventsfile.createIndex({ "time": 1 }, { expireAfterSeconds: expireEventsSeconds, name: 'ExpireTime1' }); + }); + } + }); + + // Setup MongoDB power events collection and indexes + obj.powerfile = db.collection(dbcollection + '-power'); // Collection containing all power events + obj.powerfile.getIndexes(function (err, indexes) { + // Check if we need to reset indexes + var indexesByName = {}, indexCount = 0; + for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; } + if ((indexCount != 3) || (indexesByName['NodeIdAndTime1'] == null) || (indexesByName['ExpireTime1'] == null)) { + // Reset all indexes + console.log('Resetting power events indexes...'); + obj.powerfile.dropIndexes(function (err) { + // Create all indexes + obj.powerfile.createIndex({ nodeid: 1, time: 1 }, { sparse: 1, name: 'NodeIdAndTime1' }); + obj.powerfile.createIndex({ "time": 1 }, { expireAfterSeconds: expirePowerEventsSeconds, name: 'ExpireTime1' }); + }); + } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expirePowerEventsSeconds) { + // Reset the timeout index + console.log('Resetting power events expire index...'); + obj.powerfile.dropIndex("ExpireTime1", function (err) { + // Reset the expire power events index + obj.powerfile.createIndex({ "time": 1 }, { expireAfterSeconds: expirePowerEventsSeconds, name: 'ExpireTime1' }); + }); + } + }); + } else { // Use NeDB (The default) obj.databaseType = 1; @@ -76,16 +133,27 @@ module.exports.CreateDB = function (parent) { } } - // Start NeDB + // Start NeDB main collection and setup indexes obj.file = new Datastore(datastoreOptions); - obj.file.persistence.setAutocompactionInterval(3600); - - // Setup NeDB indexes + obj.file.persistence.setAutocompactionInterval(36000); obj.file.ensureIndex({ fieldName: 'type' }); obj.file.ensureIndex({ fieldName: 'domain' }); - obj.file.ensureIndex({ fieldName: 'meshid' }); - obj.file.ensureIndex({ fieldName: 'node' }); - obj.file.ensureIndex({ fieldName: 'email' }); + obj.file.ensureIndex({ fieldName: 'meshid', sparse: true }); + obj.file.ensureIndex({ fieldName: 'nodeid', sparse: true }); + obj.file.ensureIndex({ fieldName: 'email', sparse: true }); + + // Setup the events collection and setup indexes + obj.eventsfile = new Datastore({ filename: obj.parent.getConfigFilePath('meshcentral-events.db'), autoload: true }); + obj.eventsfile.persistence.setAutocompactionInterval(36000); + obj.eventsfile.ensureIndex({ fieldName: 'ids' }); // TODO: Not sure if this is a good index, this is a array field. + obj.eventsfile.ensureIndex({ fieldName: 'nodeid', sparse: true }); + obj.eventsfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: 60 * 60 * 24 * 20 }); // Limit the power event log to 20 days (Seconds * Minutes * Hours * Days) + + // Setup the power collection and setup indexes + obj.powerfile = new Datastore({ filename: obj.parent.getConfigFilePath('meshcentral-power.db'), autoload: true }); + obj.powerfile.persistence.setAutocompactionInterval(36000); + obj.powerfile.ensureIndex({ fieldName: 'nodeid' }); + obj.powerfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: 60 * 60 * 24 * 10 }); // Limit the power event log to 10 days (Seconds * Minutes * Hours * Days) } obj.SetupDatabase = function (func) { @@ -117,10 +185,15 @@ module.exports.CreateDB = function (parent) { // TODO: Remove all mesh links to invalid users // TODO: Remove all meshes that dont have any links + // Remote all the events and power events from the main collection. + // They are all in two seperate collections now. + obj.file.remove({ type: 'event' }, { multi: true }); + obj.file.remove({ type: 'power' }, { multi: true }); + // Remove all objects that have a "meshid" that no longer points to a valid mesh. obj.GetAllType('mesh', function (err, docs) { var meshlist = []; - if (err == null && docs.length > 0) { for (var i in docs) { meshlist.push(docs[i]._id); } } + if ((err == null) && (docs.length > 0)) { for (var i in docs) { meshlist.push(docs[i]._id); } } obj.file.remove({ meshid: { $exists: true, $nin: meshlist } }, { multi: true }); // Fix all of the creating & login to ticks by seconds, not milliseconds. @@ -161,6 +234,7 @@ module.exports.CreateDB = function (parent) { }); }; + // Database actions on the main collection obj.Set = function (data, func) { obj.file.update({ _id: data._id }, data, { upsert: true }, func); }; obj.Get = function (id, func) { obj.file.find({ _id: id }, func); }; obj.GetAll = function (func) { obj.file.find({}, func); }; @@ -175,22 +249,43 @@ module.exports.CreateDB = function (parent) { obj.RemoveAll = function (func) { obj.file.remove({}, { multi: true }, func); }; obj.RemoveAllOfType = function (type, func) { obj.file.remove({ type: type }, { multi: true }, func); }; obj.InsertMany = function (data, func) { obj.file.insert(data, func); }; - obj.StoreEvent = function (ids, source, event) { obj.file.insert(event); }; - obj.GetEvents = function (ids, domain, func) { if (obj.databaseType == 1) { obj.file.find({ type: 'event', domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).exec(func); } else { obj.file.find({ type: 'event', domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }, func); } }; - obj.GetEventsWithLimit = function (ids, domain, limit, func) { if (obj.databaseType == 1) { obj.file.find({ type: 'event', domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).exec(func); } else { obj.file.find({ type: 'event', domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit, func); } }; - obj.GetNodeEventsWithLimit = function (nodeid, domain, limit, func) { if (obj.databaseType == 1) { obj.file.find({ type: 'event', domain: domain, nodeid: nodeid }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).exec(func); } else { obj.file.find({ type: 'event', domain: domain, nodeid: nodeid }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit, func); } }; - obj.RemoveMesh = function (id) { obj.file.remove({ mesh: id }, { multi: true }); obj.file.remove({ _id: id }); obj.file.remove({ _id: 'nt' + id }); }; - obj.RemoveAllEvents = function (domain) { obj.file.remove({ type: 'event', domain: domain }, { multi: true }); }; + obj.RemoveMeshDocuments = function (id) { obj.file.remove({ meshid: id }, { multi: true }); obj.file.remove({ _id: 'nt' + id }); }; obj.MakeSiteAdmin = function (username, domain) { obj.Get('user/' + domain + '/' + username, function (err, docs) { if (docs.length == 1) { docs[0].siteadmin = 0xFFFFFFFF; obj.Set(docs[0]); } }); }; obj.DeleteDomain = function (domain, func) { obj.file.remove({ domain: domain }, { multi: true }, func); }; obj.SetUser = function (user) { var u = Clone(user); if (u.subscriptions) { delete u.subscriptions; } obj.Set(u); }; obj.dispose = function () { for (var x in obj) { if (obj[x].close) { obj[x].close(); } delete obj[x]; } }; - obj.clearOldEntries = function (type, days, domain) { var cutoff = Date.now() - (1000 * 60 * 60 * 24 * days); obj.file.remove({ type: type, time: { $lt: cutoff } }, { multi: true }); }; - obj.getPowerTimeline = function (nodeid, func) { if (obj.databaseType == 1) { obj.file.find({ type: 'power', nodeid: { $in: ['*', nodeid] } }).sort({ time: 1 }).exec(func); } else { obj.file.find({ type: 'power', nodeid: { $in: ['*', nodeid] } }).sort({ time: 1 }, func); } }; obj.getLocalAmtNodes = function (func) { obj.file.find({ type: 'node', host: { $exists: true, $ne: null }, intelamt: { $exists: true } }, func); }; obj.getAmtUuidNode = function (meshid, uuid, func) { obj.file.find({ type: 'node', meshid: meshid, 'intelamt.uuid': uuid }, func); }; obj.isMaxType = function (max, type, domainid, func) { if (max == null) { func(false); } else { obj.file.count({ type: type, domain: domainid }, function (err, count) { func((err != null) || (count > max)); }); } } + // Database actions on the events collection + obj.GetAllEvents = function (func) { obj.eventsfile.find({}, func); }; + obj.StoreEvent = function (event) { obj.eventsfile.insert(event); }; + obj.GetEvents = function (ids, domain, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).exec(func); } else { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }, func); } }; + obj.GetEventsWithLimit = function (ids, domain, limit, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).exec(func); } else { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit, func); } }; + obj.GetUserEvents = function (ids, domain, username, func) { + if (obj.databaseType == 1) { + obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).exec(func); + } else { + obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }, func); + } + }; + obj.GetUserEventsWithLimit = function (ids, domain, username, limit, func) { + if (obj.databaseType == 1) { + obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).exec(func); + } else { + obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit, func); + } + }; + obj.GetNodeEventsWithLimit = function (nodeid, domain, limit, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, nodeid: nodeid }, { type: 0, etype: 0, _id: 0, domain: 0, ids: 0, node: 0, nodeid: 0 }).sort({ time: -1 }).limit(limit).exec(func); } else { obj.eventsfile.find({ domain: domain, nodeid: nodeid }, { type: 0, etype: 0, _id: 0, domain: 0, ids: 0, node: 0, nodeid: 0 }).sort({ time: -1 }).limit(limit, func); } }; + obj.RemoveAllEvents = function (domain) { obj.eventsfile.remove({ domain: domain }, { multi: true }); }; + + // Database actions on the power collection + obj.getAllPower = function (func) { obj.powerfile.find({}, func); }; + obj.storePowerEvent = function (event, multiServer, func) { if (multiServer != null) { event.server = multiServer.serverid; } obj.powerfile.insert(event, func); }; + obj.getPowerTimeline = function (nodeid, func) { if (obj.databaseType == 1) { obj.powerfile.find({ nodeid: { $in: ['*', nodeid] } }, { _id: 0, nodeid: 0, s: 0 }).sort({ time: 1 }).exec(func); } else { obj.powerfile.find({ nodeid: { $in: ['*', nodeid] } }, { _id: 0, nodeid: 0, s: 0 }).sort({ time: 1 }, func); } }; + obj.removeAllPowerEvents = function (domain) { obj.powerfile.remove({ }, { multi: true }); }; + // Read a configuration file from the database obj.getConfigFile = function (path, func) { obj.Get('cfile/' + path, func); } @@ -254,17 +349,15 @@ module.exports.CreateDB = function (parent) { obj.file.aggregate([{ "$group": { _id: "$type", count: { $sum: 1 } } }], function (err, docs) { var counters = {}, totalCount = 0; for (var i in docs) { if (docs[i]._id != null) { counters[docs[i]._id] = docs[i].count; totalCount += docs[i].count; } } - func({ nodes: counters['node'], meshes: counters['mesh'], powerEvents: counters['power'], users: counters['user'], total: totalCount }); + func({ nodes: counters['node'], meshes: counters['mesh'], users: counters['user'], total: totalCount }); }) } else { // NeDB version obj.file.count({ type: 'node' }, function (err, nodeCount) { obj.file.count({ type: 'mesh' }, function (err, meshCount) { - obj.file.count({ type: 'power' }, function (err, powerCount) { - obj.file.count({ type: 'user' }, function (err, userCount) { - obj.file.count({}, function (err, totalCount) { - func({ nodes: nodeCount, meshes: meshCount, powerEvents: powerCount, users: userCount, nodeInterfaces: nodeInterfaceCount, notes: noteCount, connectEvent: nodeLastConnectCount, smbios: nodeSmbiosCount, total: totalCount }); - }); + obj.file.count({ type: 'user' }, function (err, userCount) { + obj.file.count({}, function (err, totalCount) { + func({ nodes: nodeCount, meshes: meshCount, powerEvents: powerCount, users: userCount, nodeInterfaces: nodeInterfaceCount, notes: noteCount, connectEvent: nodeLastConnectCount, smbios: nodeSmbiosCount, total: totalCount }); }); }); }); diff --git a/meshagent.js b/meshagent.js index 3f446cfc..a196020c 100644 --- a/meshagent.js +++ b/meshagent.js @@ -439,6 +439,31 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { obj.db.SetUser(adminUser); obj.parent.parent.DispatchEvent(['*', obj.dbMeshKey, adminUser._id], obj, { etype: 'mesh', username: adminUser.name, meshid: obj.dbMeshKey, name: meshname, mtype: 2, desc: '', action: 'createmesh', links: links, msg: 'Mesh created: ' + obj.meshid, domain: domain.id }); } + } else { + if ((mesh.deleted != null) && (mesh.links)) { + // Must un-delete this mesh + var ids = ['*', mesh._id]; + + // See if users still exists, if so, add links to the mesh + for (var userid in mesh.links) { + var user = obj.parent.users[userid]; + if (user) { + if (user.links == null) { user.links = {}; } + if (user.links[mesh._id] == null) { + user.links[mesh._id] = { rights: mesh.links[userid].rights }; + ids.push(user._id); + obj.db.SetUser(user); + } + } + } + + // Send out an event indicating this mesh was "created" + obj.parent.parent.DispatchEvent(ids, obj, { etype: 'mesh', meshid: mesh._id, name: mesh.name, mtype: mesh.mtype, desc: mesh.desc, action: 'createmesh', links: mesh.links, msg: 'Mesh undeleted: ' + mesh._id, domain: domain.id }); + + // Mark the mesh as active + delete mesh.deleted; + obj.db.Set(obj.common.escapeLinksFieldName(mesh)); + } } return mesh; } @@ -515,7 +540,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { // If we disconnect, the agent will just reconnect. We need to log this or tell agent to connect in a few hours. console.log('Agent connected with invalid mesh type, holding connection (' + obj.remoteaddrport + ').'); return; - } + } // Mark when this device connected obj.connectTime = Date.now(); diff --git a/meshcentral.js b/meshcentral.js index b3582a5c..737c13ff 100644 --- a/meshcentral.js +++ b/meshcentral.js @@ -240,9 +240,9 @@ function CreateMeshCentralServer(config, args) { if (obj.args.showusers) { obj.db.GetAllType('user', function (err, docs) { console.log(docs); process.exit(); }); return; } if (obj.args.shownodes) { obj.db.GetAllType('node', function (err, docs) { console.log(docs); process.exit(); }); return; } if (obj.args.showmeshes) { obj.db.GetAllType('mesh', function (err, docs) { console.log(docs); process.exit(); }); return; } - if (obj.args.showevents) { obj.db.GetAllType('event', function (err, docs) { console.log(docs); process.exit(); }); return; } - if (obj.args.showpower) { obj.db.GetAllType('power', function (err, docs) { console.log(docs); process.exit(); }); return; } - if (obj.args.clearpower) { obj.db.RemoveAllOfType('power', function () { process.exit(); }); return; } + if (obj.args.showevents) { obj.db.GetAllEvents(function (err, docs) { console.log(docs); process.exit(); }); return; } + if (obj.args.showpower) { obj.db.getAllPower(function (err, docs) { console.log(docs); process.exit(); }); return; } + if (obj.args.clearpower) { obj.db.removeAllPowerEvents(function () { process.exit(); }); return; } if (obj.args.showiplocations) { obj.db.GetAllType('iploc', function (err, docs) { console.log(docs); process.exit(); }); return; } if (obj.args.logintoken) { obj.getLoginToken(obj.args.logintoken, function (r) { console.log(r); process.exit(); }); return; } if (obj.args.logintokenkey) { obj.showLoginTokenKey(function (r) { console.log(r); process.exit(); }); return; } @@ -535,11 +535,6 @@ function CreateMeshCentralServer(config, args) { if (obj.args.notls == null && obj.args.redirport == null) obj.args.redirport = 80; if (obj.args.minifycore === 0) obj.args.minifycore = false; - // Clear old event entries and power entires - // TODO: Replace this is delete indexes on NeDB and MongoDB. - obj.db.clearOldEntries('event', 30); // Clear all event entires that are older than 30 days. - obj.db.clearOldEntries('power', 10); // Clear all event entires that are older than 10 days. If a node is connected longer than 10 days, current power state will be used for everything. - // Setup a site administrator if ((obj.args.admin) && (typeof obj.args.admin == 'string')) { var adminname = obj.args.admin.split('/'); @@ -580,11 +575,7 @@ function CreateMeshCentralServer(config, args) { obj.db.cleanup(); // Set all nodes to power state of unknown (0) - if (obj.multiServer == null) { - obj.db.file.insert({ type: 'power', time: Date.now(), nodeid: '*', power: 0, s: 1 }); - } else { - obj.db.file.insert({ type: 'power', time: Date.now(), nodeid: '*', power: 0, s: 1, server: obj.multiServer.serverid }); - } + obj.db.storePowerEvent({ time: new Date(), nodeid: '*', power: 0, s: 1 }, obj.multiServer); // s:1 indicates that the server is starting up. // Read or setup database configuration values obj.db.Get('dbconfig', function (err, dbconfig) { @@ -800,13 +791,6 @@ function CreateMeshCentralServer(config, args) { } }); } - - // Clear old event entries and power entires - obj.db.clearOldEntries('event', 30); // Clear all event entires that are older than 30 days. - obj.db.clearOldEntries('power', 10); // Clear all event entires that are older than 10 days. If a node is connected longer than 10 days, current power state will be used for everything. - - // Perform other database cleanup - obj.db.cleanup(); }; // Stop the Meshcentral server @@ -818,9 +802,7 @@ function CreateMeshCentralServer(config, args) { obj.DispatchEvent(['*'], obj, { etype: 'server', action: 'stopped', msg: 'Server stopped' }); // Set all nodes to power state of unknown (0) - var record = { type: 'power', time: Date.now(), nodeid: '*', power: 0, s: 2 }; - if (obj.multiServer != null) { record.server = obj.multiServer.serverid; } - obj.db.file.insert(record, function () { + obj.db.storePowerEvent({ time: new Date(), nodeid: '*', power: 0, s: 2 }, obj.multiServer, function () { // s:2 indicates that the server is shutting down. if (restoreFile) { obj.debug(1, 'Server stopped, updating settings: ' + restoreFile); console.log('Updating settings folder...'); @@ -879,11 +861,13 @@ function CreateMeshCentralServer(config, args) { if (!obj.db) return; obj.debug(3, 'DispatchEvent', ids); - if (typeof event == 'object') { - event.type = 'event'; - event.time = Date.now(); - event.ids = ids; - if (!event.nolog) { obj.db.StoreEvent(ids, source, event); } + if ((typeof event == 'object') && (!event.nolog)) { + event.time = new Date(); + // The event we store is going to skip some of the fields so we don't store too much stuff in the database. + var storeEvent = {}; + for (var i in event) { if (i != 'node') { storeEvent[i] = event[i]; } } // Skip the "node" field. May skip more in the future. + storeEvent.ids = ids; + obj.db.StoreEvent(storeEvent); } var targets = []; // List of targets we dispatched the event to, we don't want to dispatch to the same target twice. for (var j in ids) { @@ -993,9 +977,9 @@ function CreateMeshCentralServer(config, args) { eventConnectChange = 1; // Set new power state in database - var record = { type: 'power', time: connectTime, nodeid: nodeid, power: powerState }; - if (oldPowerState != null) record.oldPower = oldPowerState; - obj.db.file.insert(record); + var record = { time: new Date(connectTime), nodeid: nodeid, power: powerState }; + if (oldPowerState != null) { record.oldPower = oldPowerState; } + obj.db.storePowerEvent(record, obj.multiServer); } // Event the node connection change @@ -1024,9 +1008,9 @@ function CreateMeshCentralServer(config, args) { state.powerState = powerState; // Set new power state in database - var record = { type: 'power', time: connectTime, nodeid: nodeid, power: powerState, server: obj.multiServer.serverid }; - if (oldPowerState != null) record.oldPower = oldPowerState; - obj.db.file.insert(record); + var record = { time: new Date(connectTime), nodeid: nodeid, power: powerState, server: obj.multiServer.serverid }; + if (oldPowerState != null) { record.oldPower = oldPowerState; } + obj.db.storePowerEvent(record, obj.multiServer); } // Update the combined node state @@ -1068,7 +1052,7 @@ function CreateMeshCentralServer(config, args) { eventConnectChange = 1; // Set new power state in database - obj.db.file.insert({ type: 'power', time: Date.now(), nodeid: nodeid, power: powerState, oldPower: oldPowerState }); + obj.db.storePowerEvent({ time: new Date(), nodeid: nodeid, power: powerState, oldPower: oldPowerState }, obj.multiServer); } // Event the node connection change diff --git a/meshuser.js b/meshuser.js index 2daba107..abb1cd10 100644 --- a/meshuser.js +++ b/meshuser.js @@ -270,7 +270,7 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use { // Request a list of all meshes this user as rights to var docs = []; - for (i in user.links) { if (obj.parent.meshes[i]) { docs.push(obj.parent.meshes[i]); } } + for (i in user.links) { if ((obj.parent.meshes[i]) && (obj.parent.meshes[i].deleted == null)) { docs.push(obj.parent.meshes[i]); } } try { ws.send(JSON.stringify({ action: 'meshes', meshes: docs, tag: command.tag })); } catch (ex) { } break; } @@ -325,6 +325,7 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use var timeline = [], time = null, previousPower; for (i in docs) { var doc = docs[i]; + doc.time = Date.parse(doc.time); if (time == null) { // First element time = doc.time; @@ -496,10 +497,16 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use var filter = ['user/' + domain.id + '/' + command.user.toLowerCase()]; if ((command.limit == null) || (typeof command.limit != 'number')) { // Send the list of all events for this session - obj.db.GetEvents(filter, domain.id, function (err, docs) { if (err != null) return; try { ws.send(JSON.stringify({ action: 'events', events: docs, user: command.user, tag: command.tag })); } catch (ex) { } }); + obj.db.GetUserEvents(filter, domain.id, command.user, function (err, docs) { + if (err != null) return; + try { ws.send(JSON.stringify({ action: 'events', events: docs, user: command.user, tag: command.tag })); } catch (ex) { } + }); } else { // Send the list of most recent events for this session, up to 'limit' count - obj.db.GetEventsWithLimit(filter, domain.id, command.limit, function (err, docs) { if (err != null) return; try { ws.send(JSON.stringify({ action: 'events', events: docs, user: command.user, tag: command.tag })); } catch (ex) { } }); + obj.db.GetUserEventsWithLimit(filter, domain.id, command.user, command.limit, function (err, docs) { + if (err != null) return; + try { ws.send(JSON.stringify({ action: 'events', events: docs, user: command.user, tag: command.tag })); } catch (ex) { } + }); } } else if (obj.common.validateString(command.nodeid, 0, 128) == true) { // Device filtered events // TODO: Check that the user has access to this nodeid @@ -507,16 +514,25 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use if (obj.common.validateInt(command.limit, 1, 60000) == true) { limit = command.limit; } // Send the list of most recent events for this session, up to 'limit' count - obj.db.GetNodeEventsWithLimit(command.nodeid, domain.id, limit, function (err, docs) { if (err != null) return; try { ws.send(JSON.stringify({ action: 'events', events: docs, nodeid: command.nodeid, tag: command.tag })); } catch (ex) { } }); + obj.db.GetNodeEventsWithLimit(command.nodeid, domain.id, limit, function (err, docs) { + if (err != null) return; + try { ws.send(JSON.stringify({ action: 'events', events: docs, nodeid: command.nodeid, tag: command.tag })); } catch (ex) { } + }); } else { // All events var filter = user.subscriptions; if ((command.limit == null) || (typeof command.limit != 'number')) { // Send the list of all events for this session - obj.db.GetEvents(filter, domain.id, function (err, docs) { if (err != null) return; try { ws.send(JSON.stringify({ action: 'events', events: docs, user: command.user, tag: command.tag })); } catch (ex) { } }); + obj.db.GetEvents(filter, domain.id, function (err, docs) { + if (err != null) return; + try { ws.send(JSON.stringify({ action: 'events', events: docs, user: command.user, tag: command.tag })); } catch (ex) { } + }); } else { // Send the list of most recent events for this session, up to 'limit' count - obj.db.GetEventsWithLimit(filter, domain.id, command.limit, function (err, docs) { if (err != null) return; try { ws.send(JSON.stringify({ action: 'events', events: docs, user: command.user, tag: command.tag })); } catch (ex) { } }); + obj.db.GetEventsWithLimit(filter, domain.id, command.limit, function (err, docs) { + if (err != null) return; + try { ws.send(JSON.stringify({ action: 'events', events: docs, user: command.user, tag: command.tag })); } catch (ex) { } + }); } } break; @@ -916,7 +932,7 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use if (obj.common.validateString(command.meshid, 1, 1024) == false) break; // Check the meshid obj.db.Get(command.meshid, function (err, meshes) { if (meshes.length != 1) return; - mesh = obj.common.unEscapeLinksFieldName(meshes[0]); + var mesh = obj.common.unEscapeLinksFieldName(meshes[0]); // Check if this user has rights to do this if (mesh.links[user._id] == null || mesh.links[user._id].rights != 0xFFFFFFFF) return; @@ -930,9 +946,11 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use var links = meshes[i].links; for (var j in links) { var xuser = obj.parent.users[j]; - delete xuser.links[meshes[i]._id]; - obj.db.SetUser(xuser); - obj.parent.parent.DispatchEvent([xuser._id], obj, 'resubscribe'); + if (xuser && xuser.links) { + delete xuser.links[meshes[i]._id]; + obj.db.SetUser(xuser); + obj.parent.parent.DispatchEvent([xuser._id], obj, 'resubscribe'); + } } } @@ -943,8 +961,15 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use } catch (e) { } obj.parent.parent.RemoveEventDispatchId(command.meshid); // Remove all subscriptions to this mesh - obj.db.RemoveMesh(command.meshid); // Remove mesh from database - delete obj.parent.meshes[command.meshid]; // Remove mesh from memory + + // Mark the mesh as deleted + var dbmesh = meshes[0]; + dbmesh.deleted = new Date(); // Mark the time this mesh was deleted, we can expire it at some point. + obj.db.Set(obj.common.escapeLinksFieldName(mesh)); // We don't really delete meshes because if a device connects to is again, we will up-delete it. + obj.parent.meshes[command.meshid] = mesh; // Update the mesh in memory; + + // Delete all devices attached to this mesh in the database + obj.db.RemoveMeshDocuments(command.meshid); }); break; } diff --git a/package.json b/package.json index fea01801..11161818 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "meshcentral", - "version": "0.2.8-q", + "version": "0.2.8-r", "keywords": [ "Remote Management", "Intel AMT", diff --git a/views/default-min.handlebars b/views/default-min.handlebars index 44e41270..deb1b54b 100644 --- a/views/default-min.handlebars +++ b/views/default-min.handlebars @@ -1 +1 @@ - MeshCentral
{{{title}}}
{{{title2}}}

{{{logoutControl}}}

↔
 

\ No newline at end of file + MeshCentral
{{{title}}}
{{{title2}}}

{{{logoutControl}}}

↔
 

\ No newline at end of file diff --git a/views/default.handlebars b/views/default.handlebars index d468fb33..043aede6 100644 --- a/views/default.handlebars +++ b/views/default.handlebars @@ -6210,8 +6210,7 @@ function eventsUpdate() { var x = '', dateHeader = null; for (var i in events) { - var event = events[i]; - var time = new Date(event.time); + var event = events[i], time = new Date(event.time); if (time.toLocaleDateString() != dateHeader) { if (dateHeader != null) x += ''; x += ''; diff --git a/webserver.js b/webserver.js index d5b69fc2..24118953 100644 --- a/webserver.js +++ b/webserver.js @@ -193,7 +193,7 @@ module.exports.CreateWebServer = function (parent, db, args, certificates) { // Fetch all meshes from the database, keep this in memory obj.db.GetAllType('mesh', function (err, docs) { obj.common.unEscapeAllLinksFieldName(docs); - for (var i in docs) { obj.meshes[docs[i]._id] = docs[i]; } + for (var i in docs) {obj.meshes[docs[i]._id] = docs[i]; } // Get all meshes, including deleted ones. // We loaded the users and mesh state, start the server serverStart(); @@ -2275,24 +2275,26 @@ module.exports.CreateWebServer = function (parent, db, args, certificates) { // Creates a login token using the user/pass that is passed in as URL arguments. // For example: https://localhost/createLoginToken.ashx?user=admin&pass=admin&a=3 // It's not advised to use this to create login tokens since the URL is often logged and you got credentials in the URL. - // However, people want it so here it is. - obj.app.get(url + 'createLoginToken.ashx', function (req, res) { - // A web socket session can be authenticated in many ways (Default user, session, user/pass and cookie). Check authentication here. - if ((req.query.user != null) && (req.query.pass != null)) { - // A user/pass is provided in URL arguments - obj.authenticate(req.query.user, req.query.pass, getDomain(req), function (err, userid) { - if ((err == null) && (obj.users[userid])) { - // User is authenticated, create a token - var x = { a: 3 }; for (var i in req.query) { if ((i != 'user') && (i != 'pass')) { x[i] = obj.common.toNumber(req.query[i]); } } x.u = userid; - res.send(obj.parent.encodeCookie(x, obj.parent.loginCookieEncryptionKey)); - } else { - res.sendStatus(404); - } - }); - } else { - res.sendStatus(404); - } - }); + // Since it's bad, it's only offered when an untrusted certificate is used as a way to help developers get started. + if (isTrustedCert() == false) { + obj.app.get(url + 'createLoginToken.ashx', function (req, res) { + // A web socket session can be authenticated in many ways (Default user, session, user/pass and cookie). Check authentication here. + if ((req.query.user != null) && (req.query.pass != null)) { + // A user/pass is provided in URL arguments + obj.authenticate(req.query.user, req.query.pass, getDomain(req), function (err, userid) { + if ((err == null) && (obj.users[userid])) { + // User is authenticated, create a token + var x = { a: 3 }; for (var i in req.query) { if ((i != 'user') && (i != 'pass')) { x[i] = obj.common.toNumber(req.query[i]); } } x.u = userid; + res.send(obj.parent.encodeCookie(x, obj.parent.loginCookieEncryptionKey)); + } else { + res.sendStatus(404); + } + }); + } else { + res.sendStatus(404); + } + }); + } obj.app.get(url + 'stop', function (req, res) { res.send('Stopping Server, click here to login.'); setTimeout(function () { parent.Stop(); }, 500); });
' + time.toLocaleDateString() + '