From 887f2ba22e8533ca358672c1e72d6e2d30f5655c Mon Sep 17 00:00:00 2001 From: Ylian Saint-Hilaire Date: Tue, 26 Feb 2019 14:39:45 -0800 Subject: [PATCH] Server fixes & MongoDB performance improvements. --- db.js | 5 +++-- meshagent.js | 13 +++++++------ meshcentral.js | 37 ++++++++++++++++++++++--------------- meshuser.js | 13 +++++++------ package.json | 2 +- sample-config.json | 7 ++++--- views/default.handlebars | 10 +++++++++- 7 files changed, 53 insertions(+), 34 deletions(-) diff --git a/db.js b/db.js index d0a8ff8d..a0f2f85f 100644 --- a/db.js +++ b/db.js @@ -258,7 +258,6 @@ module.exports.CreateDB = function (parent) { obj.GetUserWithEmail = function (domain, email, func) { obj.file.find({ type: 'user', domain: domain, email: email }, { type: 0 }, func); }; obj.GetUserWithVerifiedEmail = function (domain, email, func) { obj.file.find({ type: 'user', domain: domain, email: email, emailVerified: true }, { type: 0 }, func); }; obj.Remove = function (id) { obj.file.remove({ _id: id }); }; - obj.RemoveNode = function (id) { obj.file.remove({ node: id }, { multi: true }); }; obj.RemoveAll = function (func) { obj.file.remove({}, { multi: true }, func); }; obj.RemoveAllOfType = function (type, func) { obj.file.remove({ type: type }, { multi: true }, func); }; obj.InsertMany = function (data, func) { obj.file.insert(data, func); }; @@ -292,12 +291,14 @@ module.exports.CreateDB = function (parent) { }; obj.GetNodeEventsWithLimit = function (nodeid, domain, limit, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, nodeid: nodeid }, { type: 0, etype: 0, _id: 0, domain: 0, ids: 0, node: 0, nodeid: 0 }).sort({ time: -1 }).limit(limit).exec(func); } else { obj.eventsfile.find({ domain: domain, nodeid: nodeid }, { type: 0, etype: 0, _id: 0, domain: 0, ids: 0, node: 0, nodeid: 0 }).sort({ time: -1 }).limit(limit, func); } }; obj.RemoveAllEvents = function (domain) { obj.eventsfile.remove({ domain: domain }, { multi: true }); }; + obj.RemoveAllNodeEvents = function (domain, nodeid) { obj.eventsfile.remove({ domain: domain, nodeid: nodeid }, { multi: true }); }; // Database actions on the power collection obj.getAllPower = function (func) { obj.powerfile.find({}, func); }; obj.storePowerEvent = function (event, multiServer, func) { if (multiServer != null) { event.server = multiServer.serverid; } obj.powerfile.insert(event, func); }; obj.getPowerTimeline = function (nodeid, func) { if (obj.databaseType == 1) { obj.powerfile.find({ nodeid: { $in: ['*', nodeid] } }, { _id: 0, nodeid: 0, s: 0 }).sort({ time: 1 }).exec(func); } else { obj.powerfile.find({ nodeid: { $in: ['*', nodeid] } }, { _id: 0, nodeid: 0, s: 0 }).sort({ time: 1 }, func); } }; - obj.removeAllPowerEvents = function (domain) { obj.powerfile.remove({ }, { multi: true }); }; + obj.removeAllPowerEvents = function () { obj.powerfile.remove({}, { multi: true }); }; + obj.removeAllPowerEventsForNode = function (nodeid) { obj.powerfile.remove({ nodeid: nodeid }, { multi: true }); }; // Database actions on the SMBIOS collection obj.SetSMBIOS = function (smbios, func) { obj.smbiosfile.update({ _id: smbios._id }, smbios, { upsert: true }, func); }; diff --git a/meshagent.js b/meshagent.js index c882633d..c32063b7 100644 --- a/meshagent.js +++ b/meshagent.js @@ -73,12 +73,13 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { // If this is a temporary or recovery agent, or all devices in this group are temporary, remove the agent (0x20 = Temporary, 0x40 = Recovery) if (((obj.agentInfo) && (obj.agentInfo.capabilities) && ((obj.agentInfo.capabilities & 0x20) || (obj.agentInfo.capabilities & 0x40))) || ((mesh) && (mesh.flags) && (mesh.flags & 1))) { // Delete this node including network interface information and events - obj.db.Remove(obj.dbNodeKey); // Remove node with that id - obj.db.Remove('if' + obj.dbNodeKey); // Remove interface information - obj.db.Remove('nt' + obj.dbNodeKey); // Remove notes - obj.db.Remove('lc' + obj.dbNodeKey); // Remove last connect time - obj.db.RemoveSMBIOS(obj.dbNodeKey); // Remove SMBios data - obj.db.RemoveNode(obj.dbNodeKey); // Remove all entries with node:id + obj.db.Remove(obj.dbNodeKey); // Remove node with that id + obj.db.Remove('if' + obj.dbNodeKey); // Remove interface information + obj.db.Remove('nt' + obj.dbNodeKey); // Remove notes + obj.db.Remove('lc' + obj.dbNodeKey); // Remove last connect time + obj.db.RemoveSMBIOS(obj.dbNodeKey); // Remove SMBios data + obj.db.RemoveAllNodeEvents(obj.dbNodeKey); // Remove all events for this node + obj.db.removeAllPowerEventsForNode(obj.dbNodeKey); // Remove all power events for this node // Event node deletion obj.parent.parent.DispatchEvent(['*', obj.dbMeshKey], obj, { etype: 'node', action: 'removenode', nodeid: obj.dbNodeKey, domain: obj.domain.id, nolog: 1 }); diff --git a/meshcentral.js b/meshcentral.js index 7eaac236..014a588d 100644 --- a/meshcentral.js +++ b/meshcentral.js @@ -60,7 +60,7 @@ function CreateMeshCentralServer(config, args) { obj.serverKey = Buffer.from(obj.crypto.randomBytes(48), 'binary'); obj.loginCookieEncryptionKey = null; obj.serverSelfWriteAllowed = true; - obj.taskLimiter = obj.common.createTaskLimiterQueue(8, 20, 60); // This is a task limiter queue to smooth out server work. + obj.taskLimiter = obj.common.createTaskLimiterQueue(20, 20, 60); // This is a task limiter queue to smooth out server work. try { obj.currentVer = JSON.parse(obj.fs.readFileSync(obj.path.join(__dirname, 'package.json'), 'utf8')).version; } catch (e) { } // Fetch server version // Setup the default configuration and files paths @@ -281,20 +281,26 @@ function CreateMeshCentralServer(config, args) { console.log(' --dbpulldatafiles * This will import files from meshcentral-data into the db.'); process.exit(); } else { - obj.db.RemoveAllOfType('cfile', function () { - if (obj.args.dbpushconfigfiles == '*') { obj.args.dbpushconfigfiles = obj.datapath; } - obj.fs.readdir(obj.datapath, (err, files) => { - var lockCount = 1 - for (var i in files) { - const file = files[i]; - if ((file == 'config.json') || file.endsWith('.key') || file.endsWith('.crt') || (file == 'terms.txt') || file.endsWith('.jpg') || file.endsWith('.png')) { - const path = obj.path.join(obj.args.dbpushconfigfiles, files[i]), binary = Buffer.from(obj.fs.readFileSync(path, { encoding: 'binary' }), 'binary'); - console.log('Pushing ' + file + ', ' + binary.length + ' bytes.'); - lockCount++; - obj.db.setConfigFile(file, obj.db.encryptData(obj.args.configkey, binary), function () { if ((--lockCount) == 0) { console.log('Done.'); process.exit(); } }); + if (obj.args.dbpushconfigfiles == '*') { obj.args.dbpushconfigfiles = obj.datapath; } + obj.fs.readdir(obj.args.dbpushconfigfiles, function (err, files) { + if (err != null) { console.log('Unable to read from folder ' + obj.args.dbpushconfigfiles); process.exit(); return; } + var configFound = false; + for (var i in files) { if (files[i] == 'config.json') { configFound = true; } } + if (configFound == false) { console.log('No config.json in folder ' + obj.args.dbpushconfigfiles); process.exit(); return; } + obj.db.RemoveAllOfType('cfile', function () { + obj.fs.readdir(obj.args.dbpushconfigfiles, function (err, files) { + var lockCount = 1 + for (var i in files) { + const file = files[i]; + if ((file == 'config.json') || file.endsWith('.key') || file.endsWith('.crt') || (file == 'terms.txt') || file.endsWith('.jpg') || file.endsWith('.png')) { + const path = obj.path.join(obj.args.dbpushconfigfiles, files[i]), binary = Buffer.from(obj.fs.readFileSync(path, { encoding: 'binary' }), 'binary'); + console.log('Pushing ' + file + ', ' + binary.length + ' bytes.'); + lockCount++; + obj.db.setConfigFile(file, obj.db.encryptData(obj.args.configkey, binary), function () { if ((--lockCount) == 0) { console.log('Done.'); process.exit(); } }); + } } - } - if (--lockCount == 0) { process.exit(); } + if (--lockCount == 0) { process.exit(); } + }); }); }); } @@ -318,7 +324,8 @@ function CreateMeshCentralServer(config, args) { if (binary == null) { console.log('Invalid config key.'); } else { - obj.fs.writeFileSync(obj.path.join(obj.args.dbpullconfigfiles, file), binary); + var fullFileName = obj.path.join(obj.args.dbpullconfigfiles, file); + try { obj.fs.writeFileSync(fullFileName, binary); } catch (ex) { console.log('Unable to write to ' + fullFileName); process.exit(); return; } console.log('Pulling ' + file + ', ' + binary.length + ' bytes.'); } } diff --git a/meshuser.js b/meshuser.js index 0dd0c95c..a67324ed 100644 --- a/meshuser.js +++ b/meshuser.js @@ -1272,12 +1272,13 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use if (mesh.links[user._id] == null || ((mesh.links[user._id].rights & 4) == 0)) return; // Delete this node including network interface information, events and timeline - obj.db.Remove(node._id); // Remove node with that id - obj.db.Remove('if' + node._id); // Remove interface information - obj.db.Remove('nt' + node._id); // Remove notes - obj.db.Remove('lc' + node._id); // Remove last connect time - obj.db.RemoveSMBIOS(node._id); // Remove SMBios data - obj.db.RemoveNode(node._id); // Remove all entries with node:id + obj.db.Remove(node._id); // Remove node with that id + obj.db.Remove('if' + node._id); // Remove interface information + obj.db.Remove('nt' + node._id); // Remove notes + obj.db.Remove('lc' + node._id); // Remove last connect time + obj.db.RemoveSMBIOS(node._id); // Remove SMBios data + obj.db.RemoveAllNodeEvents(node._id); // Remove all events for this node + obj.db.removeAllPowerEventsForNode(node._id); // Remove all power events for this node // Event node deletion obj.parent.parent.DispatchEvent(['*', node.meshid], obj, { etype: 'node', username: user.name, action: 'removenode', nodeid: node._id, msg: 'Removed device ' + node.name + ' from group ' + mesh.name, domain: domain.id }); diff --git a/package.json b/package.json index 1548c410..d3dcfe4c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "meshcentral", - "version": "0.2.9-c", + "version": "0.2.9-d", "keywords": [ "Remote Management", "Intel AMT", diff --git a/sample-config.json b/sample-config.json index 37df4636..f247c4d3 100644 --- a/sample-config.json +++ b/sample-config.json @@ -41,10 +41,11 @@ "": { "Title": "MyServer", "Title2": "Servername", - "TitlePicture": "title-sample.png", - "UserQuota": 1048576, - "MeshQuota": 248576, + "_TitlePicture": "title-sample.png", + "_UserQuota": 1048576, + "_MeshQuota": 248576, "NewAccounts": 1, + "_NewAccountEmailDomains": [ "sample.com" ], "Footer": "Twitter", "_CertUrl": "https://192.168.2.106:443/", "_PasswordRequirements": { "min": 8, "max": 128, "upper": 1, "lower": 1, "numeric": 1, "nonalpha": 1 }, diff --git a/views/default.handlebars b/views/default.handlebars index 2e9c050d..89482b1f 100644 --- a/views/default.handlebars +++ b/views/default.handlebars @@ -1567,7 +1567,7 @@ // This is an existing mesh if (message.event.name) { meshes[message.event.meshid].name = message.event.name; } if (message.event.desc) { meshes[message.event.meshid].desc = message.event.desc; } - if (message.event.flags) { meshes[message.event.meshid].flags = message.event.flags; } + if (message.event.flags != null) { meshes[message.event.meshid].flags = message.event.flags; } if (message.event.links) { meshes[message.event.meshid].links = message.event.links; } if (message.event.amt) { meshes[message.event.meshid].amt = message.event.amt; } @@ -6562,6 +6562,14 @@ if (user.quota) x += addDeviceAttribute('Server Quota', EscapeHtml(parseInt(user.quota) / 1024) + ' k'); x += addDeviceAttribute('Creation', new Date(user.creation * 1000).toLocaleString()); if (user.login) x += addDeviceAttribute('Last Login', new Date(user.login * 1000).toLocaleString()); + + var linkCount = 0, linkCountStr = 'None'; + if (user.links) { + for (var i in user.links) { linkCount++; } + if (linkCount == 1) { linkCountStr = '1 group'; } else if (linkCount > 1) { linkCountStr = linkCount + ' groups'; } + } + x += addDeviceAttribute('Device Groups', linkCountStr); + var multiFactor = 0; if ((user.otpsecret > 0) || (user.otphkeys > 0)) { multiFactor = 1;