From 3c0e7c9700d31862bfcd03df82a1ece1f8ec88b8 Mon Sep 17 00:00:00 2001 From: Ylian Saint-Hilaire Date: Wed, 29 May 2019 14:36:14 -0700 Subject: [PATCH] Improved MongoDB change stream. --- db.js | 50 +++++++++++++++++++++++++-------- meshuser.js | 33 ++++++++++++++-------- package.json | 2 +- views/default-mobile.handlebars | 1 + views/default.handlebars | 3 +- webserver.js | 16 ++++++++--- 6 files changed, 76 insertions(+), 29 deletions(-) diff --git a/db.js b/db.js index c487b806..99e23849 100644 --- a/db.js +++ b/db.js @@ -231,19 +231,45 @@ module.exports.CreateDB = function (parent, func) { // Setup the changeStream on the MongoDB main collection if possible if (parent.args.mongodbchangestream == true) { - obj.fileChangeStream = obj.file.watch([{ $match: { 'fullDocument.type': { $in: ['node', 'mesh', 'user'] } } }], { fullDocument: 'updateLookup' }); + obj.fileChangeStream = obj.file.watch( [ { $match: { $or: [{ 'fullDocument.type': { $in: ['node', 'mesh', 'user'] } }, { 'operationType': 'delete' }] } } ], { fullDocument: 'updateLookup' }); obj.fileChangeStream.on('change', function (change) { - switch (change.fullDocument.type) { - case 'node': { dbNodeChange(change); break; } // A node has changed - case 'mesh': { dbMeshChange(change); break; } // A device group has changed - case 'user': { dbUserChange(change); break; } // A user account has changed + if (change.operationType == 'update') { + switch (change.fullDocument.type) { + case 'node': { dbNodeChange(change, false); break; } // A node has changed + case 'mesh': { dbMeshChange(change, false); break; } // A device group has changed + case 'user': { dbUserChange(change, false); break; } // A user account has changed + } + } else if (change.operationType == 'insert') { + switch (change.fullDocument.type) { + case 'node': { dbNodeChange(change, true); break; } // A node has added + case 'mesh': { dbMeshChange(change, true); break; } // A device group has created + case 'user': { dbUserChange(change, true); break; } // A user account has created + } + } else if (change.operationType == 'delete') { + var splitId = change.documentKey._id.split('/'); + switch (splitId[0]) { + case 'node': { + //Not Good: Problem here is that we don't know what meshid the node belonged to before the delete. + //parent.DispatchEvent(['*', node.meshid], obj, { etype: 'node', action: 'removenode', nodeid: change.documentKey._id, domain: splitId[1] }); + break; + } + case 'mesh': { + parent.DispatchEvent(['*', node.meshid], obj, { etype: 'mesh', action: 'deletemesh', meshid: change.documentKey._id, domain: splitId[1] }); + break; + } + case 'user': { + //Not Good: This is not a perfect user removal because we don't know what groups the user was in. + //parent.DispatchEvent(['*', 'server-users'], obj, { etype: 'user', action: 'accountremove', userid: change.documentKey._id, domain: splitId[1], username: splitId[2] }); + break; + } + } } }); obj.changeStream = true; } // Setup MongoDB events collection and indexes - obj.eventsfile = db.collection('events'); // Collection containing all events + obj.eventsfile = db.collection('events'); // Collection containing all events obj.eventsfile.indexes(function (err, indexes) { // Check if we need to reset indexes var indexesByName = {}, indexCount = 0; @@ -773,16 +799,16 @@ module.exports.CreateDB = function (parent, func) { function padNumber(number, digits) { return Array(Math.max(digits - String(number).length + 1, 0)).join(0) + number; } // Called when a node has changed - function dbNodeChange(nodeChange) { + function dbNodeChange(nodeChange, added) { const node = nodeChange.fullDocument; if (node.intelamt && node.intelamt.pass) { delete node.intelamt.pass; } // Remove the Intel AMT password before eventing this. - parent.DispatchEvent(['*', node.meshid], obj, { etype: 'node', action: 'changenode', node: node, nodeid: node._id, domain: node.domain, nolog: 1 }); + parent.DispatchEvent(['*', node.meshid], obj, { etype: 'node', action: (added ? 'addnode' : 'changenode'), node: node, nodeid: node._id, domain: node.domain, nolog: 1 }); } // Called when a device group has changed - function dbMeshChange(meshChange) { + function dbMeshChange(meshChange, added) { const mesh = meshChange.fullDocument; - mesh.action = 'meshchange'; + if (mesh.deleted) { mesh.action = 'deletemesh'; } else { mesh.action = (added ? 'createmesh' : 'meshchange'); } mesh.meshid = mesh._id; mesh.nolog = 1; delete mesh.type; @@ -791,9 +817,9 @@ module.exports.CreateDB = function (parent, func) { } // Called when a user account has changed - function dbUserChange(userChange) { + function dbUserChange(userChange, added) { const user = userChange.fullDocument; - parent.DispatchEvent(['*', 'server-users', user._id], obj, { etype: 'user', username: user.name, account: parent.webserver.CloneSafeUser(user), action: 'accountchange', domain: user.domain, nolog: 1 }); + parent.DispatchEvent(['*', 'server-users', user._id], obj, { etype: 'user', username: user.name, account: parent.webserver.CloneSafeUser(user), action: (added ? 'accountcreate' : 'accountchange'), domain: user.domain, nolog: 1 }); } return obj; diff --git a/meshuser.js b/meshuser.js index 63f131b0..e428d572 100644 --- a/meshuser.js +++ b/meshuser.js @@ -1038,13 +1038,15 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use newuser.hash = hash; db.SetUser(newuser); - var targets = ['*', 'server-users']; + var event, targets = ['*', 'server-users']; if (newuser.groups) { for (var i in newuser.groups) { targets.push('server-users:' + i); } } if (newuser.email == null) { - parent.parent.DispatchEvent(targets, obj, { etype: 'user', username: newuser.name, account: parent.CloneSafeUser(newuser), action: 'accountcreate', msg: 'Account created, username is ' + newuser.name, domain: domain.id }); + event = { etype: 'user', username: newuser.name, account: parent.CloneSafeUser(newuser), action: 'accountcreate', msg: 'Account created, username is ' + newuser.name, domain: domain.id }; } else { - parent.parent.DispatchEvent(targets, obj, { etype: 'user', username: newuser.name, account: parent.CloneSafeUser(newuser), action: 'accountcreate', msg: 'Account created, email is ' + newuser.email, domain: domain.id }); + event = { etype: 'user', username: newuser.name, account: parent.CloneSafeUser(newuser), action: 'accountcreate', msg: 'Account created, email is ' + newuser.email, domain: domain.id }; } + if (parent.db.changeStream) { event.noact = 1; } // If DB change stream is active, don't use this event to create the user. Another event will come. + parent.parent.DispatchEvent(targets, obj, event); }, newuser); } } @@ -1094,13 +1096,15 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use newuser.hash = hash; db.SetUser(newuser); - var targets = ['*', 'server-users']; + var event, targets = ['*', 'server-users']; if (newuser.groups) { for (var i in newuser.groups) { targets.push('server-users:' + i); } } if (command.email == null) { - parent.parent.DispatchEvent(targets, obj, { etype: 'user', username: newusername, account: parent.CloneSafeUser(newuser), action: 'accountcreate', msg: 'Account created, username is ' + command.user, domain: domain.id }); + event = { etype: 'user', username: newusername, account: parent.CloneSafeUser(newuser), action: 'accountcreate', msg: 'Account created, username is ' + command.user, domain: domain.id }; } else { - parent.parent.DispatchEvent(targets, obj, { etype: 'user', username: newusername, account: parent.CloneSafeUser(newuser), action: 'accountcreate', msg: 'Account created, email is ' + command.email, domain: domain.id }); + event = { etype: 'user', username: newusername, account: parent.CloneSafeUser(newuser), action: 'accountcreate', msg: 'Account created, email is ' + command.email, domain: domain.id }; } + if (parent.db.changeStream) { event.noact = 1; } // If DB change stream is active, don't use this event to create the user. Another event will come. + parent.parent.DispatchEvent(targets, obj, event); }, 0); } }); @@ -1390,7 +1394,9 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use user.links[meshid] = { rights: 0xFFFFFFFF }; user.subscriptions = parent.subscribe(user._id, ws); db.SetUser(user); - parent.parent.DispatchEvent(['*', meshid, user._id], obj, { etype: 'mesh', username: user.name, meshid: meshid, name: command.meshname, mtype: command.meshtype, desc: command.desc, action: 'createmesh', links: links, msg: 'Mesh created: ' + command.meshname, domain: domain.id }); + var event = { etype: 'mesh', username: user.name, meshid: meshid, name: command.meshname, mtype: command.meshtype, desc: command.desc, action: 'createmesh', links: links, msg: 'Mesh created: ' + command.meshname, domain: domain.id }; + if (db.changeStream) { event.noact = 1; } // If DB change stream is active, don't use this event to create the mesh. Another event will come. + parent.parent.DispatchEvent(['*', meshid, user._id], obj, event); }); } break; @@ -1408,7 +1414,9 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use if ((command.meshid.split('/').length != 3) || (command.meshid.split('/')[1] != domain.id)) return; // Invalid domain, operation only valid for current domain // Fire the removal event first, because after this, the event will not route - parent.parent.DispatchEvent(['*', command.meshid], obj, { etype: 'mesh', username: user.name, meshid: command.meshid, name: command.meshname, action: 'deletemesh', msg: 'Mesh deleted: ' + command.meshname, domain: domain.id }); + var event = { etype: 'mesh', username: user.name, meshid: command.meshid, name: command.meshname, action: 'deletemesh', msg: 'Mesh deleted: ' + command.meshname, domain: domain.id }; + if (db.changeStream) { event.noact = 1; } // If DB change stream is active, don't use this event to remove the mesh. Another event will come. + parent.parent.DispatchEvent(['*', command.meshid], obj, event); // Remove all user links to this mesh for (i in meshes) { @@ -1722,12 +1730,15 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use db.RemoveAllNodeEvents(node._id); // Remove all events for this node db.removeAllPowerEventsForNode(node._id); // Remove all power events for this node db.Get('ra' + obj.dbNodeKey, function (err, nodes) { - if ((nodes != null) && (nodes.length == 1)) { db.Remove('da' + nodes[0].daid); } // Remove diagnostic agent to real agent link - db.Remove('ra' + node._id); // Remove real agent to diagnostic agent link + if ((nodes != null) && (nodes.length == 1)) { db.Remove('da' + nodes[0].daid); } // Remove diagnostic agent to real agent link + db.Remove('ra' + node._id); // Remove real agent to diagnostic agent link }); // Event node deletion - parent.parent.DispatchEvent(['*', node.meshid], obj, { etype: 'node', username: user.name, action: 'removenode', nodeid: node._id, msg: 'Removed device ' + node.name + ' from group ' + mesh.name, domain: domain.id }); + var event = { etype: 'node', username: user.name, action: 'removenode', nodeid: node._id, msg: 'Removed device ' + node.name + ' from group ' + mesh.name, domain: domain.id }; + // TODO: We can't use the changeStream for node delete because we will not know the meshid the device was in. + //if (db.changeStream) { event.noact = 1; } // If DB change stream is active, don't use this event to remove the node. Another event will come. + parent.parent.DispatchEvent(['*', node.meshid], obj, event); // Disconnect all connections if needed var state = parent.parent.GetConnectivityState(nodeid); diff --git a/package.json b/package.json index 74c6ef57..1cd1fd20 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "meshcentral", - "version": "0.3.5-j", + "version": "0.3.5-k", "keywords": [ "Remote Management", "Intel AMT", diff --git a/views/default-mobile.handlebars b/views/default-mobile.handlebars index 7e112abc..de300305 100644 --- a/views/default-mobile.handlebars +++ b/views/default-mobile.handlebars @@ -881,6 +881,7 @@ case 'addnode': { var node = message.event.node; if (!meshes[node.meshid]) break; // This is a node for a mesh we don't know. Happens when we are site administrator, we get all messages. + if (getNodeFromId(node._id) != null) break; // This node is already known. node.namel = node.name.toLowerCase(); if (node.rname) { node.rnamel = node.rname.toLowerCase(); } else { node.rnamel = node.namel; } node.meshnamel = meshes[node.meshid].name.toLowerCase(); diff --git a/views/default.handlebars b/views/default.handlebars index 07eb604d..27daac5d 100644 --- a/views/default.handlebars +++ b/views/default.handlebars @@ -1690,7 +1690,7 @@ } case 'createmesh': { // A new mesh was created - if (message.event.links[userinfo._id] != null) { // Check if this is a mesh create for a mesh we own. If site administrator, we get all messages so need to ignore some. + if ((meshes[message.event.meshid] == null) && (message.event.links[userinfo._id] != null)) { // Check if this is a mesh create for a mesh we own. If site administrator, we get all messages so need to ignore some. meshes[message.event.meshid] = { _id: message.event.meshid, name: message.event.name, mtype: message.event.mtype, desc: message.event.desc, links: message.event.links }; masterUpdate(4 + 128); meshserver.send({ action: 'files' }); @@ -1758,6 +1758,7 @@ case 'addnode': { var node = message.event.node; if (!meshes[node.meshid]) break; // This is a node for a mesh we don't know. Happens when we are site administrator, we get all messages. + if (getNodeFromId(node._id) != null) break; // This node is already known. node.namel = node.name.toLowerCase(); if (node.rname) { node.rnamel = node.rname.toLowerCase(); } else { node.rnamel = node.namel; } node.meshnamel = meshes[node.meshid].name.toLowerCase(); diff --git a/webserver.js b/webserver.js index 5c491c7f..fdc39f88 100644 --- a/webserver.js +++ b/webserver.js @@ -309,7 +309,9 @@ module.exports.CreateWebServer = function (parent, db, args, certificates) { if (usercount == 0) { user.siteadmin = 0xFFFFFFFF; /*if (domain.newaccounts === 2) { delete domain.newaccounts; }*/ } // If this is the first user, give the account site admin. obj.users[user._id] = user; obj.db.SetUser(user); - obj.parent.DispatchEvent(['*', 'server-users'], obj, { etype: 'user', userid: userid, username: username, account: obj.CloneSafeUser(user), action: 'accountcreate', msg: 'Account created, name is ' + name, domain: domain.id }); + var event = { etype: 'user', userid: userid, username: username, account: obj.CloneSafeUser(user), action: 'accountcreate', msg: 'Account created, name is ' + name, domain: domain.id }; + if (obj.db.changeStream) { event.noact = 1; } // If DB change stream is active, don't use this event to create the user. Another event will come. + obj.parent.DispatchEvent(['*', 'server-users'], obj, event); return fn(null, user._id); } else { // This is an existing user @@ -363,7 +365,9 @@ module.exports.CreateWebServer = function (parent, db, args, certificates) { if (usercount == 0) { user.siteadmin = 0xFFFFFFFF; /*if (domain.newaccounts === 2) { delete domain.newaccounts; }*/ } // If this is the first user, give the account site admin. obj.users[user._id] = user; obj.db.SetUser(user); - obj.parent.DispatchEvent(['*', 'server-users'], obj, { etype: 'user', username: user.name, account: obj.CloneSafeUser(user), action: 'accountcreate', msg: 'Account created, name is ' + name, domain: domain.id }); + var event = { etype: 'user', username: user.name, account: obj.CloneSafeUser(user), action: 'accountcreate', msg: 'Account created, name is ' + name, domain: domain.id }; + if (obj.db.changeStream) { event.noact = 1; } // If DB change stream is active, don't use this event to create the user. Another event will come. + obj.parent.DispatchEvent(['*', 'server-users'], obj, event); return fn(null, user._id); } else { // This is an existing user @@ -814,7 +818,9 @@ module.exports.CreateWebServer = function (parent, db, args, certificates) { // Send the verification email if ((obj.parent.mailserver != null) && (domain.auth != 'sspi') && (domain.auth != 'ldap') && (obj.common.validateEmail(user.email, 1, 256) == true)) { obj.parent.mailserver.sendAccountCheckMail(domain, user.name, user.email); } }, 0); - obj.parent.DispatchEvent(['*', 'server-users'], obj, { etype: 'user', username: user.name, account: obj.CloneSafeUser(user), action: 'accountcreate', msg: 'Account created, email is ' + req.body.email, domain: domain.id }); + var event = { etype: 'user', username: user.name, account: obj.CloneSafeUser(user), action: 'accountcreate', msg: 'Account created, email is ' + req.body.email, domain: domain.id }; + if (obj.db.changeStream) { event.noact = 1; } // If DB change stream is active, don't use this event to create the user. Another event will come. + obj.parent.DispatchEvent(['*', 'server-users'], obj, event); } res.redirect(domain.url); } @@ -1239,7 +1245,9 @@ module.exports.CreateWebServer = function (parent, db, args, certificates) { if (usercount == 0) { user2.siteadmin = 0xFFFFFFFF; } // If this is the first user, give the account site admin. obj.users[req.session.userid] = user2; obj.db.SetUser(user2); - obj.parent.DispatchEvent(['*', 'server-users'], obj, { etype: 'user', username: req.connection.user, account: obj.CloneSafeUser(user2), action: 'accountcreate', msg: 'Domain account created, user ' + req.connection.user, domain: domain.id }); + var event = { etype: 'user', username: req.connection.user, account: obj.CloneSafeUser(user2), action: 'accountcreate', msg: 'Domain account created, user ' + req.connection.user, domain: domain.id }; + if (obj.db.changeStream) { event.noact = 1; } // If DB change stream is active, don't use this event to create the user. Another event will come. + obj.parent.DispatchEvent(['*', 'server-users'], obj, event); } } }