Added server instrumentation
This commit is contained in:
Ylian Saint-Hilaire
parent
2041f9bba0
commit
f8c310d39f
|
|
@ -549,14 +549,47 @@ module.exports.CertificateOperations = function (parent) {
|
|||
// Accelerators, used to dispatch work to other processes
|
||||
const fork = require("child_process").fork;
|
||||
const program = require("path").join(__dirname, "meshaccelerator.js");
|
||||
const acceleratorTotalCount = require("os").cpus().length;
|
||||
const acceleratorTotalCount = 1; //require("os").cpus().length; // TODO: Check if this accelerator can scale.
|
||||
var acceleratorCreateCount = acceleratorTotalCount;
|
||||
var freeAccelerators = [];
|
||||
var pendingAccelerator = [];
|
||||
obj.acceleratorCertStore = null;
|
||||
|
||||
// Accelerator Stats
|
||||
var getAcceleratorFuncCalls = 0;
|
||||
var acceleratorStartFuncCall = 0;
|
||||
var acceleratorPerformSignatureFuncCall = 0;
|
||||
var acceleratorPerformSignaturePushFuncCall = 0;
|
||||
var acceleratorPerformSignatureRunFuncCall = 0;
|
||||
var acceleratorMessage = 0;
|
||||
var acceleratorMessageException = 0;
|
||||
var acceleratorMessageLastException = null;
|
||||
var acceleratorException = 0;
|
||||
var acceleratorLastException = null;
|
||||
|
||||
// Get stats about the accelerators
|
||||
obj.getAcceleratorStats = function () {
|
||||
return {
|
||||
acceleratorTotalCount: acceleratorTotalCount,
|
||||
acceleratorCreateCount: acceleratorCreateCount,
|
||||
freeAccelerators: freeAccelerators.length,
|
||||
pendingAccelerator: pendingAccelerator.length,
|
||||
getAcceleratorFuncCalls: getAcceleratorFuncCalls,
|
||||
startFuncCall: acceleratorStartFuncCall,
|
||||
performSignatureFuncCall: acceleratorPerformSignatureFuncCall,
|
||||
performSignaturePushFuncCall: acceleratorPerformSignaturePushFuncCall,
|
||||
performSignatureRunFuncCall: acceleratorPerformSignatureRunFuncCall,
|
||||
message: acceleratorMessage,
|
||||
messageException: acceleratorMessageException,
|
||||
messageLastException: acceleratorMessageLastException,
|
||||
exception: acceleratorException,
|
||||
lastException: acceleratorLastException
|
||||
};
|
||||
}
|
||||
|
||||
// Create a new accelerator module
|
||||
obj.getAccelerator = function () {
|
||||
getAcceleratorFuncCalls++;
|
||||
if (obj.acceleratorCertStore == null) { return null; }
|
||||
if (freeAccelerators.length > 0) { return freeAccelerators.pop(); }
|
||||
if (acceleratorCreateCount > 0) {
|
||||
|
|
@ -564,23 +597,26 @@ module.exports.CertificateOperations = function (parent) {
|
|||
var accelerator = fork(program, [], { stdio: ["pipe", "pipe", "pipe", "ipc"] });
|
||||
accelerator.accid = acceleratorCreateCount;
|
||||
accelerator.on("message", function (message) {
|
||||
this.func(this.tag, message);
|
||||
delete this.tag;
|
||||
if (pendingAccelerator.length > 0) {
|
||||
var x = pendingAccelerator.shift();
|
||||
if (x.tag) { this.tag = x.tag; delete x.tag; }
|
||||
accelerator.send(x);
|
||||
} else { freeAccelerators.push(this); }
|
||||
acceleratorMessage++;
|
||||
try { this.func(this.tag, message); } catch (ex) { acceleratorMessageException++; acceleratorMessageLastException = ex; }
|
||||
try {
|
||||
delete this.tag;
|
||||
if (pendingAccelerator.length > 0) {
|
||||
var x = pendingAccelerator.shift();
|
||||
if (x.tag) { this.tag = x.tag; delete x.tag; }
|
||||
accelerator.send(x);
|
||||
} else { freeAccelerators.push(this); }
|
||||
} catch (ex) { acceleratorException++; acceleratorLastException = ex; }
|
||||
});
|
||||
accelerator.send({ action: "setState", certs: obj.acceleratorCertStore });
|
||||
return accelerator;
|
||||
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
// Set the state of the accelerators. This way, we don"t have to send certificate & keys to them each time.
|
||||
obj.acceleratorStart = function (certificates) {
|
||||
acceleratorStartFuncCall++;
|
||||
if (obj.acceleratorCertStore != null) { console.error("ERROR: Accelerators can only be started once."); return; }
|
||||
obj.acceleratorCertStore = [{ cert: certificates.agent.cert, key: certificates.agent.key }];
|
||||
if (certificates.swarmserver != null) { obj.acceleratorCertStore.push({ cert: certificates.swarmserver.cert, key: certificates.swarmserver.key }); }
|
||||
|
|
@ -588,19 +624,22 @@ module.exports.CertificateOperations = function (parent) {
|
|||
|
||||
// Perform any RSA signature, just pass in the private key and data.
|
||||
obj.acceleratorPerformSignature = function (privatekey, data, tag, func) {
|
||||
acceleratorPerformSignatureFuncCall++;
|
||||
if (acceleratorTotalCount <= 1) {
|
||||
// No accelerators available
|
||||
if (typeof privatekey == "number") { privatekey = obj.acceleratorCertStore[privatekey].key; }
|
||||
const sign = obj.crypto.createSign("SHA384");
|
||||
sign.end(Buffer.from(data, "binary"));
|
||||
func(tag, sign.sign(privatekey).toString("binary"));
|
||||
try { func(tag, sign.sign(privatekey).toString("binary")); } catch (ex) { acceleratorMessageException++; acceleratorMessageLastException = ex; }
|
||||
} else {
|
||||
var acc = obj.getAccelerator();
|
||||
if (acc == null) {
|
||||
// Add to pending accelerator workload
|
||||
acceleratorPerformSignaturePushFuncCall++;
|
||||
pendingAccelerator.push({ action: "sign", key: privatekey, data: data, tag: tag });
|
||||
} else {
|
||||
// Send to accelerator now
|
||||
acceleratorPerformSignatureRunFuncCall++;
|
||||
acc.func = func;
|
||||
acc.tag = tag;
|
||||
acc.send({ action: "sign", key: privatekey, data: data });
|
||||
|
|
|
|||
51
meshagent.js
51
meshagent.js
|
|
@ -19,6 +19,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
|
|||
const forge = parent.parent.certificateOperations.forge;
|
||||
const common = parent.parent.common;
|
||||
const agentUpdateBlockSize = 65531;
|
||||
parent.agentStats.createMeshAgentCount++;
|
||||
|
||||
var obj = {};
|
||||
obj.domain = domain;
|
||||
|
|
@ -155,6 +156,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
|
|||
if (meshcorehash == null) {
|
||||
// Clear the core
|
||||
obj.send(common.ShortToStr(10) + common.ShortToStr(0)); // MeshCommand_CoreModule, ask mesh agent to clear the core
|
||||
parent.agentStats.clearingCoreCount++;
|
||||
parent.parent.debug(1, 'Clearing core');
|
||||
} else {
|
||||
// Update new core
|
||||
|
|
@ -168,7 +170,8 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
|
|||
// Send the updated code.
|
||||
delete obj.agentCoreUpdatePending;
|
||||
obj.send(common.ShortToStr(10) + common.ShortToStr(0) + argument.hash + argument.core, function () { parent.parent.taskLimiter.completed(taskid); }); // MeshCommand_CoreModule, start core update
|
||||
parent.parent.debug(1, 'Updating code ' + argument.name);
|
||||
parent.agentStats.updatingCoreCount++;
|
||||
parent.parent.debug(1, 'Updating core ' + argument.name);
|
||||
agentCoreIsStable();
|
||||
} else {
|
||||
// This agent is probably disconnected, nothing to do.
|
||||
|
|
@ -359,6 +362,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
|
|||
} else {
|
||||
// Check that the server hash matches our own web certificate hash (SHA384)
|
||||
if ((getWebCertHash(domain) != msg.substring(2, 50)) && (getWebCertFullHash(domain) != msg.substring(2, 50))) {
|
||||
parent.agentStats.agentBadWebCertHashCount++;
|
||||
console.log('Agent bad web cert hash (Agent:' + (Buffer.from(msg.substring(2, 50), 'binary').toString('hex').substring(0, 10)) + ' != Server:' + (Buffer.from(getWebCertHash(domain), 'binary').toString('hex').substring(0, 10)) + ' or ' + (new Buffer(getWebCertFullHash(domain), 'binary').toString('hex').substring(0, 10)) + '), holding connection (' + obj.remoteaddrport + ').');
|
||||
console.log('Agent reported web cert hash:' + (Buffer.from(msg.substring(2, 50), 'binary').toString('hex')) + '.');
|
||||
return;
|
||||
|
|
@ -388,7 +392,10 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
|
|||
|
||||
// Check the agent signature if we can
|
||||
if (obj.unauthsign != null) {
|
||||
if (processAgentSignature(obj.unauthsign) == false) { console.log('Agent connected with bad signature, holding connection (' + obj.remoteaddrport + ').'); return; } else { completeAgentConnection(); }
|
||||
if (processAgentSignature(obj.unauthsign) == false) {
|
||||
parent.agentStats.agentBadSignature1Count++;
|
||||
console.log('Agent connected with bad signature, holding connection (' + obj.remoteaddrport + ').'); return;
|
||||
} else { completeAgentConnection(); }
|
||||
}
|
||||
}
|
||||
else if (cmd == 2) {
|
||||
|
|
@ -403,7 +410,12 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
|
|||
obj.unauth.nodeCertPem = '-----BEGIN CERTIFICATE-----\r\n' + Buffer.from(msg.substring(4, 4 + certlen), 'binary').toString('base64') + '\r\n-----END CERTIFICATE-----';
|
||||
|
||||
// Check the agent signature if we can
|
||||
if (obj.agentnonce == null) { obj.unauthsign = msg.substring(4 + certlen); } else { if (processAgentSignature(msg.substring(4 + certlen)) == false) { console.log('Agent connected with bad signature, holding connection (' + obj.remoteaddrport + ').'); return; } }
|
||||
if (obj.agentnonce == null) { obj.unauthsign = msg.substring(4 + certlen); } else {
|
||||
if (processAgentSignature(msg.substring(4 + certlen)) == false) {
|
||||
parent.agentStats.agentBadSignature2Count++;
|
||||
console.log('Agent connected with bad signature, holding connection (' + obj.remoteaddrport + ').'); return;
|
||||
}
|
||||
}
|
||||
completeAgentConnection();
|
||||
}
|
||||
else if (cmd == 3) {
|
||||
|
|
@ -541,7 +553,11 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
|
|||
for (var i in parent.wsagents) { if (parent.wsagents[i].domain.id == domain.id) { domainAgentSessionCount++; } }
|
||||
|
||||
// Check if we have too many user sessions
|
||||
if (domainAgentSessionCount >= domain.limits.maxagentsessions) { return; } // Too many, hold the connection.
|
||||
if (domainAgentSessionCount >= domain.limits.maxagentsessions) {
|
||||
// Too many, hold the connection.
|
||||
parent.agentStats.agentMaxSessionHoldCount++;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -593,6 +609,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
|
|||
// Check if the mesh exists
|
||||
if (mesh == null) {
|
||||
// If we disconnect, the agent will just reconnect. We need to log this or tell agent to connect in a few hours.
|
||||
parent.agentStats.invalidDomainMeshCount++;
|
||||
console.log('Agent connected with invalid domain/mesh, holding connection (' + obj.remoteaddrport + ', ' + obj.dbMeshKey + ').');
|
||||
return;
|
||||
}
|
||||
|
|
@ -600,6 +617,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
|
|||
// Check if the mesh is the right type
|
||||
if (mesh.mtype != 2) {
|
||||
// If we disconnect, the agent will just reconnect. We need to log this or tell agent to connect in a few hours.
|
||||
parent.agentStats.invalidMeshTypeCount++;
|
||||
console.log('Agent connected with invalid mesh type, holding connection (' + obj.remoteaddrport + ').');
|
||||
return;
|
||||
}
|
||||
|
|
@ -634,6 +652,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
|
|||
// Check if the mesh exists
|
||||
if (mesh == null) {
|
||||
// If we disconnect, the agent will just reconnect. We need to log this or tell agent to connect in a few hours.
|
||||
parent.agentStats.invalidDomainMesh2Count++;
|
||||
console.log('Agent connected with invalid domain/mesh, holding connection (' + obj.remoteaddrport + ', ' + obj.dbMeshKey + ').');
|
||||
return;
|
||||
}
|
||||
|
|
@ -641,6 +660,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
|
|||
// Check if the mesh is the right type
|
||||
if (mesh.mtype != 2) {
|
||||
// If we disconnect, the agent will just reconnect. We need to log this or tell agent to connect in a few hours.
|
||||
parent.agentStats.invalidMeshType2Count++;
|
||||
console.log('Agent connected with invalid mesh type, holding connection (' + obj.remoteaddrport + ').');
|
||||
return;
|
||||
}
|
||||
|
|
@ -686,6 +706,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
|
|||
parent.wsagents[obj.dbNodeKey] = obj;
|
||||
if (dupAgent) {
|
||||
// Close the duplicate agent
|
||||
parent.agentStats.duplicateAgentCount++;
|
||||
if (obj.nodeid != null) { parent.parent.debug(1, 'Duplicate agent ' + obj.nodeid + ' (' + obj.remoteaddrport + ')'); }
|
||||
dupAgent.close(3);
|
||||
} else {
|
||||
|
|
@ -769,6 +790,8 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
|
|||
}
|
||||
|
||||
function recoveryAgentCoreIsStable(mesh) {
|
||||
parent.agentStats.recoveryCoreIsStableCount++;
|
||||
|
||||
// Recovery agent is doing ok, lets perform main agent checking.
|
||||
//console.log('recoveryAgentCoreIsStable()');
|
||||
|
||||
|
|
@ -793,9 +816,12 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
|
|||
}
|
||||
|
||||
function agentCoreIsStable() {
|
||||
parent.agentStats.coreIsStableCount++;
|
||||
|
||||
// Check that the mesh exists
|
||||
const mesh = parent.meshes[obj.dbMeshKey];
|
||||
if (mesh == null) {
|
||||
parent.agentStats.meshDoesNotExistCount++;
|
||||
// TODO: Mark this agent as part of a mesh that does not exists.
|
||||
return; // Probably not worth doing anything else. Hold this agent.
|
||||
}
|
||||
|
|
@ -806,7 +832,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
|
|||
return;
|
||||
}
|
||||
|
||||
// Fetch the the real agent nodeid
|
||||
// Fetch the the diagnostic agent nodeid
|
||||
db.Get('ra' + obj.dbNodeKey, function (err, nodes) {
|
||||
if (nodes.length == 1) {
|
||||
obj.diagnosticNodeKey = nodes[0].daid;
|
||||
|
|
@ -902,7 +928,11 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
|
|||
verifier.update(buf);
|
||||
verified = verifier.verify(obj.unauth.nodeCertPem, sig, 'binary');
|
||||
}
|
||||
if (verified == false) { return false; } // Not a valid signature
|
||||
if (verified == false) {
|
||||
// Not a valid signature
|
||||
parent.agentStats.invalidPkcsSignatureCount++;
|
||||
return false;
|
||||
}
|
||||
} catch (ex) { };
|
||||
}
|
||||
}
|
||||
|
|
@ -914,7 +944,10 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
|
|||
if (verify.verify(obj.unauth.nodeCertPem, Buffer.from(msg, 'binary')) !== true) {
|
||||
const verify2 = parent.crypto.createVerify('SHA384');
|
||||
verify2.end(Buffer.from(getWebCertFullHash(domain) + obj.nonce + obj.agentnonce, 'binary')); // Test using the full cert hash
|
||||
if (verify2.verify(obj.unauth.nodeCertPem, Buffer.from(msg, 'binary')) !== true) { return false; }
|
||||
if (verify2.verify(obj.unauth.nodeCertPem, Buffer.from(msg, 'binary')) !== true) {
|
||||
parent.agentStats.invalidRsaSignatureCount++;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -927,6 +960,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
|
|||
delete obj.unauth;
|
||||
delete obj.receivedCommands;
|
||||
if (obj.unauthsign) delete obj.unauthsign;
|
||||
parent.agentStats.verifiedAgentConnectionCount++;
|
||||
parent.parent.debug(1, 'Verified agent connection to ' + obj.nodeid + ' (' + obj.remoteaddrport + ').');
|
||||
obj.authenticated = 1;
|
||||
return true;
|
||||
|
|
@ -936,7 +970,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
|
|||
function processAgentData(msg) {
|
||||
var i, str = msg.toString('utf8'), command = null;
|
||||
if (str[0] == '{') {
|
||||
try { command = JSON.parse(str); } catch (ex) { console.log('Unable to parse agent JSON (' + obj.remoteaddrport + '): ' + str, ex); return; } // If the command can't be parsed, ignore it.
|
||||
try { command = JSON.parse(str); } catch (ex) { parent.agentStats.invalidJsonCount++; console.log('Unable to parse agent JSON (' + obj.remoteaddrport + '): ' + str, ex); return; } // If the command can't be parsed, ignore it.
|
||||
if (typeof command != 'object') { return; }
|
||||
switch (command.action) {
|
||||
case 'msg':
|
||||
|
|
@ -1159,6 +1193,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
|
|||
break;
|
||||
}
|
||||
default: {
|
||||
parent.agentStats.unknownAgentActionCount++;
|
||||
console.log('Unknown agent action (' + obj.remoteaddrport + '): ' + command.action + '.');
|
||||
break;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -216,7 +216,7 @@ function CreateMeshCentralServer(config, args) {
|
|||
if (code == 0) { try { latestVer = xprocess.data.split(' ').join('').split('\r').join('').split('\n').join(''); } catch (e) { } }
|
||||
callback(obj.currentVer, latestVer);
|
||||
});
|
||||
} catch (ex) { callback(obj.currentVer, null); } // If the system is running out of memory, an exception here can easily happen.
|
||||
} catch (ex) { callback(obj.currentVer, null, ex); } // If the system is running out of memory, an exception here can easily happen.
|
||||
};
|
||||
|
||||
// Initiate server self-update
|
||||
|
|
|
|||
49
meshuser.js
49
meshuser.js
|
|
@ -520,7 +520,50 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use
|
|||
switch (cmd) {
|
||||
case 'help': {
|
||||
r = 'Available commands: help, info, versions, args, resetserver, showconfig, usersessions, tasklimiter, setmaxtasks, cores,\r\n'
|
||||
r += 'migrationagents, swarmstats, nodeconfig, heapdump, relays.';
|
||||
r += 'migrationagents, agentstats, webstats, mpsstats, swarmstats, acceleratorsstats, updatecheck, serverupdate, nodeconfig, heapdump, relays.';
|
||||
break;
|
||||
}
|
||||
case 'agentstats': {
|
||||
var stats = parent.getAgentStats();
|
||||
for (var i in stats) {
|
||||
if (typeof stats[i] == 'object') { r += (i + ': ' + JSON.stringify(stats[i]) + '\r\n'); } else { r += (i + ': ' + stats[i] + '\r\n'); }
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'webstats': {
|
||||
var stats = parent.getStats();
|
||||
for (var i in stats) {
|
||||
if (typeof stats[i] == 'object') { r += (i + ': ' + JSON.stringify(stats[i]) + '\r\n'); } else { r += (i + ': ' + stats[i] + '\r\n'); }
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'acceleratorsstats': {
|
||||
var stats = parent.parent.certificateOperations.getAcceleratorStats();
|
||||
for (var i in stats) {
|
||||
if (typeof stats[i] == 'object') { r += (i + ': ' + JSON.stringify(stats[i]) + '\r\n'); } else { r += (i + ': ' + stats[i] + '\r\n'); }
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'mpsstats': {
|
||||
var stats = parent.parent.mpsserver.getStats();
|
||||
for (var i in stats) {
|
||||
if (typeof stats[i] == 'object') { r += (i + ': ' + JSON.stringify(stats[i]) + '\r\n'); } else { r += (i + ': ' + stats[i] + '\r\n'); }
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'serverupdate': {
|
||||
r = 'Performing server update...';
|
||||
parent.parent.performServerUpdate();
|
||||
break;
|
||||
}
|
||||
case 'updatecheck': {
|
||||
parent.parent.getLatestServerVersion(function (currentVer, newVer, error) {
|
||||
var r2 = 'Current Version: ' + currentVer + '\r\n';
|
||||
if (newVer != null) { r2 += 'Available Version: ' + newVer + '\r\n'; }
|
||||
if (error != null) { r2 += 'Exception: ' + ex + '\r\n'; }
|
||||
try { ws.send(JSON.stringify({ action: 'serverconsole', value: r2, tag: command.tag })); } catch (ex) { }
|
||||
});
|
||||
r = 'Checking server update...';
|
||||
break;
|
||||
}
|
||||
case 'info': {
|
||||
|
|
@ -622,9 +665,9 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use
|
|||
} else {
|
||||
for (var i in parent.parent.swarmserver.stats) {
|
||||
if (typeof parent.parent.swarmserver.stats[i] == 'object') {
|
||||
r += i + ' ' + JSON.stringify(parent.parent.swarmserver.stats[i]) + '<br />';
|
||||
r += i + ': ' + JSON.stringify(parent.parent.swarmserver.stats[i]) + '\r\n';
|
||||
} else {
|
||||
r += i + ' ' + parent.parent.swarmserver.stats[i] + '<br />';
|
||||
r += i + ': ' + parent.parent.swarmserver.stats[i] + '\r\n';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
74
mpsserver.js
74
mpsserver.js
|
|
@ -106,7 +106,58 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
|
|||
ResourceShortage: 4,
|
||||
};
|
||||
|
||||
// Stat counters
|
||||
var connectionCount = 0;
|
||||
var userAuthRequestCount = 0;
|
||||
var incorrectPasswordCount = 0;
|
||||
var meshNotFoundCount = 0;
|
||||
var unknownTlsNodeCount = 0;
|
||||
var unknownTlsMeshIdCount = 0;
|
||||
var addedTlsDeviceCount = 0;
|
||||
var unknownNodeCount = 0;
|
||||
var unknownMeshIdCount = 0;
|
||||
var addedDeviceCount = 0;
|
||||
var ciraTimeoutCount = 0;
|
||||
var protocolVersionCount = 0;
|
||||
var badUserNameLengthCount = 0;
|
||||
var channelOpenCount = 0;
|
||||
var channelOpenConfirmCount = 0;
|
||||
var channelOpenFailCount = 0;
|
||||
var channelCloseCount = 0;
|
||||
var disconnectCommandCount = 0;
|
||||
var socketClosedCount = 0;
|
||||
var socketErrorCount = 0;
|
||||
|
||||
// Return statistics about this MPS server
|
||||
obj.getStats = function () {
|
||||
return {
|
||||
ciraConnections: Object.keys(obj.ciraConnections).length,
|
||||
tlsSessionStore: Object.keys(tlsSessionStore).length,
|
||||
connectionCount: connectionCount,
|
||||
userAuthRequestCount: userAuthRequestCount,
|
||||
incorrectPasswordCount: incorrectPasswordCount,
|
||||
meshNotFoundCount: meshNotFoundCount,
|
||||
unknownTlsNodeCount: unknownTlsNodeCount,
|
||||
unknownTlsMeshIdCount: unknownTlsMeshIdCount,
|
||||
addedTlsDeviceCount: addedTlsDeviceCount,
|
||||
unknownNodeCount: unknownNodeCount,
|
||||
unknownMeshIdCount: unknownMeshIdCount,
|
||||
addedDeviceCount: addedDeviceCount,
|
||||
ciraTimeoutCount: ciraTimeoutCount,
|
||||
protocolVersionCount: protocolVersionCount,
|
||||
badUserNameLengthCount: badUserNameLengthCount,
|
||||
channelOpenCount: channelOpenCount,
|
||||
channelOpenConfirmCount: channelOpenConfirmCount,
|
||||
channelOpenFailCount: channelOpenFailCount,
|
||||
channelCloseCount: channelCloseCount,
|
||||
disconnectCommandCount: disconnectCommandCount,
|
||||
socketClosedCount: socketClosedCount,
|
||||
socketErrorCount: socketErrorCount
|
||||
};
|
||||
}
|
||||
|
||||
function onConnection(socket) {
|
||||
connectionCount++;
|
||||
if (obj.args.mpstlsoffload) {
|
||||
socket.tag = { first: true, clientCert: null, accumulator: "", activetunnels: 0, boundPorts: [], socket: socket, host: null, nextchannelid: 4, channels: {}, nextsourceport: 0 };
|
||||
} else {
|
||||
|
|
@ -117,7 +168,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
|
|||
|
||||
// Setup the CIRA keep alive timer
|
||||
socket.setTimeout(MAX_IDLE);
|
||||
socket.on("timeout", () => { Debug(1, "MPS:CIRA timeout, disconnecting."); try { socket.end(); } catch (e) { } });
|
||||
socket.on("timeout", () => { ciraTimeoutCount++; Debug(1, "MPS:CIRA timeout, disconnecting."); try { socket.end(); } catch (e) { } });
|
||||
|
||||
socket.addListener("data", function (data) {
|
||||
if (args.mpsdebug) { var buf = Buffer.from(data, "binary"); console.log("MPS <-- (" + buf.length + "):" + buf.toString('hex')); } // Print out received bytes
|
||||
|
|
@ -156,12 +207,14 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
|
|||
obj.db.Set(device);
|
||||
|
||||
// Event the new node
|
||||
addedTlsDeviceCount++;
|
||||
var device2 = common.Clone(device);
|
||||
if (device2.intelamt.pass != null) delete device2.intelamt.pass; // Remove the Intel AMT password before eventing this.
|
||||
var change = 'CIRA added device ' + socket.tag.name + ' to mesh ' + mesh.name;
|
||||
obj.parent.DispatchEvent(['*', socket.tag.meshid], obj, { etype: 'node', action: 'addnode', node: device2, msg: change, domain: domainid });
|
||||
} else {
|
||||
// New CIRA connection for unknown node, disconnect.
|
||||
unknownTlsNodeCount++;
|
||||
console.log('CIRA connection for unknown node with incorrect group type. meshid: ' + socket.tag.meshid);
|
||||
socket.end();
|
||||
return;
|
||||
|
|
@ -177,6 +230,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
|
|||
obj.parent.SetConnectivityState(socket.tag.meshid, socket.tag.nodeid, socket.tag.connectTime, 2, 7); // TODO: Right now report power state as "present" (7) until we can poll.
|
||||
});
|
||||
} else {
|
||||
unknownTlsMeshIdCount++;
|
||||
console.log('ERROR: Intel AMT CIRA connected with unknown groupid: ' + socket.tag.meshid);
|
||||
socket.end();
|
||||
return;
|
||||
|
|
@ -219,6 +273,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
|
|||
}
|
||||
case APFProtocol.PROTOCOLVERSION: {
|
||||
if (len < 93) return 0;
|
||||
protocolVersionCount++;
|
||||
socket.tag.MajorVersion = common.ReadInt(data, 1);
|
||||
socket.tag.MinorVersion = common.ReadInt(data, 5);
|
||||
socket.tag.SystemId = guidToStr(common.rstr2hex(data.substring(13, 29))).toLowerCase();
|
||||
|
|
@ -227,6 +282,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
|
|||
}
|
||||
case APFProtocol.USERAUTH_REQUEST: {
|
||||
if (len < 13) return 0;
|
||||
userAuthRequestCount++;
|
||||
var usernameLen = common.ReadInt(data, 1);
|
||||
var username = data.substring(5, 5 + usernameLen);
|
||||
var serviceNameLen = common.ReadInt(data, 5 + usernameLen);
|
||||
|
|
@ -242,13 +298,13 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
|
|||
Debug(3, 'MPS:USERAUTH_REQUEST user=' + username + ', service=' + serviceName + ', method=' + methodName + ', password=' + password);
|
||||
|
||||
// Check the CIRA password
|
||||
if ((args.mpspass != null) && (password != args.mpspass)) { Debug(1, 'MPS:Incorrect password', username, password); SendUserAuthFail(socket); return -1; }
|
||||
if ((args.mpspass != null) && (password != args.mpspass)) { incorrectPasswordCount++; Debug(1, 'MPS:Incorrect password', username, password); SendUserAuthFail(socket); return -1; }
|
||||
|
||||
// Check the CIRA username, which should be the start of the MeshID.
|
||||
if (usernameLen != 16) { Debug(1, 'MPS:Username length not 16', username, password); SendUserAuthFail(socket); return -1; }
|
||||
if (usernameLen != 16) { badUserNameLengthCount++; Debug(1, 'MPS:Username length not 16', username, password); SendUserAuthFail(socket); return -1; }
|
||||
var meshIdStart = '/' + username, mesh = null;
|
||||
if (obj.parent.webserver.meshes) { for (var i in obj.parent.webserver.meshes) { if (obj.parent.webserver.meshes[i]._id.replace(/\@/g, 'X').replace(/\$/g, 'X').indexOf(meshIdStart) > 0) { mesh = obj.parent.webserver.meshes[i]; break; } } }
|
||||
if (mesh == null) { Debug(1, 'MPS:Mesh not found', username, password); SendUserAuthFail(socket); return -1; }
|
||||
if (mesh == null) { meshNotFoundCount++; Debug(1, 'MPS:Mesh not found', username, password); SendUserAuthFail(socket); return -1; }
|
||||
|
||||
// If this is a agent-less mesh, use the device guid 3 times as ID.
|
||||
if (mesh.mtype == 1) {
|
||||
|
|
@ -267,6 +323,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
|
|||
obj.db.Set(device);
|
||||
|
||||
// Event the new node
|
||||
addedDeviceCount++;
|
||||
var device2 = common.Clone(device);
|
||||
if (device2.intelamt.pass != null) delete device2.intelamt.pass; // Remove the Intel AMT password before eventing this.
|
||||
var change = 'CIRA added device ' + socket.tag.name + ' to group ' + mesh.name;
|
||||
|
|
@ -287,6 +344,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
|
|||
obj.db.getAmtUuidNode(mesh._id, socket.tag.SystemId, function (err, nodes) { // TODO: May need to optimize this request with indexes
|
||||
if (nodes.length !== 1) {
|
||||
// New CIRA connection for unknown node, disconnect.
|
||||
unknownNodeCount++;
|
||||
console.log('CIRA connection for unknown node. groupid: ' + mesh._id + ', uuid: ' + socket.tag.SystemId);
|
||||
socket.end();
|
||||
return;
|
||||
|
|
@ -306,6 +364,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
|
|||
});
|
||||
} else { // Unknown mesh type
|
||||
// New CIRA connection for unknown node, disconnect.
|
||||
unknownMeshIdCount++;
|
||||
console.log('CIRA connection to a unknown group type. groupid: ' + socket.tag.meshid);
|
||||
socket.end();
|
||||
return;
|
||||
|
|
@ -393,6 +452,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
|
|||
var Source = data.substring(29 + ChannelTypeLength + TargetLen, 29 + ChannelTypeLength + TargetLen + SourceLen);
|
||||
var SourcePort = common.ReadInt(data, 29 + ChannelTypeLength + TargetLen + SourceLen);
|
||||
|
||||
channelOpenCount++;
|
||||
Debug(3, 'MPS:CHANNEL_OPEN', ChannelType, SenderChannel, WindowSize, Target + ':' + TargetPort, Source + ':' + SourcePort);
|
||||
|
||||
// Check if we understand this channel type
|
||||
|
|
@ -423,6 +483,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
|
|||
if (cirachannel == null) { /*console.log("MPS Error in CHANNEL_OPEN_CONFIRMATION: Unable to find channelid " + RecipientChannel);*/ return 17; }
|
||||
cirachannel.amtchannelid = SenderChannel;
|
||||
cirachannel.sendcredits = cirachannel.amtCiraWindow = WindowSize;
|
||||
channelOpenConfirmCount++;
|
||||
Debug(3, 'MPS:CHANNEL_OPEN_CONFIRMATION', RecipientChannel, SenderChannel, WindowSize);
|
||||
if (cirachannel.closing == 1) {
|
||||
// Close this channel
|
||||
|
|
@ -454,6 +515,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
|
|||
if (len < 17) return 0;
|
||||
var RecipientChannel = common.ReadInt(data, 1);
|
||||
var ReasonCode = common.ReadInt(data, 5);
|
||||
channelOpenFailCount++;
|
||||
Debug(3, 'MPS:CHANNEL_OPEN_FAILURE', RecipientChannel, ReasonCode);
|
||||
var cirachannel = socket.tag.channels[RecipientChannel];
|
||||
if (cirachannel == null) { console.log("MPS Error in CHANNEL_OPEN_FAILURE: Unable to find channelid " + RecipientChannel); return 17; }
|
||||
|
|
@ -468,6 +530,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
|
|||
{
|
||||
if (len < 5) return 0;
|
||||
var RecipientChannel = common.ReadInt(data, 1);
|
||||
channelCloseCount++;
|
||||
Debug(3, 'MPS:CHANNEL_CLOSE', RecipientChannel);
|
||||
var cirachannel = socket.tag.channels[RecipientChannel];
|
||||
if (cirachannel == null) { console.log("MPS Error in CHANNEL_CLOSE: Unable to find channelid " + RecipientChannel); return 5; }
|
||||
|
|
@ -526,6 +589,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
|
|||
{
|
||||
if (len < 7) return 0;
|
||||
var ReasonCode = common.ReadInt(data, 1);
|
||||
disconnectCommandCount++;
|
||||
Debug(3, 'MPS:DISCONNECT', ReasonCode);
|
||||
try { delete obj.ciraConnections[socket.tag.nodeid]; } catch (e) { }
|
||||
obj.parent.ClearConnectivityState(socket.tag.meshid, socket.tag.nodeid, 2);
|
||||
|
|
@ -540,12 +604,14 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
|
|||
}
|
||||
|
||||
socket.addListener("close", function () {
|
||||
socketClosedCount++;
|
||||
Debug(1, 'MPS:CIRA connection closed');
|
||||
try { delete obj.ciraConnections[socket.tag.nodeid]; } catch (e) { }
|
||||
obj.parent.ClearConnectivityState(socket.tag.meshid, socket.tag.nodeid, 2);
|
||||
});
|
||||
|
||||
socket.addListener("error", function () {
|
||||
socketErrorCount++;
|
||||
//console.log("MPS Error: " + socket.remoteAddress);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "meshcentral",
|
||||
"version": "0.3.3-n",
|
||||
"version": "0.3.3-o",
|
||||
"keywords": [
|
||||
"Remote Management",
|
||||
"Intel AMT",
|
||||
|
|
|
|||
48
webserver.js
48
webserver.js
|
|
@ -214,6 +214,54 @@ module.exports.CreateWebServer = function (parent, db, args, certificates) {
|
|||
});
|
||||
});
|
||||
|
||||
// Return statistics about this web server
|
||||
obj.getStats = function () {
|
||||
return {
|
||||
users: Object.keys(obj.users).length,
|
||||
meshes: Object.keys(obj.meshes).length,
|
||||
dnsDomains: Object.keys(obj.dnsDomains).length,
|
||||
relaySessionCount: obj.relaySessionCount,
|
||||
relaySessionErrorCount: obj.relaySessionErrorCount,
|
||||
wsagents: Object.keys(obj.wsagents).length,
|
||||
wsagentsDisconnections: Object.keys(obj.wsagentsDisconnections).length,
|
||||
wsagentsDisconnectionsTimer: Object.keys(obj.wsagentsDisconnectionsTimer).length,
|
||||
wssessions: Object.keys(obj.wssessions).length,
|
||||
wssessions2: Object.keys(obj.wssessions2).length,
|
||||
wsPeerSessions: Object.keys(obj.wsPeerSessions).length,
|
||||
wsPeerSessions2: Object.keys(obj.wsPeerSessions2).length,
|
||||
wsPeerSessions3: Object.keys(obj.wsPeerSessions3).length,
|
||||
sessionsCount: Object.keys(obj.sessionsCount).length,
|
||||
wsrelays: Object.keys(obj.wsrelays).length,
|
||||
wsPeerRelays: Object.keys(obj.wsPeerRelays).length,
|
||||
tlsSessionStore: Object.keys(tlsSessionStore).length
|
||||
};
|
||||
}
|
||||
|
||||
// Agent counters
|
||||
obj.agentStats = {
|
||||
createMeshAgentCount: 0,
|
||||
coreIsStableCount: 0,
|
||||
verifiedAgentConnectionCount: 0,
|
||||
clearingCoreCount: 0,
|
||||
updatingCoreCount: 0,
|
||||
recoveryCoreIsStableCount: 0,
|
||||
meshDoesNotExistCount: 0,
|
||||
invalidPkcsSignatureCount: 0,
|
||||
invalidRsaSignatureCount: 0,
|
||||
invalidJsonCount: 0,
|
||||
unknownAgentActionCount: 0,
|
||||
agentBadWebCertHashCount: 0,
|
||||
agentBadSignature1Count: 0,
|
||||
agentBadSignature2Count: 0,
|
||||
agentMaxSessionHoldCount: 0,
|
||||
invalidDomainMeshCount: 0,
|
||||
invalidMeshTypeCount: 0,
|
||||
invalidDomainMesh2Count: 0,
|
||||
invalidMeshType2Count: 0,
|
||||
duplicateAgentCount: 0
|
||||
}
|
||||
obj.getAgentStats = function () { return obj.agentStats; }
|
||||
|
||||
// Authenticate the user
|
||||
obj.authenticate = function (name, pass, domain, fn) {
|
||||
if ((typeof (name) != 'string') || (typeof (pass) != 'string') || (typeof (domain) != 'object')) { fn(new Error('invalid fields')); return; }
|
||||
|
|
|
|||
Loading…
Reference in New Issue