Added server data collection and charting.

db.js

@@ -30,6 +30,7 @@ module.exports.CreateDB = function (parent) {
     var Datastore = null;
     var expireEventsSeconds = (60 * 60 * 24 * 20);              // By default, expire events after 20 days. (Seconds * Minutes * Hours * Days)
     var expirePowerEventsSeconds = (60 * 60 * 24 * 10);         // By default, expire power events after 10 days. (Seconds * Minutes * Hours * Days)
+    var expireServerStatsSeconds = (60 * 60 * 24 * 30);         // By default, expire power events after 30 days. (Seconds * Minutes * Hours * Days)
     obj.path = require('path');
     obj.parent = parent;
     obj.identifier = null;
@@ -39,6 +40,7 @@ module.exports.CreateDB = function (parent) {
     if (typeof obj.parent.args.dbexpire == 'object') {
         if (typeof obj.parent.args.dbexpire.events == 'number') { expireEventsSeconds = obj.parent.args.dbexpire.events; }
         if (typeof obj.parent.args.dbexpire.powerevents == 'number') { expirePowerEventsSeconds = obj.parent.args.dbexpire.powerevents; }
+        if (typeof obj.parent.args.dbexpire.statsevents == 'number') { expireServerStatsSeconds = obj.parent.args.dbexpire.statsevents; }
     }
 
     if (obj.parent.args.mongodb) {
@@ -115,6 +117,29 @@ module.exports.CreateDB = function (parent) {
 
         // Setup MongoDB smbios collection, no indexes needed
         obj.smbiosfile = db.collection('smbios');                               // Collection containing all smbios information
+
+        // Setup MongoDB server stats collection
+        obj.serverstatsfile = db.collection('serverstats');                     // Collection of server stats
+        obj.serverstatsfile.getIndexes(function (err, indexes) {
+            // Check if we need to reset indexes
+            var indexesByName = {}, indexCount = 0;
+            for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; }
+            if ((indexCount != 2) || (indexesByName['ExpireTime1'] == null)) {
+                // Reset all indexes
+                console.log('Resetting server stats indexes...');
+                obj.serverstatsfile.dropIndexes(function (err) {
+                    // Create all indexes
+                    obj.serverstatsfile.createIndex({ "time": 1 }, { expireAfterSeconds: expireServerStatsSeconds, name: 'ExpireTime1' });
+                });
+            } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expireServerStatsSeconds) {
+                // Reset the timeout index
+                console.log('Resetting server stats expire index...');
+                obj.serverstatsfile.dropIndex("ExpireTime1", function (err) {
+                    // Reset the expire server stats index
+                    obj.serverstatsfile.createIndex({ "time": 1 }, { expireAfterSeconds: expireServerStatsSeconds, name: 'ExpireTime1' });
+                });
+            }
+        });
     } else {
         // Use NeDB (The default)
         obj.databaseType = 1;
@@ -167,6 +192,11 @@ module.exports.CreateDB = function (parent) {
 
         // Setup the SMBIOS collection
         obj.smbiosfile = new Datastore({ filename: obj.parent.getConfigFilePath('meshcentral-smbios.db'), autoload: true });
+
+        // Setup the server stats collection and setup indexes
+        obj.serverstatsfile = new Datastore({ filename: obj.parent.getConfigFilePath('meshcentral-stats.db'), autoload: true });
+        obj.serverstatsfile.persistence.setAutocompactionInterval(36000);
+        obj.serverstatsfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: 60 * 60 * 24 * 30 }); // Limit the server stats log to 30 days (Seconds * Minutes * Hours * Days)
     }
 
     obj.SetupDatabase = function (func) {
@@ -305,6 +335,10 @@ module.exports.CreateDB = function (parent) {
     obj.RemoveSMBIOS = function (id) { obj.smbiosfile.remove({ _id: id }); };
     obj.GetSMBIOS = function (id, func) { obj.smbiosfile.find({ _id: id }, func); };
 
+    // Database actions on the Server Stats collection
+    obj.SetServerStats = function (data, func) { obj.serverstatsfile.insert(data, func); };
+    obj.GetServerStats = function (hours, func) { var t = new Date(); t.setTime(t.getTime() - (60 * 60 * 1000 * hours)); obj.serverstatsfile.find({ time: { $gt: t } }, { _id: 0, cpu: 0 }, func); };
+
     // Read a configuration file from the database
     obj.getConfigFile = function (path, func) { obj.Get('cfile/' + path, func); }
 

meshcentral.js

@@ -800,6 +800,29 @@ function CreateMeshCentralServer(config, args) {
                     });
                 }
 
+                // Start collecting server stats every 5 minutes
+                setInterval(function () {
+                    obj.db.getStats(function (dbstats) {
+                        var data = {
+                            time: new Date(),
+                            mem: process.memoryUsage(),
+                            db: dbstats,
+                            cpu: process.cpuUsage(),
+                            conn: {
+                                ca: Object.keys(obj.webserver.wsagents).length,
+                                cu: Object.keys(obj.webserver.wssessions).length,
+                                us: Object.keys(obj.webserver.wssessions2).length,
+                                rs: obj.webserver.relaySessionCount
+                            }
+                        };
+                        if (obj.mpsserver != null) { data.conn.am = Object.keys(obj.mpsserver.ciraConnections).length; }
+                        obj.db.SetServerStats(data);
+                    });
+
+
+                    
+                }, 300000);
+
                 //obj.debug(1, 'Server started');
                 if (obj.args.nousers == true) { obj.updateServerState('nousers', '1'); }
                 obj.updateServerState('state', 'running');
@@ -1599,7 +1622,7 @@ function InstallModules(modules, func) {
     var missingModules = [];
     if (modules.length > 0) {
         for (var i in modules) { try { var xxmodule = require(modules[i]); } catch (e) { missingModules.push(modules[i]); } }
-        if (missingModules.length > 0) { InstallModule(missingModules.join(' '), InstallModules, modules, func); } else { func(); }
+        if (missingModules.length > 0) { InstallModule(missingModules.shift(), InstallModules, modules, func); } else { func(); }
     }
 }
 
@@ -1610,7 +1633,7 @@ function InstallModule(modulename, func, tag1, tag2) {
     var child_process = require('child_process');
 
     // Looks like we need to keep a global reference to the child process object for this to work correctly.
-    InstallModuleChildProcess = child_process.exec('npm install ' + modulename + ' --no-optional --save', { maxBuffer: 512000, timeout: 10000 }, function (error, stdout, stderr) {
+    InstallModuleChildProcess = child_process.exec('npm install --no-optional --save ' + modulename, { maxBuffer: 512000, timeout: 10000 }, function (error, stdout, stderr) {
         InstallModuleChildProcess = null;
         if (error != null) { console.log('ERROR: Unable to install missing package \'' + modulename + '\', make sure npm is installed: ' + error); process.exit(); return; }
         func(tag1, tag2);

meshuser.js

@@ -261,17 +261,25 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use
                     try { ws.send(JSON.stringify({ action: 'authcookie', cookie: parent.parent.encodeCookie({ userid: user._id, domainid: domain.id }, parent.parent.loginCookieEncryptionKey) })); } catch (ex) { }
                     break;
                 }
+            case 'servertimelinestats':
+                {
+                    if ((user.siteadmin & 21) == 0) return; // Only site administrators with "site backup" or "site restore" or "site update" permissions can use this.
+                    if (common.validateInt(command.hours, 0, 24 * 30) == false) return;
+                    db.GetServerStats(command.hours, function (err, docs) {
+                        if (err == null) { ws.send(JSON.stringify({ action: 'servertimelinestats', events: docs })); }
+                    });
+                    break;
+                }
             case 'serverstats':
                 {
-                    if ((user.siteadmin & 21) != 0) { // Only site administrators with "site backup" or "site restore" or "site update" permissions can use this.
+                    if ((user.siteadmin & 21) == 0) return; // Only site administrators with "site backup" or "site restore" or "site update" permissions can use this.
-                        if (common.validateInt(command.interval, 1000, 1000000) == false) {
+                    if (common.validateInt(command.interval, 1000, 1000000) == false) {
-                            // Clear the timer
+                        // Clear the timer
-                            if (obj.serverStatsTimer != null) { clearInterval(obj.serverStatsTimer); delete obj.serverStatsTimer; }
+                        if (obj.serverStatsTimer != null) { clearInterval(obj.serverStatsTimer); delete obj.serverStatsTimer; }
-                        } else {
+                    } else {
-                            // Set the timer
+                        // Set the timer
-                            obj.SendServerStats();
+                        obj.SendServerStats();
-                            obj.serverStatsTimer = setInterval(obj.SendServerStats, command.interval);
+                        obj.serverStatsTimer = setInterval(obj.SendServerStats, command.interval);
-                        }
                     }
                     break;
                 }

package - Copy.json

@@ -1,57 +0,0 @@
-{
-  "name": "meshcentral",
-  "version": "0.3.0-x",
-  "keywords": [
-    "Remote Management",
-    "Intel AMT",
-    "Active Management",
-    "Remote Desktop"
-  ],
-  "homepage": "http://meshcommander.com",
-  "description": "Web based remote computer management and file server",
-  "author": "Ylian Saint-Hilaire <ysainthilaire@hotmail.com>",
-  "main": "meshcentral.js",
-  "bin": {
-    "meshcentral": "./bin/meshcentral"
-  },
-  "license": "Apache-2.0",
-  "files": [
-    "*.js",
-    "sample-config.json",
-    "license.txt",
-    "readme.txt",
-    "agents",
-    "public",
-    "views",
-    "bin"
-  ],
-  "dependencies": {
-    "archiver": "^3.0.0",
-    "authdog": "^0.1.1",
-    "body-parser": "^1.18.2",
-    "compression": "^1.7.3",
-    "connect-redis": "^3.4.0",
-    "cookie-session": "^2.0.0-beta.3",
-    "express": "^4.16.4",
-    "express-handlebars": "^3.0.0",
-    "express-ws": "^4.0.0",
-    "ipcheck": "^0.1.0",
-    "meshcentral": "*",
-    "minimist": "^1.2.0",
-    "mongojs": "^2.6.0",
-    "multiparty": "^4.2.1",
-    "nedb": "^1.8.0",
-    "node-forge": "^0.7.6",
-    "otplib": "^10.0.1",
-    "ws": "^6.1.2",
-    "xmldom": "^0.1.27",
-    "yauzl": "^2.10.0",
-    "yubikeyotp": "^0.2.0"
-  },
-  "devDependencies": {},
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/Ylianst/MeshCentral.git"
-  },
-  "readme": "readme.txt"
-}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "meshcentral",
-  "version": "0.3.1-a",
+  "version": "0.3.1-c",
   "keywords": [
     "Remote Management",
     "Intel AMT",

views/default.handlebars

@@ -141,6 +141,7 @@
                                 <table id=ServerSubMenu style=width:100%;height:22px cellpadding=0 cellspacing=0 class=style1>
                                     <tr>
                                         <td id=ServerGeneral style=width:100px;height:24px;cursor:pointer class=style3x onclick=go(6)>General</td>
+                                        <td id=ServerStats style=width:100px;height:24px;cursor:pointer class=style3x onclick=go(40)>Stats</td>
                                         <td id=ServerConsole style=width:100px;height:24px;cursor:pointer class=style3x onclick=go(115)>Console</td>
                                         <td class=style3 style=height:24px>&nbsp;</td>
                                     </tr>
@@ -743,6 +744,29 @@
                     </div>
                     <div id=p31events style="max-height:calc(100vh - 267px);overflow-y:scroll"></div>
                 </div>
+                <div id=p40 style="display:none">
+                    <h1>My Server Stats</h1>
+                    <div style=width:100%;height:24px;background-color:#d3d9d6;margin-bottom:4px>
+                        <div style="float:right">
+                            <select id=p40type onchange=updateServerTimelineStats()>
+                                <option value=0>Connections</option>
+                                <option value=1>Memory</option>
+                                <option value=2>Database</option>
+                            </select>&nbsp;
+                            <select id=p40time onchange=updateServerTimelineHours()>
+                                <option value=1>Last hour</option>
+                                <option value=4>Last 3 hours</option>
+                                <option value=24>Last day</option>
+                                <option value=168>Last week</option>
+                                <option value=5040>Last 30 days</option>
+                            </select>&nbsp;
+                        </div>
+                        <div>
+                            &nbsp;<input value="Refresh" type="button" onclick="refreshServerTimelineStats()" />
+                        </div>
+                    </div>
+                    <canvas id=serverMainStats style="height:calc(100vh - 250px);width:100%"></canvas>
+                </div>
                 <br id="column_l_bottomgap" />
             </div>
             <div id=footer class=noselect>
@@ -1014,8 +1038,9 @@
             for (var c = 1; c < 27; c++) x += "<option value='" + c + "'>Ctrl-" + String.fromCharCode(64 + c) + " (" + c + ")</option>";
             QH('specialkeylist', x);
 
-            // Setup server stats panel
+            // Setup server stats panels
-            setupServerStats();
+            setupGeneralServerStats();
+            setupServerTimelineStats();
         }
 
         // Toggle the web page to full screen
@@ -1209,7 +1234,11 @@
         function onMessage(server, message) {
             switch (message.action) {
                 case 'serverstats': {
-                    updateServerStats(message);
+                    updateGeneralServerStats(message);
+                    break;
+                }
+                case 'servertimelinestats': {
+                    setServerTimelineStats(message.events);
                     break;
                 }
                 case 'authcookie': {
@@ -7169,10 +7198,10 @@
         }
 
         //
-        // Server Statistics
+        // MyServer General
         //
 
-        function setupServerStats() {
+        function setupGeneralServerStats() {
             window.serverStatCpu = new Chart(document.getElementById('serverCpuChart').getContext('2d'), {
                 type: 'doughnut',
                 data: { datasets: [{ data: [0, 0], backgroundColor: ['#AAAAAA', '#00AA00'] }], labels: ['Used', 'Free'] },
@@ -7186,7 +7215,7 @@
         }
 
         var lastServerStats = null;
-        function updateServerStats(message) {
+        function updateGeneralServerStats(message) {
             if (message != null) { lastServerStats = message; } else { message = lastServerStats; }
             if (message == null) return;
 
@@ -7220,6 +7249,91 @@
             QH('serverStatsTable', x);
         }
 
+        //
+        // MyServer Stats
+        //
+
+        var serverTimelineStats = null;
+        var serverTimelineConfig = {
+            type: 'line',
+            data: { labels: [], datasets: [{ label: '', backgroundColor: 'rgba(255, 99, 132, .5)', borderColor: 'rgb(255, 99, 132)', data: [], fill: true }] },
+            options: {
+                responsive: true,
+                scales: {
+                    xAxes: [{ type: 'time', time: { tooltipFormat: 'll HH:mm' }, display: true, scaleLabel: { display: false, labelString: '' } }],
+                    yAxes: [{ display: true, scaleLabel: { display: true, labelString: '' } }]
+                }
+            }
+        };
+
+        function refreshServerTimelineStats(stats) { meshserver.send({ action: 'servertimelinestats', hours: 24 }); }
+        function pastDate(hours) { var t = new Date(); t.setTime(t.getTime() - (60 * 60 * 1000 * hours)); return t; }
+        function setServerTimelineStats(stats) { serverTimelineStats = stats; updateServerTimelineStats(); }
+        function updateServerTimelineHours() { serverTimelineConfig.data.labels = [pastDate(0), pastDate(Q('p40time').value)]; window.serverMainStats.update(); }
+        function setupServerTimelineStats() { window.serverMainStats = new Chart(document.getElementById('serverMainStats').getContext('2d'), serverTimelineConfig); }
+
+        function updateServerTimelineStats() {
+            var data, chartType = Q('p40type').value;
+            if (chartType == 0) { // Connections
+                serverTimelineConfig.options.scales.yAxes[0].scaleLabel.labelString = 'Connection Count';
+                data = {
+                    labels: [pastDate(0), pastDate(Q('p40time').value)],
+                    datasets: [
+                        { label: 'Agents', data: [], backgroundColor: 'rgba(158, 151, 16, .1)', borderColor: 'rgb(158, 151, 16)', fill: true },
+                        { label: 'Users', data: [], backgroundColor: 'rgba(16, 84, 158, .1)', borderColor: 'rgb(16, 84, 158)', fill: true },
+                        { label: 'User Sessions', data: [], backgroundColor: 'rgba(255, 99, 132, .1)', borderColor: 'rgb(255, 99, 132)', fill: true },
+                        { label: 'Relay Sessions', data: [], backgroundColor: 'rgba(39, 158, 16, .1)', borderColor: 'rgb(39, 158, 16)', fill: true },
+                        { label: 'Intel AMT', data: [], backgroundColor: 'rgba(134, 16, 158, .1)', borderColor: 'rgb(134, 16, 158)', fill: true }
+                    ]
+                };
+                for (var i = 0; i < serverTimelineStats.length; i++) {
+                    if (serverTimelineStats[i].conn) {
+                        data.datasets[0].data.push({ x: serverTimelineStats[i].time, y: serverTimelineStats[i].conn.ca });
+                        data.datasets[1].data.push({ x: serverTimelineStats[i].time, y: serverTimelineStats[i].conn.cu });
+                        data.datasets[2].data.push({ x: serverTimelineStats[i].time, y: serverTimelineStats[i].conn.us });
+                        data.datasets[3].data.push({ x: serverTimelineStats[i].time, y: serverTimelineStats[i].conn.rs });
+                        if (serverTimelineStats[i].conn.am != null) { data.datasets[4].data.push({ x: serverTimelineStats[i].time, y: serverTimelineStats[i].conn.am }); }
+                    }
+                }
+            } else if (chartType == 1) { // Memory
+                serverTimelineConfig.options.scales.yAxes[0].scaleLabel.labelString = 'Megabytes';
+                data = {
+                    labels: [pastDate(0), pastDate(Q('p40time').value)],
+                    datasets: [
+                        { label: 'External', data: [], backgroundColor: 'rgba(158, 151, 16, .1)', borderColor: 'rgb(158, 151, 16)', fill: true },
+                        { label: 'Heap Used', data: [], backgroundColor: 'rgba(16, 84, 158, .1)', borderColor: 'rgb(16, 84, 158)', fill: true },
+                        { label: 'Heap Total', data: [], backgroundColor: 'rgba(255, 99, 132, .1)', borderColor: 'rgb(255, 99, 132)', fill: true },
+                        { label: 'RSS', data: [], backgroundColor: 'rgba(39, 158, 16, .1)', borderColor: 'rgb(39, 158, 16)', fill: true }
+                    ]
+                };
+                for (var i = 0; i < serverTimelineStats.length; i++) {
+                    data.datasets[0].data.push({ x: serverTimelineStats[i].time, y: serverTimelineStats[i].mem.external / (1024 * 1024) });
+                    data.datasets[1].data.push({ x: serverTimelineStats[i].time, y: serverTimelineStats[i].mem.heapUsed / (1024 * 1024) });
+                    data.datasets[2].data.push({ x: serverTimelineStats[i].time, y: serverTimelineStats[i].mem.heapTotal / (1024 * 1024) });
+                    data.datasets[3].data.push({ x: serverTimelineStats[i].time, y: serverTimelineStats[i].mem.rss / (1024 * 1024) });
+                }
+            } else if (chartType == 2) { // Database
+                serverTimelineConfig.options.scales.yAxes[0].scaleLabel.labelString = 'Records';
+                data = {
+                    labels: [pastDate(0), pastDate(Q('p40time').value)],
+                    datasets: [
+                        { label: 'Groups', data: [], backgroundColor: 'rgba(158, 151, 16, .1)', borderColor: 'rgb(158, 151, 16)', fill: true },
+                        { label: 'Devices', data: [], backgroundColor: 'rgba(16, 84, 158, .1)', borderColor: 'rgb(16, 84, 158)', fill: true },
+                        { label: 'Users', data: [], backgroundColor: 'rgba(255, 99, 132, .1)', borderColor: 'rgb(255, 99, 132)', fill: true },
+                        { label: 'Records', data: [], backgroundColor: 'rgba(39, 158, 16, .1)', borderColor: 'rgb(39, 158, 16)', fill: true }
+                    ]
+                };
+                for (var i = 0; i < serverTimelineStats.length; i++) {
+                    data.datasets[0].data.push({ x: serverTimelineStats[i].time, y: serverTimelineStats[i].db.meshes });
+                    data.datasets[1].data.push({ x: serverTimelineStats[i].time, y: serverTimelineStats[i].db.nodes });
+                    data.datasets[2].data.push({ x: serverTimelineStats[i].time, y: serverTimelineStats[i].db.users });
+                    data.datasets[3].data.push({ x: serverTimelineStats[i].time, y: serverTimelineStats[i].db.total });
+                }
+            }
+            serverTimelineConfig.data = data;
+            window.serverMainStats.update();
+        }
+
         //
         // POPUP DIALOG
         //
@@ -7274,7 +7388,7 @@
         function go(x) {
             if (xxdialogMode || xxcurrentView == x) return;
             // Edit this line when adding a new screen
-            for (var i = 0; i < 32; i++) { QV('p' + i, i == x); }
+            for (var i = 0; i < 41; i++) { QV('p' + i, i == x); }
             xxcurrentView = x;
 
             // Remove left bar selection
@@ -7307,7 +7421,7 @@
 
             // My Server
             QS('MainMenuMyServer').backgroundColor = (((x == 6) || (x == 115)) ? "#003366" : "#808080");
-            if (((x == 6) || (x == 115))) { Q('LeftMenuMyServer').classList.add('lbbuttonsel', 'lbbuttonsel2'); }
+            if (((x == 6) || (x == 115) || (x == 40))) { Q('LeftMenuMyServer').classList.add('lbbuttonsel', 'lbbuttonsel2'); }
 
             // column_l max-height
             if (webPageFullScreen) {
@@ -7326,8 +7440,8 @@
             QV('UserDummyMenuSpan', (x < 10) && (x != 6) && webPageFullScreen);
             QV('MeshSubMenuSpan', x >= 20 && x < 30);
             QV('UserSubMenuSpan', x >= 30 && x < 40);
-            QV('ServerSubMenuSpan', x == 6 || x == 115);
+            QV('ServerSubMenuSpan', x == 6 || x == 115 || x == 40);
-            var panels = { 10: 'MainDev', 11: 'MainDevDesktop', 12: 'MainDevTerminal', 13: 'MainDevFiles', 14: 'MainDevAmt', 15: 'MainDevConsole', 20: 'MeshGeneral', 30: 'UserGeneral', 31: 'UserEvents', 6: 'ServerGeneral', 115: 'ServerConsole' };
+            var panels = { 10: 'MainDev', 11: 'MainDevDesktop', 12: 'MainDevTerminal', 13: 'MainDevFiles', 14: 'MainDevAmt', 15: 'MainDevConsole', 20: 'MeshGeneral', 30: 'UserGeneral', 31: 'UserEvents', 6: 'ServerGeneral', 40: 'ServerStats', 115: 'ServerConsole' };
             for (var i in panels) {
                 Q(panels[i]).classList.remove('style3x');
                 Q(panels[i]).classList.remove('style3sel');
@@ -7342,6 +7456,9 @@
 
             if (x == 1) masterUpdate(4);
 
+            // Fetch the server timeline stats if needed
+            if ((x == 40) && (serverTimelineStats == null)) { refreshServerTimelineStats(); }
+
             // Update the web page title
             if ((currentNode) && (x >= 10) && (x < 20)) { document.title = 'MeshCentral - ' + currentNode.name; } else { document.title = 'MeshCentral'; }
         }