Added server watchdog feature.

meshcentral.js

@@ -435,7 +435,7 @@ function CreateMeshCentralServer(config, args) {
             try {
                 var errlogpath = null;
                 if (typeof obj.args.mesherrorlogpath == 'string') { errlogpath = obj.path.join(obj.args.mesherrorlogpath, 'mesherrors.txt'); } else { errlogpath = obj.getConfigFilePath('mesherrors.txt'); }
-                obj.fs.appendFileSync(obj.getConfigFilePath('mesherrors.txt'), '-------- ' + new Date().toLocaleString() + ' ---- ' + getCurrentVerion() + ' --------\r\n\r\n' + data + '\r\n\r\n\r\n');
+                obj.fs.appendFileSync(errlogpath, '-------- ' + new Date().toLocaleString() + ' ---- ' + getCurrentVerion() + ' --------\r\n\r\n' + data + '\r\n\r\n\r\n');
             } catch (ex) { console.log('ERROR: Unable to write to mesherrors.txt.'); }
         });
         childProcess.on('close', function (code) { if ((code != 0) && (code != 123)) { /* console.log("Exited with code " + code); */ } });
@@ -1431,6 +1431,34 @@ function CreateMeshCentralServer(config, args) {
                         }
                     }
                 }
+
+                // Start watchdog timer if needed
+                // This is used to monitor if NodeJS is servicing IO correctly or getting held up a lot. Add this line to the settings section of config.json
+                //   "watchDog": { "interval": 100, "timeout": 150 }
+                // This will check every 100ms, if the timer is more than 150ms late, it will warn.
+                if ((typeof config.settings.watchdog == 'object') && (typeof config.settings.watchdog.interval == 'number') && (typeof config.settings.watchdog.timeout == 'number') && (config.settings.watchdog.interval >= 50) && (config.settings.watchdog.timeout >= 50)) {
+                    obj.watchdogtime = Date.now();
+                    obj.watchdogmax = 0;
+                    obj.watchdogmaxtime = null;
+                    obj.watchdogtable = [];
+                    obj.watchdog = setInterval(function () {
+                        var now = Date.now(), delta = now - obj.watchdogtime - config.settings.watchdog.interval;
+                        if (delta > obj.watchdogmax) { obj.watchdogmax = delta; obj.watchdogmaxtime = new Date().toLocaleString(); }
+                        if (delta > config.settings.watchdog.timeout) {
+                            const msg = obj.common.format("Watchdog timer timeout, {0}ms.", delta);
+                            obj.watchdogtable.push(new Date().toLocaleString() + ', ' + delta + 'ms');
+                            while (obj.watchdogtable.length > 10) { obj.watchdogtable.shift(); }
+                            obj.debug('main', msg);
+                            try {
+                                var errlogpath = null;
+                                if (typeof obj.args.mesherrorlogpath == 'string') { errlogpath = obj.path.join(obj.args.mesherrorlogpath, 'mesherrors.txt'); } else { errlogpath = obj.getConfigFilePath('mesherrors.txt'); }
+                                obj.fs.appendFileSync(errlogpath, new Date().toLocaleString() + ': ' + msg + '\r\n');
+                            } catch (ex) { console.log('ERROR: Unable to write to mesherrors.txt.'); }
+                        }
+                        obj.watchdogtime = now;
+                    }, config.settings.watchdog.interval);
+                    obj.debug('main', "Started watchdog timer.");
+                }
             });
         });
     };

meshuser.js

@@ -775,7 +775,7 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use
 
                     switch (cmd) {
                         case 'help': {
-                            var fin = '', f = '', availcommands = 'help,info,versions,args,resetserver,showconfig,usersessions,closeusersessions,tasklimiter,setmaxtasks,cores,migrationagents,agentstats,webstats,mpsstats,swarmstats,acceleratorsstats,updatecheck,serverupdate,nodeconfig,heapdump,relays,autobackup,backupconfig,dupagents,dispatchtable,badlogins,showpaths,le,lecheck,leevents,dbstats,sms,amtacm,certhashes';
+                            var fin = '', f = '', availcommands = 'help,info,versions,args,resetserver,showconfig,usersessions,closeusersessions,tasklimiter,setmaxtasks,cores,migrationagents,agentstats,webstats,mpsstats,swarmstats,acceleratorsstats,updatecheck,serverupdate,nodeconfig,heapdump,relays,autobackup,backupconfig,dupagents,dispatchtable,badlogins,showpaths,le,lecheck,leevents,dbstats,sms,amtacm,certhashes,watchdog';
                             if (parent.parent.config.settings.heapdump === true) { availcommands += ',heapdump'; }
                             availcommands = availcommands.split(',').sort();
                             while (availcommands.length > 0) { if (f.length > 80) { fin += (f + ',\r\n'); f = ''; } f += (((f != '') ? ', ' : ' ') + availcommands.shift()); }
@@ -931,6 +931,16 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use
                             }
                             break;
                         }
+                        case 'watchdog': {
+                            if (parent.parent.watchdog == null) {
+                                r = 'Server watchdog not active.';
+                            } else {
+                                r = 'Server watchdog active.\r\n';
+                                if (parent.parent.watchdogmaxtime != null) { r += 'Largest timeout was ' + parent.parent.watchdogmax + 'ms on ' + parent.parent.watchdogmaxtime + '\r\n'; }
+                                for (var i in parent.parent.watchdogtable) { r += parent.parent.watchdogtable[i] + '\r\n'; }
+                            }
+                            break;
+                        }
                         case 'acceleratorsstats': {
                             var stats = parent.parent.certificateOperations.getAcceleratorStats();
                             for (var i in stats) {
@@ -1091,23 +1101,6 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use
                             if (parent.parent.webViewsOverridePath) { r += 'XWebViews:  ' + parent.parent.webPublicOverridePath + '\r\n'; }
                             break;
                         }
-                        case 'showconfig': {
-                            // Make a copy of the configuration and hide any secrets
-                            var config = common.Clone(parent.parent.config);
-                            if (config.settings) {
-                                if (config.settings.configkey) { config.settings.configkey = '(present)'; }
-                                if (config.settings.sessionkey) { config.settings.sessionkey = '(present)'; }
-                                if (config.settings.dbencryptkey) { config.settings.dbencryptkey = '(present)'; }
-                            }
-                            if (config.domains) {
-                                for (var i in config.domains) {
-                                    if (config.domains[i].yubikey && config.domains[i].yubikey.secret) { config.domains[i].yubikey.secret = '(present)'; }
-                                }
-                            }
-
-                            r = JSON.stringify(removeAllUnderScore(config), null, 4);
-                            break;
-                        }
                         case 'migrationagents': {
                             if (parent.parent.swarmserver == null) {
                                 r = 'Swarm server not running.';

sample-config-advanced.json

@@ -96,6 +96,7 @@
       "count": 10,
       "coolofftime": 10
     },
+    "watchDog": { "interval": 100, "timeout": 400 },
     "_plugins": { "enabled": true }
   },
   "_domaindefaults": {
@@ -194,7 +195,7 @@
         "x-frame-options": "SAMEORIGIN",
         "Content-Security-Policy": "default-src 'none'; script-src 'self' 'unsafe-inline'; connect-src 'self'; img-src 'self' data:; style-src 'self' 'unsafe-inline'; frame-src 'self'; media-src 'self'"
       },
-      "_agentConfig": [ "webSocketMaskOverride=1" ],
+      "_agentConfig": [ "webSocketMaskOverride=1", "coreDumpEnabled=1" ],
       "_sessionRecording": {
         "_filepath": "C:\\temp",
         "_index": true,