From 28b31b2476a8a8bb2d3a433bdf3453df44d8a2d9 Mon Sep 17 00:00:00 2001 From: si458 Date: Sun, 8 Jun 2025 22:50:22 +0100 Subject: [PATCH] use html-minifier-terser instead for translate and fix translate multi-threaded Signed-off-by: si458 --- docker/Dockerfile | 2 +- meshcentral.js | 2 +- translate/translate.js | 83 ++++++++++++++++++++++++------------------ 3 files changed, 50 insertions(+), 37 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 3cdeb76e..8627dba2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,7 +15,7 @@ COPY ./ /opt/meshcentral/meshcentral/ RUN if [ -n "$DISABLE_MINIFY" ] || [ -n "$DISABLE_TRANSLATE" ]; then \ echo -e "----------\nPREPARING ENVIRONMENT...\n----------"; \ cd meshcentral && \ - npm install html-minifier@4.0.0 jsdom@26.0.0 esprima@4.0.1 && \ + npm install html-minifier-terser@7.2.0 jsdom@26.0.0 esprima@4.0.1 && \ cd translate && \ echo -e "----------\nSTARTING THE EXTRACTING PROCESS...\n----------"; \ node translate.js extractall && \ diff --git a/meshcentral.js b/meshcentral.js index 56aee5d4..dd4e215f 100644 --- a/meshcentral.js +++ b/meshcentral.js @@ -4310,7 +4310,7 @@ function mainStart() { else if (config.settings.xmongodb != null) { modules.push('mongojs@3.1.0'); } // Add MongoJS, old driver. if (nodemailer || ((config.smtp != null) && (config.smtp.name != 'console')) || (config.sendmail != null)) { modules.push('nodemailer@6.9.16'); } // Add SMTP support if (sendgrid || (config.sendgrid != null)) { modules.push('@sendgrid/mail'); } // Add SendGrid support - if ((args.translate || args.dev) && (Number(process.version.match(/^v(\d+\.\d+)/)[1]) >= 16)) { modules.push('jsdom@22.1.0'); modules.push('esprima@4.0.1'); modules.push('html-minifier@4.0.0'); } // Translation support + if ((args.translate || args.dev) && (Number(process.version.match(/^v(\d+\.\d+)/)[1]) >= 16)) { modules.push('jsdom@22.1.0'); modules.push('esprima@4.0.1'); modules.push('html-minifier-terser@7.2.0'); } // Translation support if (typeof config.settings.crowdsec == 'object') { modules.push('@crowdsec/express-bouncer@0.1.0'); } // Add CrowdSec bounser module (https://www.npmjs.com/package/@crowdsec/express-bouncer) if (config.settings.prometheus != null) { modules.push('prom-client'); } // Add Prometheus Metrics support diff --git a/translate/translate.js b/translate/translate.js index d29ed8b1..c0dbbfb9 100644 --- a/translate/translate.js +++ b/translate/translate.js @@ -143,8 +143,8 @@ if (directRun && (NodeJSVer >= 12)) { // Get things setup jsdom = require('jsdom'); esprima = require('esprima'); // https://www.npmjs.com/package/esprima - if (minifyLib == 1) { log("minify-js is no longer used, please switch to \"html-minifier\""); process.exit(); return; } - if (minifyLib == 2) { minify = require('html-minifier').minify; } // https://www.npmjs.com/package/html-minifier + if (minifyLib == 1) { log("minify-js is no longer used, please switch to \"html-minifier-terser\""); process.exit(); return; } + if (minifyLib == 2) { minify = require('html-minifier-terser').minify; } // https://www.npmjs.com/package/html-minifier switch (op) { case 'translate': { @@ -161,8 +161,8 @@ if (directRun) { setup(); } function setup() { var libs = ['jsdom@22.1.0', 'esprima@4.0.1']; - if (minifyLib == 1) { log("minify-js is no longer used, please switch to \"html-minifier\""); process.exit(); return; } - if (minifyLib == 2) { libs.push('html-minifier@4.0.0'); } + if (minifyLib == 1) { log("minify-js is no longer used, please switch to \"html-minifier-terser\""); process.exit(); return; } + if (minifyLib == 2) { libs.push('html-minifier-terser@7.2.0'); } InstallModules(libs, start); } @@ -172,8 +172,8 @@ function startEx(argv) { // Load dependencies jsdom = require('jsdom'); esprima = require('esprima'); // https://www.npmjs.com/package/esprima - if (minifyLib == 1) { log("minify-js is no longer used, please switch to \"html-minifier\""); process.exit(); return; } - if (minifyLib == 2) { minify = require('html-minifier').minify; } // https://www.npmjs.com/package/html-minifier + if (minifyLib == 1) { log("minify-js is no longer used, please switch to \"html-minifier-terser\""); process.exit(); return; } + if (minifyLib == 2) { minify = require('html-minifier-terser').minify; } // https://www.npmjs.com/package/html-minifier-terser var command = null; if (argv.length > 2) { command = argv[2].toLowerCase(); } @@ -366,7 +366,7 @@ function startEx(argv) { if (sourceFile.endsWith('.handlebars') >= 0) { inFile = inFile.split('{{{pluginHandler}}}').join('"{{{pluginHandler}}}"'); } if (sourceFile.endsWith('.js')) { inFile = ''; } - var minifiedOut = minify(inFile, { + minify(inFile, { collapseBooleanAttributes: true, collapseInlineTagWhitespace: false, // This is not good. collapseWhitespace: true, @@ -381,12 +381,12 @@ function startEx(argv) { removeTagWhitespace: true, preserveLineBreaks: false, useShortDoctype: true + }).then(function (minifiedOut) { + // Perform minification post-processing + if (sourceFile.endsWith('.js')) { minifiedOut = minifiedOut.substring(8, minifiedOut.length - 9); } + if (sourceFile.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); } + fs.writeFileSync(destinationFile, minifiedOut, { flag: 'w+' }); }); - - // Perform minification post-processing - if (sourceFile.endsWith('.js')) { minifiedOut = minifiedOut.substring(8, minifiedOut.length - 9); } - if (sourceFile.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); } - fs.writeFileSync(destinationFile, minifiedOut, { flag: 'w+' }); } } else if (sourceFiles[i].endsWith('.json')) { // Minify the file .json file @@ -440,7 +440,7 @@ function startEx(argv) { var minifiedOut = null; try { - minifiedOut = minify(inFile, { + minify(inFile, { collapseBooleanAttributes: true, collapseInlineTagWhitespace: false, // This is not good. collapseWhitespace: true, @@ -456,17 +456,16 @@ function startEx(argv) { preserveLineBreaks: false, useShortDoctype: true, log: function(a) { if (typeof a !== 'string') { console.log(a); } } // Log errors from UglifyJS to console output + }).then(function (minifiedOut) { + // Perform minification post-processing + if (outname.endsWith('.js')) { minifiedOut = minifiedOut.substring(8, minifiedOut.length - 9); } + if (outname.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); } + fs.writeFileSync(outnamemin, minifiedOut, { flag: 'w+' }); }); } catch (ex) { console.log(ex); } - // Perform minification post-processing - if (outname.endsWith('.js')) { minifiedOut = minifiedOut.substring(8, minifiedOut.length - 9); } - if (outname.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); } - - fs.writeFileSync(outnamemin, minifiedOut, { flag: 'w+' }); - /* if (outname.endsWith('.js')) { var compressHandler = function compressHandlerFunc(err, buffer, outnamemin2) { @@ -509,7 +508,7 @@ function startEx(argv) { if (outname.endsWith('.handlebars') >= 0) { inFile = inFile.split('{{{pluginHandler}}}').join('"{{{pluginHandler}}}"'); } if (outname.endsWith('.js')) { inFile = ''; } - var minifiedOut = minify(inFile, { + minify(inFile, { collapseBooleanAttributes: true, collapseInlineTagWhitespace: false, // This is not good. collapseWhitespace: true, @@ -524,12 +523,12 @@ function startEx(argv) { removeTagWhitespace: true, preserveLineBreaks: false, useShortDoctype: true + }).then(function (minifiedOut) { + // Perform minification post-processing + if (outname.endsWith('.js')) { minifiedOut = minifiedOut.substring(8, minifiedOut.length - 9); } + if (outname.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); } + fs.writeFileSync(outnamemin, minifiedOut, { flag: 'w+' }); }); - - // Perform minification post-processing - if (outname.endsWith('.js')) { minifiedOut = minifiedOut.substring(8, minifiedOut.length - 9); } - if (outname.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); } - fs.writeFileSync(outnamemin, minifiedOut, { flag: 'w+' }); } } } @@ -656,14 +655,27 @@ function translate(lang, langFile, sources, createSubDir) { langs = {}; for (var i in langFileData.strings) { var entry = langFileData.strings[i]; for (var j in entry) { if ((j != 'en') && (j != 'xloc') && (j != '*')) { langs[j.toLowerCase()] = true; } } } - for (var i in langs) { - const { Worker } = require('worker_threads') - const worker = new Worker('./translate.js', { stdout: true, workerData: { op: 'translate', args: [i, langFile, sources, createSubDir] } }); - worker.stdout.on('data', function (msg) { console.log('wstdio:', msg.toString()); }); - worker.on('message', function (message) { console.log(message.msg); }); - worker.on('error', function (error) { console.log('error', error); }); - worker.on('exit', function (code) { /*console.log('exit', code);*/ }) + var Worker = require('worker_threads').Worker; + var MAX_WORKERS = os.cpus().length; // limit to the number of CPU cores for now + var activeWorkers = 0; + var taskQueue = []; + function processNextTask() { + if (activeWorkers < MAX_WORKERS && taskQueue.length > 0) { + var nextTask = taskQueue.shift(); + activeWorkers++; + var worker = new Worker('./translate.js', { stdout: true, workerData: { op: 'translate', args: [nextTask.lang, nextTask.langFile, nextTask.sources, nextTask.createSubDir] } }); + worker.stdout.on('data', function (msg) { console.log('wstdio:', msg.toString()); }); + worker.on('message', function (message) { console.log(message.msg); }); + worker.on('error', function (error) { console.log('error', error); activeWorkers--; processNextTask(); }); + worker.on('exit', function (code) { /*console.log('exit', code);*/ activeWorkers--; processNextTask(); }); + } } + for (var lang in langs) { + if (langs.hasOwnProperty(lang)) { + taskQueue.push({ lang: lang, langFile: langFile, sources: sources, createSubDir: createSubDir}); + } + } + for (var i = 0; i < Math.min(MAX_WORKERS, taskQueue.length); i++) { processNextTask(); } } else { // Single threaded translation translateSingleThreaded(lang, langFile, sources, createSubDir); @@ -976,7 +988,7 @@ function translateFromHtml(lang, file, createSubDir) { // Minify the file if (minifyLib = 2) { if (outnamemin.endsWith('.handlebars') >= 0) { out = out.split('{{{pluginHandler}}}').join('"{{{pluginHandler}}}"'); } - var minifiedOut = minify(out, { + minify(out, { collapseBooleanAttributes: true, collapseInlineTagWhitespace: false, // This is not good. collapseWhitespace: true, @@ -991,9 +1003,10 @@ function translateFromHtml(lang, file, createSubDir) { removeTagWhitespace: true, preserveLineBreaks: false, useShortDoctype: true + }).then(function (minifiedOut) { + if (outnamemin.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); } + fs.writeFileSync(outnamemin, minifiedOut, { flag: 'w+' }); }); - if (outnamemin.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); } - fs.writeFileSync(outnamemin, minifiedOut, { flag: 'w+' }); } }