MyFavMirrors/MeshCentral
1
0
Fork 0
mirror of https://github.com/Ylianst/MeshCentral.git synced 2025-07-19 05:28:50 -04:00
Code Issues Packages Projects Releases Wiki Activity

use html-minifier-terser instead for translate and fix translate multi-threaded

Browse Source 
Signed-off-by: si458 <simonsmith5521@gmail.com>
This commit is contained in:
si458 2025-06-08 22:50:22 +01:00
parent 809e6e6ef8
commit 28b31b2476
3 changed files with 50 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docker/Dockerfile
View File

@ -15,7 +15,7 @@ COPY ./ /opt/meshcentral/meshcentral/
RUN if [ -n "$DISABLE_MINIFY" ] || [ -n "$DISABLE_TRANSLATE" ]; then \ RUN if [ -n "$DISABLE_MINIFY" ] || [ -n "$DISABLE_TRANSLATE" ]; then \
echo -e "----------\nPREPARING ENVIRONMENT...\n----------"; \ echo -e "----------\nPREPARING ENVIRONMENT...\n----------"; \
cd meshcentral && \ cd meshcentral && \
npm install html-minifier@4.0.0 jsdom@26.0.0 esprima@4.0.1 && \ npm install html-minifier-terser@7.2.0 jsdom@26.0.0 esprima@4.0.1 && \
cd translate && \ cd translate && \
echo -e "----------\nSTARTING THE EXTRACTING PROCESS...\n----------"; \ echo -e "----------\nSTARTING THE EXTRACTING PROCESS...\n----------"; \
node translate.js extractall && \ node translate.js extractall && \

2
meshcentral.js
View File

@ -4310,7 +4310,7 @@ function mainStart() {
else if (config.settings.xmongodb != null) { modules.push('mongojs@3.1.0'); } // Add MongoJS, old driver. else if (config.settings.xmongodb != null) { modules.push('mongojs@3.1.0'); } // Add MongoJS, old driver.
if (nodemailer || ((config.smtp != null) && (config.smtp.name != 'console')) || (config.sendmail != null)) { modules.push('nodemailer@6.9.16'); } // Add SMTP support if (nodemailer || ((config.smtp != null) && (config.smtp.name != 'console')) || (config.sendmail != null)) { modules.push('nodemailer@6.9.16'); } // Add SMTP support
if (sendgrid || (config.sendgrid != null)) { modules.push('@sendgrid/mail'); } // Add SendGrid support if (sendgrid || (config.sendgrid != null)) { modules.push('@sendgrid/mail'); } // Add SendGrid support
if ((args.translate || args.dev) && (Number(process.version.match(/^v(\d+\.\d+)/)[1]) >= 16)) { modules.push('jsdom@22.1.0'); modules.push('esprima@4.0.1'); modules.push('html-minifier@4.0.0'); } // Translation support if ((args.translate || args.dev) && (Number(process.version.match(/^v(\d+\.\d+)/)[1]) >= 16)) { modules.push('jsdom@22.1.0'); modules.push('esprima@4.0.1'); modules.push('html-minifier-terser@7.2.0'); } // Translation support
if (typeof config.settings.crowdsec == 'object') { modules.push('@crowdsec/express-bouncer@0.1.0'); } // Add CrowdSec bounser module (https://www.npmjs.com/package/@crowdsec/express-bouncer) if (typeof config.settings.crowdsec == 'object') { modules.push('@crowdsec/express-bouncer@0.1.0'); } // Add CrowdSec bounser module (https://www.npmjs.com/package/@crowdsec/express-bouncer)
if (config.settings.prometheus != null) { modules.push('prom-client'); } // Add Prometheus Metrics support if (config.settings.prometheus != null) { modules.push('prom-client'); } // Add Prometheus Metrics support

83
translate/translate.js
View File

@ -143,8 +143,8 @@ if (directRun && (NodeJSVer >= 12)) {
// Get things setup // Get things setup
jsdom = require('jsdom'); jsdom = require('jsdom');
esprima = require('esprima'); // https://www.npmjs.com/package/esprima esprima = require('esprima'); // https://www.npmjs.com/package/esprima
if (minifyLib == 1) { log("minify-js is no longer used, please switch to \"html-minifier\""); process.exit(); return; } if (minifyLib == 1) { log("minify-js is no longer used, please switch to \"html-minifier-terser\""); process.exit(); return; }
if (minifyLib == 2) { minify = require('html-minifier').minify; } // https://www.npmjs.com/package/html-minifier if (minifyLib == 2) { minify = require('html-minifier-terser').minify; } // https://www.npmjs.com/package/html-minifier
switch (op) { switch (op) {
case 'translate': { case 'translate': {
@ -161,8 +161,8 @@ if (directRun) { setup(); }
function setup() { function setup() {
var libs = ['jsdom@22.1.0', 'esprima@4.0.1']; var libs = ['jsdom@22.1.0', 'esprima@4.0.1'];
if (minifyLib == 1) { log("minify-js is no longer used, please switch to \"html-minifier\""); process.exit(); return; } if (minifyLib == 1) { log("minify-js is no longer used, please switch to \"html-minifier-terser\""); process.exit(); return; }
if (minifyLib == 2) { libs.push('html-minifier@4.0.0'); } if (minifyLib == 2) { libs.push('html-minifier-terser@7.2.0'); }
InstallModules(libs, start); InstallModules(libs, start);
} }
@ -172,8 +172,8 @@ function startEx(argv) {
// Load dependencies // Load dependencies
jsdom = require('jsdom'); jsdom = require('jsdom');
esprima = require('esprima'); // https://www.npmjs.com/package/esprima esprima = require('esprima'); // https://www.npmjs.com/package/esprima
if (minifyLib == 1) { log("minify-js is no longer used, please switch to \"html-minifier\""); process.exit(); return; } if (minifyLib == 1) { log("minify-js is no longer used, please switch to \"html-minifier-terser\""); process.exit(); return; }
if (minifyLib == 2) { minify = require('html-minifier').minify; } // https://www.npmjs.com/package/html-minifier if (minifyLib == 2) { minify = require('html-minifier-terser').minify; } // https://www.npmjs.com/package/html-minifier-terser
var command = null; var command = null;
if (argv.length > 2) { command = argv[2].toLowerCase(); } if (argv.length > 2) { command = argv[2].toLowerCase(); }
@ -366,7 +366,7 @@ function startEx(argv) {
if (sourceFile.endsWith('.handlebars') >= 0) { inFile = inFile.split('{{{pluginHandler}}}').join('"{{{pluginHandler}}}"'); } if (sourceFile.endsWith('.handlebars') >= 0) { inFile = inFile.split('{{{pluginHandler}}}').join('"{{{pluginHandler}}}"'); }
if (sourceFile.endsWith('.js')) { inFile = '<script>' + inFile + '</script>'; } if (sourceFile.endsWith('.js')) { inFile = '<script>' + inFile + '</script>'; }
var minifiedOut = minify(inFile, { minify(inFile, {
collapseBooleanAttributes: true, collapseBooleanAttributes: true,
collapseInlineTagWhitespace: false, // This is not good. collapseInlineTagWhitespace: false, // This is not good.
collapseWhitespace: true, collapseWhitespace: true,
@ -381,12 +381,12 @@ function startEx(argv) {
removeTagWhitespace: true, removeTagWhitespace: true,
preserveLineBreaks: false, preserveLineBreaks: false,
useShortDoctype: true useShortDoctype: true
}).then(function (minifiedOut) {
// Perform minification post-processing
if (sourceFile.endsWith('.js')) { minifiedOut = minifiedOut.substring(8, minifiedOut.length - 9); }
if (sourceFile.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); }
fs.writeFileSync(destinationFile, minifiedOut, { flag: 'w+' });
}); });
// Perform minification post-processing
if (sourceFile.endsWith('.js')) { minifiedOut = minifiedOut.substring(8, minifiedOut.length - 9); }
if (sourceFile.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); }
fs.writeFileSync(destinationFile, minifiedOut, { flag: 'w+' });
} }
} else if (sourceFiles[i].endsWith('.json')) { } else if (sourceFiles[i].endsWith('.json')) {
// Minify the file .json file // Minify the file .json file
@ -440,7 +440,7 @@ function startEx(argv) {
var minifiedOut = null; var minifiedOut = null;
try { try {
minifiedOut = minify(inFile, { minify(inFile, {
collapseBooleanAttributes: true, collapseBooleanAttributes: true,
collapseInlineTagWhitespace: false, // This is not good. collapseInlineTagWhitespace: false, // This is not good.
collapseWhitespace: true, collapseWhitespace: true,
@ -456,17 +456,16 @@ function startEx(argv) {
preserveLineBreaks: false, preserveLineBreaks: false,
useShortDoctype: true, useShortDoctype: true,
log: function(a) { if (typeof a !== 'string') { console.log(a); } } // Log errors from UglifyJS to console output log: function(a) { if (typeof a !== 'string') { console.log(a); } } // Log errors from UglifyJS to console output
}).then(function (minifiedOut) {
// Perform minification post-processing
if (outname.endsWith('.js')) { minifiedOut = minifiedOut.substring(8, minifiedOut.length - 9); }
if (outname.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); }
fs.writeFileSync(outnamemin, minifiedOut, { flag: 'w+' });
}); });
} catch (ex) { } catch (ex) {
console.log(ex); console.log(ex);
} }
// Perform minification post-processing
if (outname.endsWith('.js')) { minifiedOut = minifiedOut.substring(8, minifiedOut.length - 9); }
if (outname.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); }
fs.writeFileSync(outnamemin, minifiedOut, { flag: 'w+' });
/* /*
if (outname.endsWith('.js')) { if (outname.endsWith('.js')) {
var compressHandler = function compressHandlerFunc(err, buffer, outnamemin2) { var compressHandler = function compressHandlerFunc(err, buffer, outnamemin2) {
@ -509,7 +508,7 @@ function startEx(argv) {
if (outname.endsWith('.handlebars') >= 0) { inFile = inFile.split('{{{pluginHandler}}}').join('"{{{pluginHandler}}}"'); } if (outname.endsWith('.handlebars') >= 0) { inFile = inFile.split('{{{pluginHandler}}}').join('"{{{pluginHandler}}}"'); }
if (outname.endsWith('.js')) { inFile = '<script>' + inFile + '</script>'; } if (outname.endsWith('.js')) { inFile = '<script>' + inFile + '</script>'; }
var minifiedOut = minify(inFile, { minify(inFile, {
collapseBooleanAttributes: true, collapseBooleanAttributes: true,
collapseInlineTagWhitespace: false, // This is not good. collapseInlineTagWhitespace: false, // This is not good.
collapseWhitespace: true, collapseWhitespace: true,
@ -524,12 +523,12 @@ function startEx(argv) {
removeTagWhitespace: true, removeTagWhitespace: true,
preserveLineBreaks: false, preserveLineBreaks: false,
useShortDoctype: true useShortDoctype: true
}).then(function (minifiedOut) {
// Perform minification post-processing
if (outname.endsWith('.js')) { minifiedOut = minifiedOut.substring(8, minifiedOut.length - 9); }
if (outname.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); }
fs.writeFileSync(outnamemin, minifiedOut, { flag: 'w+' });
}); });
// Perform minification post-processing
if (outname.endsWith('.js')) { minifiedOut = minifiedOut.substring(8, minifiedOut.length - 9); }
if (outname.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); }
fs.writeFileSync(outnamemin, minifiedOut, { flag: 'w+' });
} }
} }
} }
@ -656,14 +655,27 @@ function translate(lang, langFile, sources, createSubDir) {
langs = {}; langs = {};
for (var i in langFileData.strings) { var entry = langFileData.strings[i]; for (var j in entry) { if ((j != 'en') && (j != 'xloc') && (j != '*')) { langs[j.toLowerCase()] = true; } } } for (var i in langFileData.strings) { var entry = langFileData.strings[i]; for (var j in entry) { if ((j != 'en') && (j != 'xloc') && (j != '*')) { langs[j.toLowerCase()] = true; } } }
for (var i in langs) { var Worker = require('worker_threads').Worker;
const { Worker } = require('worker_threads') var MAX_WORKERS = os.cpus().length; // limit to the number of CPU cores for now
const worker = new Worker('./translate.js', { stdout: true, workerData: { op: 'translate', args: [i, langFile, sources, createSubDir] } }); var activeWorkers = 0;
worker.stdout.on('data', function (msg) { console.log('wstdio:', msg.toString()); }); var taskQueue = [];
worker.on('message', function (message) { console.log(message.msg); }); function processNextTask() {
worker.on('error', function (error) { console.log('error', error); }); if (activeWorkers < MAX_WORKERS && taskQueue.length > 0) {
worker.on('exit', function (code) { /*console.log('exit', code);*/ }) var nextTask = taskQueue.shift();
activeWorkers++;
var worker = new Worker('./translate.js', { stdout: true, workerData: { op: 'translate', args: [nextTask.lang, nextTask.langFile, nextTask.sources, nextTask.createSubDir] } });
worker.stdout.on('data', function (msg) { console.log('wstdio:', msg.toString()); });
worker.on('message', function (message) { console.log(message.msg); });
worker.on('error', function (error) { console.log('error', error); activeWorkers--; processNextTask(); });
worker.on('exit', function (code) { /*console.log('exit', code);*/ activeWorkers--; processNextTask(); });
}
} }
for (var lang in langs) {
if (langs.hasOwnProperty(lang)) {
taskQueue.push({ lang: lang, langFile: langFile, sources: sources, createSubDir: createSubDir});
}
}
for (var i = 0; i < Math.min(MAX_WORKERS, taskQueue.length); i++) { processNextTask(); }
} else { } else {
// Single threaded translation // Single threaded translation
translateSingleThreaded(lang, langFile, sources, createSubDir); translateSingleThreaded(lang, langFile, sources, createSubDir);
@ -976,7 +988,7 @@ function translateFromHtml(lang, file, createSubDir) {
// Minify the file // Minify the file
if (minifyLib = 2) { if (minifyLib = 2) {
if (outnamemin.endsWith('.handlebars') >= 0) { out = out.split('{{{pluginHandler}}}').join('"{{{pluginHandler}}}"'); } if (outnamemin.endsWith('.handlebars') >= 0) { out = out.split('{{{pluginHandler}}}').join('"{{{pluginHandler}}}"'); }
var minifiedOut = minify(out, { minify(out, {
collapseBooleanAttributes: true, collapseBooleanAttributes: true,
collapseInlineTagWhitespace: false, // This is not good. collapseInlineTagWhitespace: false, // This is not good.
collapseWhitespace: true, collapseWhitespace: true,
@ -991,9 +1003,10 @@ function translateFromHtml(lang, file, createSubDir) {
removeTagWhitespace: true, removeTagWhitespace: true,
preserveLineBreaks: false, preserveLineBreaks: false,
useShortDoctype: true useShortDoctype: true
}).then(function (minifiedOut) {
if (outnamemin.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); }
fs.writeFileSync(outnamemin, minifiedOut, { flag: 'w+' });
}); });
if (outnamemin.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); }
fs.writeFileSync(outnamemin, minifiedOut, { flag: 'w+' });
} }
} }