add s3 autobackup support (#6280)
Signed-off-by: si458 <simonsmith5521@gmail.com>
This commit is contained in:
Simon Smith
GitHub
parent
10b57dcf9e
commit
6da9222871
93
db.js
93
db.js
|
|
@ -3575,6 +3575,99 @@ module.exports.CreateDB = function (parent, func) {
|
|||
});
|
||||
});
|
||||
}
|
||||
|
||||
// S3 Backup
|
||||
if ((typeof parent.config.settings.autobackup == 'object') && (typeof parent.config.settings.autobackup.s3 == 'object')) {
|
||||
var s3folderName = 'MeshCentral-Backups';
|
||||
if (typeof parent.config.settings.autobackup.s3.foldername == 'string') { s3folderName = parent.config.settings.autobackup.s3.foldername; }
|
||||
// Construct the config object
|
||||
var accessKey = parent.config.settings.autobackup.s3.accesskey,
|
||||
secretKey = parent.config.settings.autobackup.s3.secretkey,
|
||||
endpoint = parent.config.settings.autobackup.s3.endpoint ? parent.config.settings.autobackup.s3.endpoint : 's3.amazonaws.com',
|
||||
port = parent.config.settings.autobackup.s3.port ? parent.config.settings.autobackup.s3.port : 443,
|
||||
useSsl = parent.config.settings.autobackup.s3.ssl ? parent.config.settings.autobackup.s3.ssl : true,
|
||||
bucketName = parent.config.settings.autobackup.s3.bucketname,
|
||||
pathPrefix = s3folderName,
|
||||
threshold = parent.config.settings.autobackup.s3.maxfiles ? parent.config.settings.autobackup.s3.maxfiles : 0,
|
||||
fileToUpload = filename;
|
||||
// Create a MinIO client
|
||||
const Minio = require('minio');
|
||||
var minioClient = new Minio.Client({
|
||||
endPoint: endpoint,
|
||||
port: port,
|
||||
useSSL: useSsl,
|
||||
accessKey: accessKey,
|
||||
secretKey: secretKey
|
||||
});
|
||||
// List objects in the specified bucket and path prefix
|
||||
var listObjectsPromise = new Promise(function(resolve, reject) {
|
||||
var items = [];
|
||||
var stream = minioClient.listObjects(bucketName, pathPrefix, true);
|
||||
stream.on('data', function(item) {
|
||||
if (!item.name.endsWith('/')) { // Exclude directories
|
||||
items.push(item);
|
||||
}
|
||||
});
|
||||
stream.on('end', function() {
|
||||
resolve(items);
|
||||
});
|
||||
stream.on('error', function(err) {
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
listObjectsPromise.then(function(objects) {
|
||||
// Count the number of files
|
||||
var fileCount = objects.length;
|
||||
// Return if no files to carry on uploading
|
||||
if (fileCount === 0) { return Promise.resolve(); }
|
||||
// Sort the files by LastModified date (oldest first)
|
||||
objects.sort(function(a, b) { return new Date(a.lastModified) - new Date(b.lastModified); });
|
||||
// Check if the threshold is zero and return if
|
||||
if (threshold === 0) { return Promise.resolve(); }
|
||||
// Check if the number of files exceeds the threshold (maxfiles) is 0
|
||||
if (fileCount >= threshold) {
|
||||
// Calculate how many files need to be deleted to make space for the new file
|
||||
var filesToDelete = fileCount - threshold + 1; // +1 to make space for the new file
|
||||
if (func) { func('Deleting ' + filesToDelete + ' older ' + (filesToDelete == 1 ? 'file' : 'files') + ' from S3 ...'); }
|
||||
// Create an array of promises for deleting files
|
||||
var deletePromises = objects.slice(0, filesToDelete).map(function(fileToDelete) {
|
||||
return new Promise(function(resolve, reject) {
|
||||
minioClient.removeObject(bucketName, fileToDelete.name, function(err) {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
if (func) { func('Deleted file: ' + fileToDelete.name + ' from S3'); }
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
// Wait for all deletions to complete
|
||||
return Promise.all(deletePromises);
|
||||
} else {
|
||||
return Promise.resolve(); // No deletion needed
|
||||
}
|
||||
}).then(function() {
|
||||
// Determine the upload path by combining the pathPrefix with the filename
|
||||
var fileName = require('path').basename(fileToUpload);
|
||||
var uploadPath = require('path').join(pathPrefix, fileName);
|
||||
// Upload a new file
|
||||
var uploadPromise = new Promise(function(resolve, reject) {
|
||||
if (func) { func('Uploading file ' + uploadPath + ' to S3'); }
|
||||
minioClient.fPutObject(bucketName, uploadPath, fileToUpload, function(err, etag) {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
if (func) { func('Uploaded file: ' + uploadPath + ' to S3'); }
|
||||
resolve(etag);
|
||||
}
|
||||
});
|
||||
});
|
||||
return uploadPromise;
|
||||
}).catch(function(error) {
|
||||
if (func) { func('Error managing files in S3: ' + error); }
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Transfer NeDB data into the current database
|
||||
|
|
|
|||
|
|
@ -897,6 +897,54 @@
|
|||
"description": "The maximum number of files to keep in the WebDAV folder, older files will be removed if needed."
|
||||
}
|
||||
}
|
||||
},
|
||||
"s3": {
|
||||
"type": "object",
|
||||
"description": "Enabled automated upload of the server backups to an S3 server.",
|
||||
"required": [
|
||||
"accessKey",
|
||||
"secretKey",
|
||||
"bucketName"
|
||||
],
|
||||
"properties": {
|
||||
"endpoint": {
|
||||
"type": "string",
|
||||
"default": "s3.amazonaws.com",
|
||||
"description": "S3 Endpoint address e.g. myS3.myserver.com"
|
||||
},
|
||||
"accessKey": {
|
||||
"type": "string",
|
||||
"description": "S3 accessKey, Required"
|
||||
},
|
||||
"secretKey": {
|
||||
"type": "string",
|
||||
"description": "S3 secretKey, Required"
|
||||
},
|
||||
"port": {
|
||||
"type": "integer",
|
||||
"default": 443,
|
||||
"description": "S3 Endpoint port number, Default is 443"
|
||||
},
|
||||
"ssl": {
|
||||
"type": "boolean",
|
||||
"default": true,
|
||||
"description": "If \"true\", the S3 Endpoint will use \"https\", else it will use \"http\", Default is true"
|
||||
},
|
||||
"bucketName": {
|
||||
"type": "string",
|
||||
"description": "S3 Bucket Name, Required"
|
||||
},
|
||||
"folderName": {
|
||||
"type": "string",
|
||||
"default": "MeshCentral-Backups",
|
||||
"description": "The name of the folder to create in the S3 Bucket. Defaults to \"MeshCentral-Backups\"."
|
||||
},
|
||||
"maxFiles": {
|
||||
"type": "integer",
|
||||
"default": 0,
|
||||
"description": "The maximum number of files to keep in the S3 folder, older files will be removed if needed. Default is 0 which is keep everything."
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -4101,6 +4101,8 @@ function mainStart() {
|
|||
if (typeof config.settings.autobackup.webdav == 'object') {
|
||||
if ((typeof config.settings.autobackup.webdav.url != 'string') || (typeof config.settings.autobackup.webdav.username != 'string') || (typeof config.settings.autobackup.webdav.password != 'string')) { addServerWarning("Missing WebDAV parameters.", 2, null, !args.launch); } else { modules.push('webdav@4.11.3'); }
|
||||
}
|
||||
// Enable S3 Support
|
||||
if (typeof config.settings.autobackup.s3 == 'object') { modules.push('minio@8.0.1'); }
|
||||
}
|
||||
|
||||
// Setup common password blocking
|
||||
|
|
|
|||
|
|
@ -132,6 +132,16 @@
|
|||
"password": "pass",
|
||||
"folderName": "MeshCentral-Backups",
|
||||
"maxFiles": 10
|
||||
},
|
||||
"_s3": {
|
||||
"accessKey": "MYLONGACCESSKEY",
|
||||
"secretKey": "MYLONGSECRETKEY",
|
||||
"endpoint": "myS3.myserver.com",
|
||||
"port": 9000,
|
||||
"ssl": false,
|
||||
"bucketName": "test",
|
||||
"folderName": "MeshCentral-Backups",
|
||||
"maxfiles": 10
|
||||
}
|
||||
},
|
||||
"_redirects": {
|
||||
|
|
|
|||
Loading…
Reference in New Issue