MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-09 21:49:46 -05:00
Code Issues Packages Projects Releases Wiki Activity

erasure: Support cleaning up of stale multipart objects (#5250)

Browse Source 
Just like our single directory/disk setup, this PR brings
the functionality to cleanup stale multipart objects
older > 2 weeks.
This commit is contained in:
Harshavardhana
2017-11-30 18:11:42 -08:00
committed by GitHub
parent 59749a2b85
commit 490c30f853
7 changed files with 341 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
cmd/fs-v1-multipart.go
View File

@@ -30,13 +30,6 @@ import (
"github.com/minio/minio/pkg/lock"
)
const (
// Expiry duration after which the multipart uploads are deemed stale.
fsMultipartExpiry = time.Hour * 24 * 14 // 2 weeks.
// Cleanup interval when the stale multipart cleanup is initiated.
fsMultipartCleanupInterval = time.Hour * 24 // 24 hrs.
)
// Returns if the prefix is a multipart upload.
func (fs fsObjects) isMultipartUpload(bucket, prefix string) bool {
uploadsIDPath := pathJoin(fs.fsPath, bucket, prefix, uploadsJSONFile)
@@ -1042,61 +1035,3 @@ func (fs fsObjects) AbortMultipartUpload(bucket, object, uploadID string) error
return nil
}
// Removes multipart uploads if any older than `expiry` duration in a given bucket.
func (fs fsObjects) cleanupStaleMultipartUpload(bucket string, expiry time.Duration) (err error) {
var lmi ListMultipartsInfo
var st os.FileInfo
for {
// List multipart uploads in a bucket 1000 at a time
prefix := ""
lmi, err = fs.listMultipartUploadsCleanup(bucket, prefix, lmi.KeyMarker, lmi.UploadIDMarker, "", 1000)
if err != nil {
errorIf(err, "Unable to list uploads")
return err
}
// Remove uploads (and its parts) older than expiry duration.
for _, upload := range lmi.Uploads {
uploadIDPath := pathJoin(fs.fsPath, minioMetaMultipartBucket, bucket, upload.Object, upload.UploadID)
if st, err = fsStatDir(uploadIDPath); err != nil {
errorIf(err, "Failed to lookup uploads directory path %s", uploadIDPath)
continue
}
if time.Since(st.ModTime()) > expiry {
fs.AbortMultipartUpload(bucket, upload.Object, upload.UploadID)
}
}
// No more incomplete uploads remain, break and return.
if !lmi.IsTruncated {
break
}
}
return nil
}
// Removes multipart uploads if any older than `expiry` duration
// on all buckets for every `cleanupInterval`, this function is
// blocking and should be run in a go-routine.
func (fs fsObjects) cleanupStaleMultipartUploads(cleanupInterval, expiry time.Duration, doneCh chan struct{}) {
ticker := time.NewTicker(cleanupInterval)
for {
select {
case <-doneCh:
// Stop the timer.
ticker.Stop()
return
case <-ticker.C:
bucketInfos, err := fs.ListBuckets()
if err != nil {
errorIf(err, "Unable to list buckets")
continue
}
for _, bucketInfo := range bucketInfos {
fs.cleanupStaleMultipartUpload(bucketInfo.Name, expiry)
}
}
}
}