// Copyright (c) 2015-2023 MinIO, Inc. // // This file is part of MinIO Object Storage stack // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . package cmd import ( "context" "encoding/xml" "fmt" "slices" "strings" "sync" "testing" "time" "github.com/google/uuid" "github.com/minio/minio/internal/amztime" "github.com/minio/minio/internal/bucket/lifecycle" objectlock "github.com/minio/minio/internal/bucket/object/lock" "github.com/minio/minio/internal/bucket/replication" "github.com/minio/minio/internal/bucket/versioning" xhttp "github.com/minio/minio/internal/http" ) func TestApplyNewerNoncurrentVersionsLimit(t *testing.T) { // Prepare object layer objAPI, disks, err := prepareErasure(context.Background(), 8) if err != nil { t.Fatalf("Failed to initialize object layer: %v", err) } defer removeRoots(disks) setObjectLayer(objAPI) // Prepare bucket metadata globalBucketMetadataSys = NewBucketMetadataSys() globalBucketObjectLockSys = &BucketObjectLockSys{} globalBucketVersioningSys = &BucketVersioningSys{} lcXML := ` max-versions Enabled 2 delete-all-versions Enabled del-all true 1 true ` lc, err := lifecycle.ParseLifecycleConfig(strings.NewReader(lcXML)) if err != nil { t.Fatalf("Failed to unmarshal lifecycle config: %v", err) } vcfg := versioning.Versioning{ Status: "Enabled", } vcfgXML, err := xml.Marshal(vcfg) if err != nil { t.Fatalf("Failed to marshal versioning config: %v", err) } bucket := "bucket" now := time.Now() meta := BucketMetadata{ Name: bucket, Created: now, LifecycleConfigXML: []byte(lcXML), VersioningConfigXML: vcfgXML, VersioningConfigUpdatedAt: now, LifecycleConfigUpdatedAt: now, lifecycleConfig: lc, versioningConfig: &vcfg, } globalBucketMetadataSys.Set(bucket, meta) // Prepare lifecycle expiration workers es := newExpiryState(context.Background(), objAPI, 0) globalExpiryState = es // Prepare object versions obj := "obj-1" // Simulate objects uploaded 30 hours ago modTime := now.Add(-48 * time.Hour) uuids := make([]uuid.UUID, 5) for i := range uuids { uuids[i] = uuid.UUID([16]byte{15: uint8(i + 1)}) } fivs := make([]FileInfo, 5) objInfos := make([]ObjectInfo, 5) objRetentionMeta := make(map[string]string) objRetentionMeta[strings.ToLower(xhttp.AmzObjectLockMode)] = string(objectlock.RetCompliance) // Set retain until date 12 hours into the future objRetentionMeta[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(now.Add(12 * time.Hour)) /* objInfos: version stack for obj-1 v5 uuid-5 modTime v4 uuid-4 modTime -1m v3 uuid-3 modTime -2m v2 uuid-2 modTime -3m v1 uuid-1 modTime -4m */ for i := 0; i < 5; i++ { fivs[i] = FileInfo{ Volume: bucket, Name: obj, VersionID: uuids[i].String(), IsLatest: i == 0, ModTime: modTime.Add(-1 * time.Duration(i) * time.Minute), Size: 1 << 10, NumVersions: 5, } objInfos[i] = fivs[i].ToObjectInfo(bucket, obj, true) } /* lrObjInfos: objInfos with following modifications version stack for obj-1 v2 uuid-2 modTime -3m objRetentionMeta */ lrObjInfos := slices.Clone(objInfos) lrObjInfos[3].UserDefined = objRetentionMeta var lrWants []ObjectInfo lrWants = append(lrWants, lrObjInfos[:4]...) /* replObjInfos: objInfos with following modifications version stack for obj-1 v1 uuid-1 modTime -4m "VersionPurgeStatus: replication.VersionPurgePending" */ replObjInfos := slices.Clone(objInfos) replObjInfos[4].VersionPurgeStatus = replication.VersionPurgePending var replWants []ObjectInfo replWants = append(replWants, replObjInfos[:3]...) replWants = append(replWants, replObjInfos[4]) allVersExpObjInfos := slices.Clone(objInfos) allVersExpObjInfos[0].UserTags = "del-all=true" replCfg := replication.Config{ Rules: []replication.Rule{ { ID: "", Status: "Enabled", Priority: 1, Destination: replication.Destination{ ARN: "arn:minio:replication:::dest-bucket", Bucket: "dest-bucket", }, }, }, } lr := objectlock.Retention{ Mode: objectlock.RetCompliance, Validity: 12 * time.Hour, LockEnabled: true, } expiryWorker := func(wg *sync.WaitGroup, readyCh chan<- struct{}, taskCh <-chan expiryOp, gotExpired *[]ObjectToDelete) { defer wg.Done() // signal the calling goroutine that the worker is ready tor receive tasks close(readyCh) var expired []ObjectToDelete for t := range taskCh { switch v := t.(type) { case noncurrentVersionsTask: expired = append(expired, v.versions...) case expiryTask: expired = append(expired, ObjectToDelete{ ObjectV: ObjectV{ ObjectName: v.objInfo.Name, VersionID: v.objInfo.VersionID, }, }) } } if len(expired) > 0 { *gotExpired = expired } } tests := []struct { replCfg replicationConfig lr objectlock.Retention objInfos []ObjectInfo wants []ObjectInfo wantExpired []ObjectToDelete }{ { // With replication configured, version(s) with PENDING purge status replCfg: replicationConfig{Config: &replCfg}, objInfos: replObjInfos, wants: replWants, wantExpired: []ObjectToDelete{ {ObjectV: ObjectV{ObjectName: obj, VersionID: objInfos[3].VersionID}}, }, }, { // With lock retention configured and version(s) with retention metadata lr: lr, objInfos: lrObjInfos, wants: lrWants, wantExpired: []ObjectToDelete{ {ObjectV: ObjectV{ObjectName: obj, VersionID: objInfos[4].VersionID}}, }, }, { // With replication configured, but no versions with PENDING purge status replCfg: replicationConfig{Config: &replCfg}, objInfos: objInfos, wants: objInfos[:3], wantExpired: []ObjectToDelete{ {ObjectV: ObjectV{ObjectName: obj, VersionID: objInfos[3].VersionID}}, {ObjectV: ObjectV{ObjectName: obj, VersionID: objInfos[4].VersionID}}, }, }, { objInfos: allVersExpObjInfos, wants: nil, wantExpired: []ObjectToDelete{{ObjectV: ObjectV{ObjectName: obj, VersionID: allVersExpObjInfos[0].VersionID}}}, }, } for i, test := range tests { t.Run(fmt.Sprintf("TestApplyNewerNoncurrentVersionsLimit-%d", i), func(t *testing.T) { workers := []chan expiryOp{make(chan expiryOp)} es.workers.Store(&workers) workerReady := make(chan struct{}) var wg sync.WaitGroup wg.Add(1) var gotExpired []ObjectToDelete go expiryWorker(&wg, workerReady, workers[0], &gotExpired) <-workerReady item := scannerItem{ Path: obj, bucket: bucket, prefix: "", objectName: obj, lifeCycle: lc, replication: test.replCfg, } var ( sizeS sizeSummary gots []ObjectInfo ) item.applyActions(context.TODO(), objAPI, test.objInfos, test.lr, &sizeS, func(oi ObjectInfo, sz, _ int64, _ *sizeSummary) { if sz != 0 { gots = append(gots, oi) } }) if len(gots) != len(test.wants) { t.Fatalf("Expected %d objects but got %d", len(test.wants), len(gots)) } if slices.CompareFunc(gots, test.wants, func(g, w ObjectInfo) int { if g.VersionID == w.VersionID { return 0 } return -1 }) != 0 { t.Fatalf("Expected %v but got %v", test.wants, gots) } // verify the objects to be deleted close(workers[0]) wg.Wait() if len(gotExpired) != len(test.wantExpired) { t.Fatalf("Expected expiry of %d objects but got %d", len(test.wantExpired), len(gotExpired)) } if slices.CompareFunc(gotExpired, test.wantExpired, func(g, w ObjectToDelete) int { if g.VersionID == w.VersionID { return 0 } return -1 }) != 0 { t.Fatalf("Expected %v but got %v", test.wantExpired, gotExpired) } }) } } func TestEvalActionFromLifecycle(t *testing.T) { // Tests cover only ExpiredObjectDeleteAllVersions and DelMarkerExpiration actions numVersions := 4 obj := ObjectInfo{ Name: "foo", ModTime: time.Now().Add(-31 * 24 * time.Hour), Size: 100 << 20, VersionID: uuid.New().String(), IsLatest: true, NumVersions: numVersions, } delMarker := ObjectInfo{ Name: "foo-deleted", ModTime: time.Now().Add(-61 * 24 * time.Hour), Size: 0, VersionID: uuid.New().String(), IsLatest: true, DeleteMarker: true, NumVersions: numVersions, } deleteAllILM := ` 30 true Enabled DeleteAllVersions ` delMarkerILM := ` DelMarkerExpiration Enabled 60 ` deleteAllLc, err := lifecycle.ParseLifecycleConfig(strings.NewReader(deleteAllILM)) if err != nil { t.Fatalf("Failed to parse deleteAllILM test ILM policy %v", err) } delMarkerLc, err := lifecycle.ParseLifecycleConfig(strings.NewReader(delMarkerILM)) if err != nil { t.Fatalf("Failed to parse delMarkerILM test ILM policy %v", err) } tests := []struct { ilm lifecycle.Lifecycle retention *objectlock.Retention obj ObjectInfo want lifecycle.Action }{ { // with object locking ilm: *deleteAllLc, retention: &objectlock.Retention{LockEnabled: true}, obj: obj, want: lifecycle.NoneAction, }, { // without object locking ilm: *deleteAllLc, retention: &objectlock.Retention{}, obj: obj, want: lifecycle.DeleteAllVersionsAction, }, { // with object locking ilm: *delMarkerLc, retention: &objectlock.Retention{LockEnabled: true}, obj: delMarker, want: lifecycle.NoneAction, }, { // without object locking ilm: *delMarkerLc, retention: &objectlock.Retention{}, obj: delMarker, want: lifecycle.DelMarkerDeleteAllVersionsAction, }, } for i, test := range tests { t.Run(fmt.Sprintf("TestEvalAction-%d", i), func(t *testing.T) { gotEvent := evalActionFromLifecycle(context.Background(), test.ilm, *test.retention, nil, test.obj) if gotEvent.Action != test.want { t.Fatalf("Expected %v but got %v", test.want, gotEvent.Action) } }) } }