diff --git a/mint/build/versioning/bucket.go b/mint/build/versioning/bucket.go new file mode 100644 index 000000000..f2a687488 --- /dev/null +++ b/mint/build/versioning/bucket.go @@ -0,0 +1,85 @@ +/* +* +* Mint, (C) 2021 Minio, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software + +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ + +package main + +import ( + "errors" + "math/rand" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" +) + +// Tests bucket versioned bucket and get its versioning configuration to check +func testMakeBucket() { + s3Client.Config.Region = aws.String("us-east-1") + + // initialize logging params + startTime := time.Now() + function := "testCreateVersioningBucket" + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-") + args := map[string]interface{}{ + "bucketName": bucketName, + } + _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucketName), + }) + if err != nil { + failureLog(function, args, startTime, "", "Versioning CreateBucket Failed", err).Fatal() + return + } + defer cleanupBucket(bucketName, function, args, startTime) + + putVersioningInput := &s3.PutBucketVersioningInput{ + Bucket: aws.String(bucketName), + VersioningConfiguration: &s3.VersioningConfiguration{ + MFADelete: aws.String("Disabled"), + Status: aws.String("Enabled"), + }, + } + + _, err = s3Client.PutBucketVersioning(putVersioningInput) + if err != nil { + if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") { + ignoreLog(function, args, startTime, "Versioning is not implemented").Info() + return + } + failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal() + return + } + + getVersioningInput := &s3.GetBucketVersioningInput{ + Bucket: aws.String(bucketName), + } + + result, err := s3Client.GetBucketVersioning(getVersioningInput) + if err != nil { + failureLog(function, args, startTime, "", "Get Versioning failed", err).Fatal() + return + } + + if *result.Status != "Enabled" { + failureLog(function, args, startTime, "", "Get Versioning status failed", errors.New("unexpected versioning status")).Fatal() + } + + successLogger(function, args, startTime).Info() +} diff --git a/mint/build/versioning/delete.go b/mint/build/versioning/delete.go new file mode 100644 index 000000000..2fe84f40a --- /dev/null +++ b/mint/build/versioning/delete.go @@ -0,0 +1,174 @@ +/* +* +* Mint, (C) 2021 Minio, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software + +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ + +package main + +import ( + "fmt" + "io/ioutil" + "math/rand" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/s3" +) + +func testDeleteObject() { + startTime := time.Now() + function := "testDeleteObject" + bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-") + object := "testObject" + objectContent := "my object content" + expiry := 1 * time.Minute + args := map[string]interface{}{ + "bucketName": bucket, + "objectName": object, + "expiry": expiry, + } + + _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() + return + } + defer cleanupBucket(bucket, function, args, startTime) + + putVersioningInput := &s3.PutBucketVersioningInput{ + Bucket: aws.String(bucket), + VersioningConfiguration: &s3.VersioningConfiguration{ + Status: aws.String("Enabled"), + }, + } + + _, err = s3Client.PutBucketVersioning(putVersioningInput) + if err != nil { + if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") { + ignoreLog(function, args, startTime, "Versioning is not implemented").Info() + return + } + failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal() + return + } + + putInput := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader(objectContent)), + Bucket: aws.String(bucket), + Key: aws.String(object), + } + + putOutput, err := s3Client.PutObject(putInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() + return + } + + // First delete without version ID + deleteInput := &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + } + delOutput, err := s3Client.DeleteObject(deleteInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("Delete expected to succeed but got %v", err), err).Fatal() + return + } + + // Get the delete marker version, should lead to an error + getInput := &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + VersionId: aws.String(*delOutput.VersionId), + } + + result, err := s3Client.GetObject(getInput) + if err == nil { + failureLog(function, args, startTime, "", "GetObject expected to fail but succeeded", nil).Fatal() + return + } + if err != nil { + aerr, ok := err.(awserr.Error) + if !ok { + failureLog(function, args, startTime, "", "GetObject unexpected error with delete marker", err).Fatal() + return + } + if aerr.Code() != "MethodNotAllowed" { + failureLog(function, args, startTime, "", "GetObject unexpected error with delete marker", err).Fatal() + return + } + } + + // Get the older version, make sure it is preserved + getInput = &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + VersionId: aws.String(*putOutput.VersionId), + } + + result, err = s3Client.GetObject(getInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("GetObject expected to succeed but failed with %v", err), err).Fatal() + return + } + + body, err := ioutil.ReadAll(result.Body) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("GetObject expected to return data but failed with %v", err), err).Fatal() + return + } + result.Body.Close() + + if string(body) != objectContent { + failureLog(function, args, startTime, "", "GetObject unexpected body content", nil).Fatal() + return + } + + for i, versionID := range []string{*delOutput.VersionId, *putOutput.VersionId} { + delInput := &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + VersionId: aws.String(versionID), + } + _, err := s3Client.DeleteObject(delInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("DeleteObject (%d) expected to succeed but failed", i+1), err).Fatal() + return + } + } + + listInput := &s3.ListObjectVersionsInput{ + Bucket: aws.String(bucket), + } + + listOutput, err := s3Client.ListObjectVersions(listInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal() + return + } + + if len(listOutput.DeleteMarkers) != 0 || len(listOutput.CommonPrefixes) != 0 || len(listOutput.Versions) != 0 { + failureLog(function, args, startTime, "", "ListObjectVersions returned some entries but expected to return nothing", nil).Fatal() + return + } + + successLogger(function, args, startTime).Info() +} diff --git a/mint/build/versioning/get.go b/mint/build/versioning/get.go new file mode 100644 index 000000000..e930488ed --- /dev/null +++ b/mint/build/versioning/get.go @@ -0,0 +1,171 @@ +/* +* +* Mint, (C) 2021 Minio, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software + +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ + +package main + +import ( + "fmt" + "io/ioutil" + "math/rand" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/s3" +) + +// testGetObject tests all get object features - picking a particular +// version id, check content and its metadata +func testGetObject() { + startTime := time.Now() + function := "testGetObject" + bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-") + object := "testObject" + expiry := 1 * time.Minute + args := map[string]interface{}{ + "bucketName": bucket, + "objectName": object, + "expiry": expiry, + } + + _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() + return + } + defer cleanupBucket(bucket, function, args, startTime) + + putVersioningInput := &s3.PutBucketVersioningInput{ + Bucket: aws.String(bucket), + VersioningConfiguration: &s3.VersioningConfiguration{ + Status: aws.String("Enabled"), + }, + } + + _, err = s3Client.PutBucketVersioning(putVersioningInput) + if err != nil { + if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") { + ignoreLog(function, args, startTime, "Versioning is not implemented").Info() + return + } + failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal() + return + } + + putInput1 := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader("my content 1")), + Bucket: aws.String(bucket), + Key: aws.String(object), + } + _, err = s3Client.PutObject(putInput1) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() + return + } + putInput2 := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader("content file 2")), + Bucket: aws.String(bucket), + Key: aws.String(object), + } + _, err = s3Client.PutObject(putInput2) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() + return + } + + deleteInput := &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + } + + _, err = s3Client.DeleteObject(deleteInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("Delete expected to succeed but got %v", err), err).Fatal() + return + } + + input := &s3.ListObjectVersionsInput{ + Bucket: aws.String(bucket), + } + + result, err := s3Client.ListObjectVersions(input) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal() + return + } + + testCases := []struct { + content string + versionId string + deleteMarker bool + }{ + {"", *(*result.DeleteMarkers[0]).VersionId, true}, + {"content file 2", *(*result.Versions[0]).VersionId, false}, + {"my content 1", *(*result.Versions[1]).VersionId, false}, + } + + for i, testCase := range testCases { + getInput := &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + VersionId: aws.String(testCase.versionId), + } + + result, err := s3Client.GetObject(getInput) + if testCase.deleteMarker && err == nil { + failureLog(function, args, startTime, "", fmt.Sprintf("GetObject(%d) expected to fail but succeeded", i+1), nil).Fatal() + return + } + + if !testCase.deleteMarker && err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("GetObject(%d) expected to succeed but failed", i+1), err).Fatal() + return + } + + if testCase.deleteMarker { + aerr, ok := err.(awserr.Error) + if !ok { + failureLog(function, args, startTime, "", fmt.Sprintf("GetObject(%d) unexpected error with delete marker", i+1), err).Fatal() + return + } + if aerr.Code() != "MethodNotAllowed" { + failureLog(function, args, startTime, "", fmt.Sprintf("GetObject(%d) unexpected error with delete marker", i+1), err).Fatal() + return + } + continue + } + + body, err := ioutil.ReadAll(result.Body) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("GetObject(%d) expected to return data but failed", i+1), err).Fatal() + return + } + result.Body.Close() + + if string(body) != testCase.content { + failureLog(function, args, startTime, "", fmt.Sprintf("GetObject(%d) unexpected body content", i+1), err).Fatal() + return + } + } + + successLogger(function, args, startTime).Info() +} diff --git a/mint/build/versioning/go.mod b/mint/build/versioning/go.mod new file mode 100644 index 000000000..861421b4e --- /dev/null +++ b/mint/build/versioning/go.mod @@ -0,0 +1,8 @@ +module mint.minio.io/versioning/tests + +go 1.16 + +require ( + github.com/aws/aws-sdk-go v1.37.9 + github.com/sirupsen/logrus v1.7.0 +) diff --git a/mint/build/versioning/go.sum b/mint/build/versioning/go.sum new file mode 100644 index 000000000..c509c581b --- /dev/null +++ b/mint/build/versioning/go.sum @@ -0,0 +1,36 @@ +github.com/aws/aws-sdk-go v1.37.9 h1:sgRbr+heubkgSwkn9fQMF80l9xjXkmhzk9DLdsaYh+c= +github.com/aws/aws-sdk-go v1.37.9/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/mint/build/versioning/install.sh b/mint/build/versioning/install.sh new file mode 100755 index 000000000..744115201 --- /dev/null +++ b/mint/build/versioning/install.sh @@ -0,0 +1,21 @@ +#!/bin/bash -e +# +# Mint (C) 2017-2021 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +test_run_dir="$MINT_RUN_CORE_DIR/versioning" +test_build_dir="$MINT_RUN_BUILD_DIR/versioning" + +(cd "$test_build_dir" && GO111MODULE=on CGO_ENABLED=0 go build --ldflags "-s -w" -o "$test_run_dir/tests") diff --git a/mint/build/versioning/legalhold.go b/mint/build/versioning/legalhold.go new file mode 100644 index 000000000..dfa677abb --- /dev/null +++ b/mint/build/versioning/legalhold.go @@ -0,0 +1,140 @@ +/* +* +* Mint, (C) 2021 Minio, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software + +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ + +package main + +import ( + "fmt" + "math/rand" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" +) + +// Test locking for different versions +func testLockingLegalhold() { + startTime := time.Now() + function := "testLockingLegalhold" + bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-") + object := "testObject" + expiry := 1 * time.Minute + args := map[string]interface{}{ + "bucketName": bucket, + "objectName": object, + "expiry": expiry, + } + + _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + ObjectLockEnabledForBucket: aws.Bool(true), + }) + if err != nil { + if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") { + ignoreLog(function, args, startTime, "Versioning is not implemented").Info() + return + } + failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() + return + } + defer cleanupBucket(bucket, function, args, startTime) + + type uploadedObject struct { + legalhold string + successfulRemove bool + versionId string + deleteMarker bool + } + + uploads := []uploadedObject{ + {legalhold: "ON"}, + {legalhold: "OFF"}, + } + + // Upload versions and save their version IDs + for i := range uploads { + putInput := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader("content")), + Bucket: aws.String(bucket), + Key: aws.String(object), + ObjectLockLegalHoldStatus: aws.String(uploads[i].legalhold), + } + output, err := s3Client.PutObject(putInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() + return + } + uploads[i].versionId = *output.VersionId + } + + // In all cases, we can remove an object by creating a delete marker + // First delete without version ID + deleteInput := &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + } + deleteOutput, err := s3Client.DeleteObject(deleteInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("DELETE expected to succeed but got %v", err), err).Fatal() + return + } + + uploads = append(uploads, uploadedObject{versionId: *deleteOutput.VersionId, deleteMarker: true}) + + // Put tagging on each version + for i := range uploads { + if uploads[i].deleteMarker { + continue + } + deleteInput := &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + VersionId: aws.String(uploads[i].versionId), + } + _, err = s3Client.DeleteObject(deleteInput) + if err == nil && uploads[i].legalhold == "ON" { + failureLog(function, args, startTime, "", "DELETE expected to fail but succeed instead", nil).Fatal() + return + } + if err != nil && uploads[i].legalhold == "OFF" { + failureLog(function, args, startTime, "", fmt.Sprintf("DELETE expected to succeed but got %v", err), err).Fatal() + return + } + } + + for i := range uploads { + if uploads[i].deleteMarker || uploads[i].legalhold == "OFF" { + continue + } + input := &s3.PutObjectLegalHoldInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + LegalHold: &s3.ObjectLockLegalHold{Status: aws.String("OFF")}, + VersionId: aws.String(uploads[i].versionId), + } + _, err := s3Client.PutObjectLegalHold(input) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("Turning off legalhold failed with %v", err), err).Fatal() + return + } + } + + successLogger(function, args, startTime).Info() +} diff --git a/mint/build/versioning/list.go b/mint/build/versioning/list.go new file mode 100644 index 000000000..8b30cda78 --- /dev/null +++ b/mint/build/versioning/list.go @@ -0,0 +1,670 @@ +/* +* +* Mint, (C) 2021 Minio, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software + +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ + +package main + +import ( + "fmt" + "math/rand" + "reflect" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" +) + +// Test regular listing result with simple use cases: +// Upload an object ten times, delete it once (delete marker) +// and check listing result +func testListObjectVersionsSimple() { + startTime := time.Now() + function := "testListObjectVersionsSimple" + bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-") + object := "testObject" + expiry := 1 * time.Minute + args := map[string]interface{}{ + "bucketName": bucket, + "objectName": object, + "expiry": expiry, + } + + _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() + return + } + defer cleanupBucket(bucket, function, args, startTime) + + putVersioningInput := &s3.PutBucketVersioningInput{ + Bucket: aws.String(bucket), + VersioningConfiguration: &s3.VersioningConfiguration{ + Status: aws.String("Enabled"), + }, + } + + _, err = s3Client.PutBucketVersioning(putVersioningInput) + if err != nil { + if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") { + ignoreLog(function, args, startTime, "Versioning is not implemented").Info() + return + } + failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal() + return + } + + for i := 0; i < 10; i++ { + putInput1 := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader("my content 1")), + Bucket: aws.String(bucket), + Key: aws.String(object), + } + _, err = s3Client.PutObject(putInput1) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() + return + } + } + + input := &s3.ListObjectVersionsInput{ + Bucket: aws.String(bucket), + } + + deleteInput := &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + } + _, err = s3Client.DeleteObject(deleteInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("Delete expected to succeed but got %v", err), err).Fatal() + return + } + + // Accumulate all versions IDs + var versionIDs = make(map[string]struct{}) + + result, err := s3Client.ListObjectVersions(input) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal() + return + } + + // Check the delete marker entries + if len(result.DeleteMarkers) != 1 { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected result", nil).Fatal() + return + } + dm := *result.DeleteMarkers[0] + if !*dm.IsLatest { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected result", nil).Fatal() + return + } + if *dm.Key != object { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected result", nil).Fatal() + return + } + if time.Since(*dm.LastModified) > time.Hour { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected result", nil).Fatal() + return + } + if *dm.VersionId == "" { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected result", nil).Fatal() + return + } + versionIDs[*dm.VersionId] = struct{}{} + + // Check versions entries + if len(result.Versions) != 10 { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected result", nil).Fatal() + return + } + + for _, version := range result.Versions { + v := *version + if *v.IsLatest { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected IsLatest field", nil).Fatal() + return + } + if *v.Key != object { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected Key field", nil).Fatal() + return + } + if time.Since(*v.LastModified) > time.Hour { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected LastModified field", nil).Fatal() + return + } + if *v.VersionId == "" { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected VersionId field", nil).Fatal() + return + } + if *v.ETag != "\"094459df8fcebffc70d9aa08d75f9944\"" { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected ETag field", nil).Fatal() + return + } + if *v.Size != 12 { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected Size field", nil).Fatal() + return + } + if *v.StorageClass != "STANDARD" { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected StorageClass field", nil).Fatal() + return + } + + versionIDs[*v.VersionId] = struct{}{} + } + + // Ensure that we have 11 distinct versions IDs + if len(versionIDs) != 11 { + failureLog(function, args, startTime, "", "ListObjectVersions didn't return 11 different version IDs", nil).Fatal() + return + } + + successLogger(function, args, startTime).Info() +} + +func testListObjectVersionsWithPrefixAndDelimiter() { + startTime := time.Now() + function := "testListObjectVersionsWithPrefixAndDelimiter" + bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-") + object := "testObject" + expiry := 1 * time.Minute + args := map[string]interface{}{ + "bucketName": bucket, + "objectName": object, + "expiry": expiry, + } + + _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() + return + } + defer cleanupBucket(bucket, function, args, startTime) + + putVersioningInput := &s3.PutBucketVersioningInput{ + Bucket: aws.String(bucket), + VersioningConfiguration: &s3.VersioningConfiguration{ + Status: aws.String("Enabled"), + }, + } + + _, err = s3Client.PutBucketVersioning(putVersioningInput) + if err != nil { + if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") { + ignoreLog(function, args, startTime, "Versioning is not implemented").Info() + return + } + failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal() + return + } + + for _, objectName := range []string{"dir/object", "dir/dir/object", "object"} { + putInput := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader("my content 1")), + Bucket: aws.String(bucket), + Key: aws.String(objectName), + } + _, err = s3Client.PutObject(putInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() + return + } + } + + type objectResult struct { + name string + isLatest bool + } + type listResult struct { + versions []objectResult + commonPrefixes []string + } + + simplifyListingResult := func(out *s3.ListObjectVersionsOutput) (result listResult) { + for _, commonPrefix := range out.CommonPrefixes { + result.commonPrefixes = append(result.commonPrefixes, *commonPrefix.Prefix) + } + for _, version := range out.Versions { + result.versions = append(result.versions, objectResult{name: *version.Key, isLatest: *version.IsLatest}) + } + return + } + + // Recursive listing + input := &s3.ListObjectVersionsInput{ + Bucket: aws.String(bucket), + } + result, err := s3Client.ListObjectVersions(input) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal() + return + } + gotResult := simplifyListingResult(result) + expectedResult := listResult{ + versions: []objectResult{ + objectResult{name: "dir/dir/object", isLatest: true}, + objectResult{name: "dir/object", isLatest: true}, + objectResult{name: "object", isLatest: true}, + }} + if !reflect.DeepEqual(gotResult, expectedResult) { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected listing result", nil).Fatal() + return + } + + // Listing with delimiter + input = &s3.ListObjectVersionsInput{ + Bucket: aws.String(bucket), + Delimiter: aws.String("/"), + } + result, err = s3Client.ListObjectVersions(input) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal() + return + } + gotResult = simplifyListingResult(result) + expectedResult = listResult{ + versions: []objectResult{ + objectResult{name: "object", isLatest: true}, + }, + commonPrefixes: []string{"dir/"}} + if !reflect.DeepEqual(gotResult, expectedResult) { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected listing result", nil).Fatal() + return + } + + // Listing with prefix and delimiter + input = &s3.ListObjectVersionsInput{ + Bucket: aws.String(bucket), + Delimiter: aws.String("/"), + Prefix: aws.String("dir/"), + } + result, err = s3Client.ListObjectVersions(input) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal() + return + } + gotResult = simplifyListingResult(result) + expectedResult = listResult{ + versions: []objectResult{ + objectResult{name: "dir/object", isLatest: true}, + }, + commonPrefixes: []string{"dir/dir/"}} + if !reflect.DeepEqual(gotResult, expectedResult) { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected listing result", nil).Fatal() + return + } + + successLogger(function, args, startTime).Info() +} + +// Test if key marker continuation works in listing works well +func testListObjectVersionsKeysContinuation() { + startTime := time.Now() + function := "testListObjectKeysContinuation" + bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-") + object := "testObject" + expiry := 1 * time.Minute + args := map[string]interface{}{ + "bucketName": bucket, + "objectName": object, + "expiry": expiry, + } + + _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() + return + } + defer cleanupBucket(bucket, function, args, startTime) + + putVersioningInput := &s3.PutBucketVersioningInput{ + Bucket: aws.String(bucket), + VersioningConfiguration: &s3.VersioningConfiguration{ + Status: aws.String("Enabled"), + }, + } + + _, err = s3Client.PutBucketVersioning(putVersioningInput) + if err != nil { + if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") { + ignoreLog(function, args, startTime, "Versioning is not implemented").Info() + return + } + failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal() + return + } + + for i := 0; i < 10; i++ { + putInput1 := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader("my content 1")), + Bucket: aws.String(bucket), + Key: aws.String(fmt.Sprintf("testobject-%d", i)), + } + _, err = s3Client.PutObject(putInput1) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() + return + } + } + + input := &s3.ListObjectVersionsInput{ + Bucket: aws.String(bucket), + MaxKeys: aws.Int64(5), + } + + type resultPage struct { + versions []string + nextKeyMarker string + lastPage bool + } + + var gotResult []resultPage + var numPages int + + err = s3Client.ListObjectVersionsPages(input, + func(page *s3.ListObjectVersionsOutput, lastPage bool) bool { + numPages++ + resultPage := resultPage{lastPage: lastPage} + if page.NextKeyMarker != nil { + resultPage.nextKeyMarker = *page.NextKeyMarker + } + for _, v := range page.Versions { + resultPage.versions = append(resultPage.versions, *v.Key) + } + gotResult = append(gotResult, resultPage) + return true + }) + + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal() + return + } + + if numPages != 2 { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected number of pages", nil).Fatal() + return + } + + expectedResult := []resultPage{ + resultPage{versions: []string{"testobject-0", "testobject-1", "testobject-2", "testobject-3", "testobject-4"}, nextKeyMarker: "testobject-4", lastPage: false}, + resultPage{versions: []string{"testobject-5", "testobject-6", "testobject-7", "testobject-8", "testobject-9"}, nextKeyMarker: "", lastPage: true}, + } + + if !reflect.DeepEqual(expectedResult, gotResult) { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected listing result", nil).Fatal() + return + } + + successLogger(function, args, startTime).Info() +} + +// Test if version id marker continuation works in listing works well +func testListObjectVersionsVersionIDContinuation() { + startTime := time.Now() + function := "testListObjectVersionIDContinuation" + bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-") + object := "testObject" + expiry := 1 * time.Minute + args := map[string]interface{}{ + "bucketName": bucket, + "objectName": object, + "expiry": expiry, + } + + _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() + return + } + defer cleanupBucket(bucket, function, args, startTime) + + putVersioningInput := &s3.PutBucketVersioningInput{ + Bucket: aws.String(bucket), + VersioningConfiguration: &s3.VersioningConfiguration{ + Status: aws.String("Enabled"), + }, + } + + _, err = s3Client.PutBucketVersioning(putVersioningInput) + if err != nil { + if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") { + ignoreLog(function, args, startTime, "Versioning is not implemented").Info() + return + } + failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal() + return + } + + for i := 0; i < 10; i++ { + putInput1 := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader("my content 1")), + Bucket: aws.String(bucket), + Key: aws.String("testobject"), + } + _, err = s3Client.PutObject(putInput1) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() + return + } + } + + input := &s3.ListObjectVersionsInput{ + Bucket: aws.String(bucket), + MaxKeys: aws.Int64(5), + } + + type resultPage struct { + versions []string + nextVersionIDMarker string + lastPage bool + } + + var gotResult []resultPage + var gotNextVersionIDMarker string + var numPages int + + err = s3Client.ListObjectVersionsPages(input, + func(page *s3.ListObjectVersionsOutput, lastPage bool) bool { + numPages++ + resultPage := resultPage{lastPage: lastPage} + if page.NextVersionIdMarker != nil { + resultPage.nextVersionIDMarker = *page.NextVersionIdMarker + } + for _, v := range page.Versions { + resultPage.versions = append(resultPage.versions, *v.Key) + } + if !lastPage { + // There is only two pages, so here we are saving the version id + // of the last element in the first page of listing + gotNextVersionIDMarker = *(*page.Versions[len(page.Versions)-1]).VersionId + } + gotResult = append(gotResult, resultPage) + return true + }) + + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal() + return + } + + if numPages != 2 { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected number of pages", nil).Fatal() + return + } + + expectedResult := []resultPage{ + resultPage{versions: []string{"testobject", "testobject", "testobject", "testobject", "testobject"}, nextVersionIDMarker: gotNextVersionIDMarker, lastPage: false}, + resultPage{versions: []string{"testobject", "testobject", "testobject", "testobject", "testobject"}, lastPage: true}, + } + + if !reflect.DeepEqual(expectedResult, gotResult) { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected listing result", nil).Fatal() + return + } + + successLogger(function, args, startTime).Info() +} + +// Test listing object when there is some empty directory object +func testListObjectsVersionsWithEmptyDirObject() { + startTime := time.Now() + function := "testListObjectsVersionsWithEmptyDirObject" + bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-") + object := "testObject" + expiry := 1 * time.Minute + args := map[string]interface{}{ + "bucketName": bucket, + "objectName": object, + "expiry": expiry, + } + + _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() + return + } + defer cleanupBucket(bucket, function, args, startTime) + + putVersioningInput := &s3.PutBucketVersioningInput{ + Bucket: aws.String(bucket), + VersioningConfiguration: &s3.VersioningConfiguration{ + Status: aws.String("Enabled"), + }, + } + + _, err = s3Client.PutBucketVersioning(putVersioningInput) + if err != nil { + if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") { + ignoreLog(function, args, startTime, "Versioning is not implemented").Info() + return + } + failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal() + return + } + + for _, objectName := range []string{"dir/object", "dir/"} { + putInput := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader("")), + Bucket: aws.String(bucket), + Key: aws.String(objectName), + } + _, err = s3Client.PutObject(putInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() + return + } + } + + type objectResult struct { + name string + etag string + isLatest bool + } + type listResult struct { + versions []objectResult + commonPrefixes []string + } + + simplifyListingResult := func(out *s3.ListObjectVersionsOutput) (result listResult) { + for _, commonPrefix := range out.CommonPrefixes { + result.commonPrefixes = append(result.commonPrefixes, *commonPrefix.Prefix) + } + for _, version := range out.Versions { + result.versions = append(result.versions, objectResult{ + name: *version.Key, + etag: *version.ETag, + isLatest: *version.IsLatest, + }) + } + return + } + + // Recursive listing + input := &s3.ListObjectVersionsInput{ + Bucket: aws.String(bucket), + } + result, err := s3Client.ListObjectVersions(input) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal() + return + } + gotResult := simplifyListingResult(result) + expectedResult := listResult{ + versions: []objectResult{ + objectResult{name: "dir/", etag: "\"d41d8cd98f00b204e9800998ecf8427e\"", isLatest: true}, + objectResult{name: "dir/object", etag: "\"d41d8cd98f00b204e9800998ecf8427e\"", isLatest: true}, + }} + if !reflect.DeepEqual(gotResult, expectedResult) { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected listing result", nil).Fatal() + return + } + + // Listing with delimiter + input = &s3.ListObjectVersionsInput{ + Bucket: aws.String(bucket), + Delimiter: aws.String("/"), + } + result, err = s3Client.ListObjectVersions(input) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal() + return + } + gotResult = simplifyListingResult(result) + expectedResult = listResult{ + commonPrefixes: []string{"dir/"}} + if !reflect.DeepEqual(gotResult, expectedResult) { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected listing result", nil).Fatal() + return + } + + // Listing with prefix and delimiter + input = &s3.ListObjectVersionsInput{ + Bucket: aws.String(bucket), + Delimiter: aws.String("/"), + Prefix: aws.String("dir/"), + } + result, err = s3Client.ListObjectVersions(input) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal() + return + } + gotResult = simplifyListingResult(result) + expectedResult = listResult{ + versions: []objectResult{ + {name: "dir/", etag: "\"d41d8cd98f00b204e9800998ecf8427e\"", isLatest: true}, + {name: "dir/object", etag: "\"d41d8cd98f00b204e9800998ecf8427e\"", isLatest: true}, + }, + } + if !reflect.DeepEqual(gotResult, expectedResult) { + failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected listing result", nil).Fatal() + return + } + + successLogger(function, args, startTime).Info() +} diff --git a/mint/build/versioning/main.go b/mint/build/versioning/main.go new file mode 100644 index 000000000..aaf9fe76b --- /dev/null +++ b/mint/build/versioning/main.go @@ -0,0 +1,122 @@ +// Mint, (C) 2021 Minio, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + log "github.com/sirupsen/logrus" +) + +// S3 client for testing +var s3Client *s3.S3 + +func cleanupBucket(bucket string, function string, args map[string]interface{}, startTime time.Time) { + input := &s3.ListObjectVersionsInput{ + Bucket: aws.String(bucket), + } + + err := s3Client.ListObjectVersionsPages(input, + func(page *s3.ListObjectVersionsOutput, lastPage bool) bool { + for _, v := range page.Versions { + input := &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: v.Key, + VersionId: v.VersionId, + BypassGovernanceRetention: aws.Bool(true), + } + _, err := s3Client.DeleteObject(input) + if err != nil { + log.Fatalln("cleanupBucket:", err) + return true + } + } + for _, v := range page.DeleteMarkers { + input := &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: v.Key, + VersionId: v.VersionId, + } + _, err := s3Client.DeleteObject(input) + if err != nil { + log.Fatalln("cleanupBucket:", err) + return true + } + } + return true + }) + + _, err = s3Client.DeleteBucket(&s3.DeleteBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + failureLog(function, args, startTime, "", "Cleanup bucket Failed", err).Fatal() + return + } +} + +func main() { + endpoint := os.Getenv("SERVER_ENDPOINT") + accessKey := os.Getenv("ACCESS_KEY") + secretKey := os.Getenv("SECRET_KEY") + secure := os.Getenv("ENABLE_HTTPS") + sdkEndpoint := "http://" + endpoint + if secure == "1" { + sdkEndpoint = "https://" + endpoint + } + + creds := credentials.NewStaticCredentials(accessKey, secretKey, "") + newSession := session.New() + s3Config := &aws.Config{ + Credentials: creds, + Endpoint: aws.String(sdkEndpoint), + Region: aws.String("us-east-1"), + S3ForcePathStyle: aws.Bool(true), + } + + // Create an S3 service object in the default region. + s3Client = s3.New(newSession, s3Config) + + // Output to stdout instead of the default stderr + log.SetOutput(os.Stdout) + // create custom formatter + mintFormatter := mintJSONFormatter{} + // set custom formatter + log.SetFormatter(&mintFormatter) + // log Info or above -- success cases are Info level, failures are Fatal level + log.SetLevel(log.InfoLevel) + + testMakeBucket() + testPutObject() + testPutObjectWithTaggingAndMetadata() + testGetObject() + testStatObject() + testDeleteObject() + testListObjectVersionsSimple() + testListObjectVersionsWithPrefixAndDelimiter() + testListObjectVersionsKeysContinuation() + testListObjectVersionsVersionIDContinuation() + testListObjectsVersionsWithEmptyDirObject() + testTagging() + testLockingLegalhold() + testLockingRetentionGovernance() + testLockingRetentionCompliance() +} diff --git a/mint/build/versioning/put.go b/mint/build/versioning/put.go new file mode 100644 index 000000000..179036fd7 --- /dev/null +++ b/mint/build/versioning/put.go @@ -0,0 +1,294 @@ +/* +* +* Mint, (C) 2021 Minio, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software + +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ + +package main + +import ( + "errors" + "fmt" + "math/rand" + "net/url" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" +) + +// Put two objects with the same name but with different content +func testPutObject() { + startTime := time.Now() + function := "testPutObject" + bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-") + object := "testObject" + expiry := 1 * time.Minute + args := map[string]interface{}{ + "bucketName": bucket, + "objectName": object, + "expiry": expiry, + } + + _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() + return + } + defer cleanupBucket(bucket, function, args, startTime) + + putVersioningInput := &s3.PutBucketVersioningInput{ + Bucket: aws.String(bucket), + VersioningConfiguration: &s3.VersioningConfiguration{ + Status: aws.String("Enabled"), + }, + } + + _, err = s3Client.PutBucketVersioning(putVersioningInput) + if err != nil { + if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") { + ignoreLog(function, args, startTime, "Versioning is not implemented").Info() + return + } + failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal() + return + } + + putInput1 := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader("my content 1")), + Bucket: aws.String(bucket), + Key: aws.String(object), + } + _, err = s3Client.PutObject(putInput1) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() + return + } + putInput2 := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader("content file 2")), + Bucket: aws.String(bucket), + Key: aws.String(object), + } + _, err = s3Client.PutObject(putInput2) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() + return + } + + input := &s3.ListObjectVersionsInput{ + Bucket: aws.String(bucket), + } + + result, err := s3Client.ListObjectVersions(input) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() + return + } + + if len(result.Versions) != 2 { + failureLog(function, args, startTime, "", "Unexpected list content", errors.New("unexpected number of versions")).Fatal() + return + } + + vid1 := *result.Versions[0] + vid2 := *result.Versions[1] + + if *vid1.VersionId == "" || *vid2.VersionId == "" || *vid1.VersionId == *vid2.VersionId { + failureLog(function, args, startTime, "", "Unexpected list content", errors.New("unexpected VersionId field")).Fatal() + return + } + + if *vid1.IsLatest == false || *vid2.IsLatest == true { + failureLog(function, args, startTime, "", "Unexpected list content", errors.New("unexpected IsLatest field")).Fatal() + return + } + + if *vid1.Size != 14 || *vid2.Size != 12 { + failureLog(function, args, startTime, "", "Unexpected list content", errors.New("unexpected Size field")).Fatal() + return + } + + if *vid1.ETag != "\"e847032b45d3d76230058a80d8ca909b\"" || *vid2.ETag != "\"094459df8fcebffc70d9aa08d75f9944\"" { + failureLog(function, args, startTime, "", "Unexpected list content", errors.New("unexpected ETag field")).Fatal() + return + } + + if *vid1.Key != "testObject" || *vid2.Key != "testObject" { + failureLog(function, args, startTime, "", "Unexpected list content", errors.New("unexpected Key field")).Fatal() + return + } + + if (*vid1.LastModified).Before(*vid2.LastModified) { + failureLog(function, args, startTime, "", "Unexpected list content", errors.New("unexpected Last modified field")).Fatal() + return + } + + successLogger(function, args, startTime).Info() +} + +// Upload object versions with tagging and metadata and check them +func testPutObjectWithTaggingAndMetadata() { + startTime := time.Now() + function := "testPutObjectWithTaggingAndMetadata" + bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-") + object := "testObject" + expiry := 1 * time.Minute + args := map[string]interface{}{ + "bucketName": bucket, + "objectName": object, + "expiry": expiry, + } + + _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() + return + } + defer cleanupBucket(bucket, function, args, startTime) + + putVersioningInput := &s3.PutBucketVersioningInput{ + Bucket: aws.String(bucket), + VersioningConfiguration: &s3.VersioningConfiguration{ + Status: aws.String("Enabled"), + }, + } + + _, err = s3Client.PutBucketVersioning(putVersioningInput) + if err != nil { + if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") { + ignoreLog(function, args, startTime, "Versioning is not implemented").Info() + return + } + failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal() + return + } + + type objectUpload struct { + tags string + metadata map[string]string + versionId string + } + + uploads := []objectUpload{ + {tags: "key=value"}, + {}, + {metadata: map[string]string{"My-Metadata-Key": "my-metadata-val"}}, + {tags: "key1=value1&key2=value2", metadata: map[string]string{"Foo-Key": "foo-val"}}, + } + + for i := range uploads { + putInput := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader("foocontent")), + Bucket: aws.String(bucket), + Key: aws.String(object), + } + if uploads[i].tags != "" { + putInput.Tagging = aws.String(uploads[i].tags) + } + if uploads[i].metadata != nil { + putInput.Metadata = make(map[string]*string) + for k, v := range uploads[i].metadata { + putInput.Metadata[k] = aws.String(v) + } + } + result, err := s3Client.PutObject(putInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("PUT object expected to succeed but got %v", err), err).Fatal() + return + } + uploads[i].versionId = *result.VersionId + } + + for i := range uploads { + putInput := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader("foocontent")), + Bucket: aws.String(bucket), + Key: aws.String(object), + } + if uploads[i].tags != "" { + putInput.Tagging = aws.String(uploads[i].tags) + } + if uploads[i].metadata != nil { + putInput.Metadata = make(map[string]*string) + for k, v := range uploads[i].metadata { + putInput.Metadata[k] = aws.String(v) + } + } + result, err := s3Client.PutObject(putInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("PUT object expected to succeed but got %v", err), err).Fatal() + return + } + uploads[i].versionId = *result.VersionId + } + + // Check for tagging after removal + for i := range uploads { + if uploads[i].tags != "" { + input := &s3.GetObjectTaggingInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + VersionId: aws.String(uploads[i].versionId), + } + tagResult, err := s3Client.GetObjectTagging(input) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("GET Object tagging expected to succeed but got %v", err), err).Fatal() + return + } + var vals = make(url.Values) + for _, tag := range tagResult.TagSet { + vals.Add(*tag.Key, *tag.Value) + } + if uploads[i].tags != vals.Encode() { + failureLog(function, args, startTime, "", "PUT Object with tagging header returned unexpected result", nil).Fatal() + return + + } + } + + if uploads[i].metadata != nil { + input := &s3.HeadObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + VersionId: aws.String(uploads[i].versionId), + } + result, err := s3Client.HeadObject(input) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("HEAD Object expected to succeed but got %v", err), err).Fatal() + return + } + + for expectedKey, expectedVal := range uploads[i].metadata { + gotValue, ok := result.Metadata[expectedKey] + if !ok { + failureLog(function, args, startTime, "", "HEAD Object returned unexpected metadata key result", nil).Fatal() + return + } + if expectedVal != *gotValue { + failureLog(function, args, startTime, "", "HEAD Object returned unexpected metadata value result", nil).Fatal() + return + } + } + } + } + + successLogger(function, args, startTime).Info() +} diff --git a/mint/build/versioning/retention.go b/mint/build/versioning/retention.go new file mode 100644 index 000000000..5e9e6c966 --- /dev/null +++ b/mint/build/versioning/retention.go @@ -0,0 +1,277 @@ +/* +* +* Mint, (C) 2021 Minio, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software + +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ + +package main + +import ( + "fmt" + "math/rand" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" +) + +// Test locking retention governance +func testLockingRetentionGovernance() { + startTime := time.Now() + function := "testLockingRetentionGovernance" + bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-") + object := "testObject" + expiry := 1 * time.Minute + args := map[string]interface{}{ + "bucketName": bucket, + "objectName": object, + "expiry": expiry, + } + + _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + ObjectLockEnabledForBucket: aws.Bool(true), + }) + if err != nil { + if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") { + ignoreLog(function, args, startTime, "Versioning is not implemented").Info() + return + } + failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() + return + } + defer cleanupBucket(bucket, function, args, startTime) + + type uploadedObject struct { + retention string + retentionUntil time.Time + successfulRemove bool + versionId string + deleteMarker bool + } + + uploads := []uploadedObject{ + {}, + {retention: "GOVERNANCE", retentionUntil: time.Now().UTC().Add(time.Hour)}, + {}, + } + + // Upload versions and save their version IDs + for i := range uploads { + putInput := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader("content")), + Bucket: aws.String(bucket), + Key: aws.String(object), + } + if uploads[i].retention != "" { + putInput.ObjectLockMode = aws.String(uploads[i].retention) + putInput.ObjectLockRetainUntilDate = aws.Time(uploads[i].retentionUntil) + + } + output, err := s3Client.PutObject(putInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() + return + } + uploads[i].versionId = *output.VersionId + } + + // In all cases, we can remove an object by creating a delete marker + // First delete without version ID + deleteInput := &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + } + deleteOutput, err := s3Client.DeleteObject(deleteInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("DELETE expected to succeed but got %v", err), err).Fatal() + return + } + + uploads = append(uploads, uploadedObject{versionId: *deleteOutput.VersionId, deleteMarker: true}) + + // Put tagging on each version + for i := range uploads { + if uploads[i].deleteMarker { + continue + } + deleteInput := &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + VersionId: aws.String(uploads[i].versionId), + } + _, err = s3Client.DeleteObject(deleteInput) + if err == nil && uploads[i].retention != "" { + failureLog(function, args, startTime, "", "DELETE expected to fail but succeed instead", nil).Fatal() + return + } + if err != nil && uploads[i].retention == "" { + failureLog(function, args, startTime, "", fmt.Sprintf("DELETE expected to succeed but got %v", err), err).Fatal() + return + } + } + + successLogger(function, args, startTime).Info() +} + +// Test locking retention compliance +func testLockingRetentionCompliance() { + startTime := time.Now() + function := "testLockingRetentionCompliance" + bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-") + object := "testObject" + expiry := 1 * time.Minute + args := map[string]interface{}{ + "bucketName": bucket, + "objectName": object, + "expiry": expiry, + } + + _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + ObjectLockEnabledForBucket: aws.Bool(true), + }) + if err != nil { + if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") { + ignoreLog(function, args, startTime, "Versioning is not implemented").Info() + return + } + failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() + return + } + + defer func() { + start := time.Now() + + input := &s3.ListObjectVersionsInput{ + Bucket: aws.String(bucket), + } + + for time.Since(start) < 30*time.Minute { + err := s3Client.ListObjectVersionsPages(input, + func(page *s3.ListObjectVersionsOutput, lastPage bool) bool { + for _, v := range page.Versions { + input := &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: v.Key, + VersionId: v.VersionId, + } + _, err := s3Client.DeleteObject(input) + if err != nil { + return true + } + } + for _, v := range page.DeleteMarkers { + input := &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: v.Key, + VersionId: v.VersionId, + } + _, err := s3Client.DeleteObject(input) + if err != nil { + return true + } + } + return true + }) + + _, err = s3Client.DeleteBucket(&s3.DeleteBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + time.Sleep(30 * time.Second) + continue + } + return + } + + failureLog(function, args, startTime, "", "Unable to cleanup bucket after compliance tests", nil).Fatal() + return + + }() + + type uploadedObject struct { + retention string + retentionUntil time.Time + successfulRemove bool + versionId string + deleteMarker bool + } + + uploads := []uploadedObject{ + {}, + {retention: "COMPLIANCE", retentionUntil: time.Now().UTC().Add(time.Minute)}, + {}, + } + + // Upload versions and save their version IDs + for i := range uploads { + putInput := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader("content")), + Bucket: aws.String(bucket), + Key: aws.String(object), + } + if uploads[i].retention != "" { + putInput.ObjectLockMode = aws.String(uploads[i].retention) + putInput.ObjectLockRetainUntilDate = aws.Time(uploads[i].retentionUntil) + + } + output, err := s3Client.PutObject(putInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() + return + } + uploads[i].versionId = *output.VersionId + } + + // In all cases, we can remove an object by creating a delete marker + // First delete without version ID + deleteInput := &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + } + deleteOutput, err := s3Client.DeleteObject(deleteInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("DELETE expected to succeed but got %v", err), err).Fatal() + return + } + + uploads = append(uploads, uploadedObject{versionId: *deleteOutput.VersionId, deleteMarker: true}) + + // Put tagging on each version + for i := range uploads { + if uploads[i].deleteMarker { + continue + } + deleteInput := &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + VersionId: aws.String(uploads[i].versionId), + } + _, err = s3Client.DeleteObject(deleteInput) + if err == nil && uploads[i].retention != "" { + failureLog(function, args, startTime, "", "DELETE expected to fail but succeed instead", nil).Fatal() + return + } + if err != nil && uploads[i].retention == "" { + failureLog(function, args, startTime, "", fmt.Sprintf("DELETE expected to succeed but got %v", err), err).Fatal() + return + } + } + + successLogger(function, args, startTime).Info() +} diff --git a/mint/build/versioning/stat.go b/mint/build/versioning/stat.go new file mode 100644 index 000000000..52d294c65 --- /dev/null +++ b/mint/build/versioning/stat.go @@ -0,0 +1,183 @@ +/* +* +* Mint, (C) 2021 Minio, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software + +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ + +package main + +import ( + "fmt" + "math/rand" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/s3" +) + +func testStatObject() { + startTime := time.Now() + function := "testStatObject" + bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-") + object := "testObject" + expiry := 1 * time.Minute + args := map[string]interface{}{ + "bucketName": bucket, + "objectName": object, + "expiry": expiry, + } + + _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() + return + } + defer cleanupBucket(bucket, function, args, startTime) + + putVersioningInput := &s3.PutBucketVersioningInput{ + Bucket: aws.String(bucket), + VersioningConfiguration: &s3.VersioningConfiguration{ + Status: aws.String("Enabled"), + }, + } + + _, err = s3Client.PutBucketVersioning(putVersioningInput) + if err != nil { + if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") { + ignoreLog(function, args, startTime, "Versioning is not implemented").Info() + return + } + failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal() + return + } + + putInput1 := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader("my content 1")), + Bucket: aws.String(bucket), + Key: aws.String(object), + } + _, err = s3Client.PutObject(putInput1) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() + return + } + putInput2 := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader("content file 2")), + Bucket: aws.String(bucket), + Key: aws.String(object), + } + _, err = s3Client.PutObject(putInput2) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() + return + } + + deleteInput := &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + } + + _, err = s3Client.DeleteObject(deleteInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("Delete expected to succeed but got %v", err), err).Fatal() + return + } + + input := &s3.ListObjectVersionsInput{ + Bucket: aws.String(bucket), + } + + result, err := s3Client.ListObjectVersions(input) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal() + return + } + + testCases := []struct { + size int64 + versionId string + etag string + contentType string + deleteMarker bool + }{ + {0, *(*result.DeleteMarkers[0]).VersionId, "", "", true}, + {14, *(*result.Versions[0]).VersionId, "\"e847032b45d3d76230058a80d8ca909b\"", "binary/octet-stream", false}, + {12, *(*result.Versions[1]).VersionId, "\"094459df8fcebffc70d9aa08d75f9944\"", "binary/octet-stream", false}, + } + + for i, testCase := range testCases { + headInput := &s3.HeadObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + VersionId: aws.String(testCase.versionId), + } + + result, err := s3Client.HeadObject(headInput) + if testCase.deleteMarker && err == nil { + failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) expected to fail but succeeded", i+1), nil).Fatal() + return + } + + if !testCase.deleteMarker && err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) expected to succeed but failed", i+1), err).Fatal() + return + } + + if testCase.deleteMarker { + aerr, ok := err.(awserr.Error) + if !ok { + failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) unexpected error with delete marker", i+1), err).Fatal() + return + } + if aerr.Code() != "MethodNotAllowed" { + failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) unexpected error code with delete marker", i+1), err).Fatal() + return + } + continue + } + + if *result.ContentLength != testCase.size { + failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) unexpected Content-Length", i+1), err).Fatal() + return + } + + if *result.ETag != testCase.etag { + failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) unexpected ETag", i+1), err).Fatal() + return + } + + if *result.ContentType != testCase.contentType { + failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) unexpected Content-Type", i+1), err).Fatal() + return + } + + if result.DeleteMarker != nil && *result.DeleteMarker { + failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) unexpected DeleteMarker", i+1), err).Fatal() + return + } + + if time.Since(*result.LastModified) > time.Hour { + failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) unexpected LastModified", i+1), err).Fatal() + return + } + } + + successLogger(function, args, startTime).Info() +} diff --git a/mint/build/versioning/tagging.go b/mint/build/versioning/tagging.go new file mode 100644 index 000000000..f41a35837 --- /dev/null +++ b/mint/build/versioning/tagging.go @@ -0,0 +1,203 @@ +/* +* +* Mint, (C) 2021 Minio, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software + +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ + +package main + +import ( + "fmt" + "math/rand" + "reflect" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" +) + +// Test PUT/GET/DELETE tagging for separate versions +func testTagging() { + startTime := time.Now() + function := "testTagging" + bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-") + object := "testObject" + expiry := 1 * time.Minute + args := map[string]interface{}{ + "bucketName": bucket, + "objectName": object, + "expiry": expiry, + } + + _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() + return + } + defer cleanupBucket(bucket, function, args, startTime) + + putVersioningInput := &s3.PutBucketVersioningInput{ + Bucket: aws.String(bucket), + VersioningConfiguration: &s3.VersioningConfiguration{ + Status: aws.String("Enabled"), + }, + } + + _, err = s3Client.PutBucketVersioning(putVersioningInput) + if err != nil { + if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") { + ignoreLog(function, args, startTime, "Versioning is not implemented").Info() + return + } + failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal() + return + } + + type uploadedObject struct { + content string + tagging []*s3.Tag + versionId string + deleteMarker bool + } + + uploads := []uploadedObject{ + {content: "my content 1", tagging: []*s3.Tag{{Key: aws.String("type"), Value: aws.String("text")}}}, + {content: "content file 2"}, + {content: "\"%32&é", tagging: []*s3.Tag{{Key: aws.String("type"), Value: aws.String("garbage")}}}, + {deleteMarker: true}, + } + + // Upload versions and save their version IDs + for i := range uploads { + if uploads[i].deleteMarker { + // Delete the current object to create a delete marker) + deleteInput := &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + } + deleteOutput, err := s3Client.DeleteObject(deleteInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("DELETE object expected to succeed but got %v", err), err).Fatal() + return + } + uploads[i].versionId = *deleteOutput.VersionId + continue + } + + putInput := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader(uploads[i].content)), + Bucket: aws.String(bucket), + Key: aws.String(object), + } + output, err := s3Client.PutObject(putInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() + return + } + uploads[i].versionId = *output.VersionId + } + + // Put tagging on each version + for i := range uploads { + if uploads[i].tagging == nil { + continue + } + putTaggingInput := &s3.PutObjectTaggingInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + Tagging: &s3.Tagging{TagSet: uploads[i].tagging}, + VersionId: aws.String(uploads[i].versionId), + } + _, err = s3Client.PutObjectTagging(putTaggingInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("PUT Object tagging expected to succeed but got %v", err), err).Fatal() + return + } + } + + // Check versions tagging + for i := range uploads { + input := &s3.GetObjectTaggingInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + VersionId: aws.String(uploads[i].versionId), + } + result, err := s3Client.GetObjectTagging(input) + if err == nil && uploads[i].deleteMarker { + failureLog(function, args, startTime, "", "GET Object tagging expected to fail with delete marker but succeded", err).Fatal() + return + } + if err != nil && !uploads[i].deleteMarker { + failureLog(function, args, startTime, "", fmt.Sprintf("GET Object tagging expected to succeed but got %v", err), err).Fatal() + return + } + + if uploads[i].deleteMarker { + continue + } + + if !reflect.DeepEqual(result.TagSet, uploads[i].tagging) { + failureLog(function, args, startTime, "", "GET Object tagging returned unexpected result", nil).Fatal() + return + } + } + + // Remove all tagging for all objects + for i := range uploads { + input := &s3.DeleteObjectTaggingInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + VersionId: aws.String(uploads[i].versionId), + } + _, err := s3Client.DeleteObjectTagging(input) + if err == nil && uploads[i].deleteMarker { + failureLog(function, args, startTime, "", "DELETE Object tagging expected to fail with delete marker but succeded", err).Fatal() + return + } + if err != nil && !uploads[i].deleteMarker { + failureLog(function, args, startTime, "", fmt.Sprintf("GET Object tagging expected to succeed but got %v", err), err).Fatal() + return + } + } + + // Check for tagging after removal + for i := range uploads { + if uploads[i].deleteMarker { + // Avoid testing this use case since already tested earlier + continue + } + input := &s3.GetObjectTaggingInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + VersionId: aws.String(uploads[i].versionId), + } + result, err := s3Client.GetObjectTagging(input) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("GET Object tagging expected to succeed but got %v", err), err).Fatal() + return + } + var nilTagSet []*s3.Tag + if !reflect.DeepEqual(result.TagSet, nilTagSet) { + failureLog(function, args, startTime, "", "GET Object tagging after DELETE returned unexpected result", nil).Fatal() + return + } + } + + successLogger(function, args, startTime).Info() +} diff --git a/mint/build/versioning/utils.go b/mint/build/versioning/utils.go new file mode 100644 index 000000000..3c2b1ac24 --- /dev/null +++ b/mint/build/versioning/utils.go @@ -0,0 +1,137 @@ +/* +* +* Mint, (C) 2021 Minio, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software + +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ + +package main + +import ( + "encoding/json" + "encoding/xml" + "fmt" + "math/rand" + "net/http" + "strings" + "time" + + log "github.com/sirupsen/logrus" +) + +const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" +const ( + letterIdxBits = 6 // 6 bits to represent a letter index + letterIdxMask = 1<= 0; { + if remain == 0 { + cache, remain = src.Int63(), letterIdxMax + } + if idx := int(cache & letterIdxMask); idx < len(letterBytes) { + b[i] = letterBytes[idx] + i-- + } + cache >>= letterIdxBits + remain-- + } + return prefix + string(b[0:30-len(prefix)]) +} diff --git a/mint/release.sh b/mint/release.sh index 8ee2f7ec4..40b07e7c7 100755 --- a/mint/release.sh +++ b/mint/release.sh @@ -17,6 +17,7 @@ export MINT_ROOT_DIR=${MINT_ROOT_DIR:-/mint} export MINT_RUN_CORE_DIR="$MINT_ROOT_DIR/run/core" +export MINT_RUN_BUILD_DIR="$MINT_ROOT_DIR/build" export MINT_RUN_SECURITY_DIR="$MINT_ROOT_DIR/run/security" export WGET="wget --quiet --no-check-certificate" diff --git a/mint/run/core/versioning/run.sh b/mint/run/core/versioning/run.sh new file mode 100755 index 000000000..0ebd9f7a1 --- /dev/null +++ b/mint/run/core/versioning/run.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Mint (C) 2021 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# handle command line arguments +if [ $# -ne 2 ]; then + echo "usage: run.sh " + exit 1 +fi + +output_log_file="$1" +error_log_file="$2" + +# run tests +/mint/run/core/versioning/tests 1>>"$output_log_file" 2>"$error_log_file"