MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-24 06:05:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add mint versioning tests (#11500)

Browse Source 
Co-authored-by: Harshavardhana <harsha@minio.io>
This commit is contained in:
Anis Elleuch 2021-03-06 04:15:42 +01:00 committed by GitHub
parent 79b0d056a2
commit 7ea95fcec8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 2550 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
mint/build/versioning/bucket.go Normal file
View File

@ -0,0 +1,85 @@
/*
*
* Mint, (C) 2021 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package main
import (
"errors"
"math/rand"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
)
// Tests bucket versioned bucket and get its versioning configuration to check
func testMakeBucket() {
s3Client.Config.Region = aws.String("us-east-1")
// initialize logging params
startTime := time.Now()
function := "testCreateVersioningBucket"
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-")
args := map[string]interface{}{
"bucketName": bucketName,
}
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucketName),
})
if err != nil {
failureLog(function, args, startTime, "", "Versioning CreateBucket Failed", err).Fatal()
return
}
defer cleanupBucket(bucketName, function, args, startTime)
putVersioningInput := &s3.PutBucketVersioningInput{
Bucket: aws.String(bucketName),
VersioningConfiguration: &s3.VersioningConfiguration{
MFADelete: aws.String("Disabled"),
Status: aws.String("Enabled"),
},
}
_, err = s3Client.PutBucketVersioning(putVersioningInput)
if err != nil {
if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") {
ignoreLog(function, args, startTime, "Versioning is not implemented").Info()
return
}
failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal()
return
}
getVersioningInput := &s3.GetBucketVersioningInput{
Bucket: aws.String(bucketName),
}
result, err := s3Client.GetBucketVersioning(getVersioningInput)
if err != nil {
failureLog(function, args, startTime, "", "Get Versioning failed", err).Fatal()
return
}
if *result.Status != "Enabled" {
failureLog(function, args, startTime, "", "Get Versioning status failed", errors.New("unexpected versioning status")).Fatal()
}
successLogger(function, args, startTime).Info()
}

174
mint/build/versioning/delete.go Normal file
View File

@ -0,0 +1,174 @@
/*
*
* Mint, (C) 2021 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package main
import (
"fmt"
"io/ioutil"
"math/rand"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/s3"
)
func testDeleteObject() {
startTime := time.Now()
function := "testDeleteObject"
bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-")
object := "testObject"
objectContent := "my object content"
expiry := 1 * time.Minute
args := map[string]interface{}{
"bucketName": bucket,
"objectName": object,
"expiry": expiry,
}
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
return
}
defer cleanupBucket(bucket, function, args, startTime)
putVersioningInput := &s3.PutBucketVersioningInput{
Bucket: aws.String(bucket),
VersioningConfiguration: &s3.VersioningConfiguration{
Status: aws.String("Enabled"),
},
}
_, err = s3Client.PutBucketVersioning(putVersioningInput)
if err != nil {
if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") {
ignoreLog(function, args, startTime, "Versioning is not implemented").Info()
return
}
failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal()
return
}
putInput := &s3.PutObjectInput{
Body: aws.ReadSeekCloser(strings.NewReader(objectContent)),
Bucket: aws.String(bucket),
Key: aws.String(object),
}
putOutput, err := s3Client.PutObject(putInput)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal()
return
}
// First delete without version ID
deleteInput := &s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
}
delOutput, err := s3Client.DeleteObject(deleteInput)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("Delete expected to succeed but got %v", err), err).Fatal()
return
}
// Get the delete marker version, should lead to an error
getInput := &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
VersionId: aws.String(*delOutput.VersionId),
}
result, err := s3Client.GetObject(getInput)
if err == nil {
failureLog(function, args, startTime, "", "GetObject expected to fail but succeeded", nil).Fatal()
return
}
if err != nil {
aerr, ok := err.(awserr.Error)
if !ok {
failureLog(function, args, startTime, "", "GetObject unexpected error with delete marker", err).Fatal()
return
}
if aerr.Code() != "MethodNotAllowed" {
failureLog(function, args, startTime, "", "GetObject unexpected error with delete marker", err).Fatal()
return
}
}
// Get the older version, make sure it is preserved
getInput = &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
VersionId: aws.String(*putOutput.VersionId),
}
result, err = s3Client.GetObject(getInput)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("GetObject expected to succeed but failed with %v", err), err).Fatal()
return
}
body, err := ioutil.ReadAll(result.Body)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("GetObject expected to return data but failed with %v", err), err).Fatal()
return
}
result.Body.Close()
if string(body) != objectContent {
failureLog(function, args, startTime, "", "GetObject unexpected body content", nil).Fatal()
return
}
for i, versionID := range []string{*delOutput.VersionId, *putOutput.VersionId} {
delInput := &s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
VersionId: aws.String(versionID),
}
_, err := s3Client.DeleteObject(delInput)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("DeleteObject (%d) expected to succeed but failed", i+1), err).Fatal()
return
}
}
listInput := &s3.ListObjectVersionsInput{
Bucket: aws.String(bucket),
}
listOutput, err := s3Client.ListObjectVersions(listInput)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal()
return
}
if len(listOutput.DeleteMarkers) != 0 || len(listOutput.CommonPrefixes) != 0 || len(listOutput.Versions) != 0 {
failureLog(function, args, startTime, "", "ListObjectVersions returned some entries but expected to return nothing", nil).Fatal()
return
}
successLogger(function, args, startTime).Info()
}

171
mint/build/versioning/get.go Normal file
View File

@ -0,0 +1,171 @@
/*
*
* Mint, (C) 2021 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package main
import (
"fmt"
"io/ioutil"
"math/rand"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/s3"
)
// testGetObject tests all get object features - picking a particular
// version id, check content and its metadata
func testGetObject() {
startTime := time.Now()
function := "testGetObject"
bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-")
object := "testObject"
expiry := 1 * time.Minute
args := map[string]interface{}{
"bucketName": bucket,
"objectName": object,
"expiry": expiry,
}
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
return
}
defer cleanupBucket(bucket, function, args, startTime)
putVersioningInput := &s3.PutBucketVersioningInput{
Bucket: aws.String(bucket),
VersioningConfiguration: &s3.VersioningConfiguration{
Status: aws.String("Enabled"),
},
}
_, err = s3Client.PutBucketVersioning(putVersioningInput)
if err != nil {
if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") {
ignoreLog(function, args, startTime, "Versioning is not implemented").Info()
return
}
failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal()
return
}
putInput1 := &s3.PutObjectInput{
Body: aws.ReadSeekCloser(strings.NewReader("my content 1")),
Bucket: aws.String(bucket),
Key: aws.String(object),
}
_, err = s3Client.PutObject(putInput1)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal()
return
}
putInput2 := &s3.PutObjectInput{
Body: aws.ReadSeekCloser(strings.NewReader("content file 2")),
Bucket: aws.String(bucket),
Key: aws.String(object),
}
_, err = s3Client.PutObject(putInput2)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal()
return
}
deleteInput := &s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
}
_, err = s3Client.DeleteObject(deleteInput)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("Delete expected to succeed but got %v", err), err).Fatal()
return
}
input := &s3.ListObjectVersionsInput{
Bucket: aws.String(bucket),
}
result, err := s3Client.ListObjectVersions(input)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal()
return
}
testCases := []struct {
content string
versionId string
deleteMarker bool
}{
{"", *(*result.DeleteMarkers[0]).VersionId, true},
{"content file 2", *(*result.Versions[0]).VersionId, false},
{"my content 1", *(*result.Versions[1]).VersionId, false},
}
for i, testCase := range testCases {
getInput := &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
VersionId: aws.String(testCase.versionId),
}
result, err := s3Client.GetObject(getInput)
if testCase.deleteMarker && err == nil {
failureLog(function, args, startTime, "", fmt.Sprintf("GetObject(%d) expected to fail but succeeded", i+1), nil).Fatal()
return
}
if !testCase.deleteMarker && err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("GetObject(%d) expected to succeed but failed", i+1), err).Fatal()
return
}
if testCase.deleteMarker {
aerr, ok := err.(awserr.Error)
if !ok {
failureLog(function, args, startTime, "", fmt.Sprintf("GetObject(%d) unexpected error with delete marker", i+1), err).Fatal()
return
}
if aerr.Code() != "MethodNotAllowed" {
failureLog(function, args, startTime, "", fmt.Sprintf("GetObject(%d) unexpected error with delete marker", i+1), err).Fatal()
return
}
continue
}
body, err := ioutil.ReadAll(result.Body)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("GetObject(%d) expected to return data but failed", i+1), err).Fatal()
return
}
result.Body.Close()
if string(body) != testCase.content {
failureLog(function, args, startTime, "", fmt.Sprintf("GetObject(%d) unexpected body content", i+1), err).Fatal()
return
}
}
successLogger(function, args, startTime).Info()
}

8
mint/build/versioning/go.mod Normal file
View File

@ -0,0 +1,8 @@
module mint.minio.io/versioning/tests
go 1.16
require (
github.com/aws/aws-sdk-go v1.37.9
github.com/sirupsen/logrus v1.7.0
)

36
mint/build/versioning/go.sum Normal file
View File

@ -0,0 +1,36 @@
github.com/aws/aws-sdk-go v1.37.9 h1:sgRbr+heubkgSwkn9fQMF80l9xjXkmhzk9DLdsaYh+c=
github.com/aws/aws-sdk-go v1.37.9/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

21
mint/build/versioning/install.sh Executable file
View File

@ -0,0 +1,21 @@
#!/bin/bash -e
#
# Mint (C) 2017-2021 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
test_run_dir="$MINT_RUN_CORE_DIR/versioning"
test_build_dir="$MINT_RUN_BUILD_DIR/versioning"
(cd "$test_build_dir" && GO111MODULE=on CGO_ENABLED=0 go build --ldflags "-s -w" -o "$test_run_dir/tests")

140
mint/build/versioning/legalhold.go Normal file
View File

@ -0,0 +1,140 @@
/*
*
* Mint, (C) 2021 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package main
import (
"fmt"
"math/rand"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
)
// Test locking for different versions
func testLockingLegalhold() {
startTime := time.Now()
function := "testLockingLegalhold"
bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-")
object := "testObject"
expiry := 1 * time.Minute
args := map[string]interface{}{
"bucketName": bucket,
"objectName": object,
"expiry": expiry,
}
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucket),
ObjectLockEnabledForBucket: aws.Bool(true),
})
if err != nil {
if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") {
ignoreLog(function, args, startTime, "Versioning is not implemented").Info()
return
}
failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
return
}
defer cleanupBucket(bucket, function, args, startTime)
type uploadedObject struct {
legalhold string
successfulRemove bool
versionId string
deleteMarker bool
}
uploads := []uploadedObject{
{legalhold: "ON"},
{legalhold: "OFF"},
}
// Upload versions and save their version IDs
for i := range uploads {
putInput := &s3.PutObjectInput{
Body: aws.ReadSeekCloser(strings.NewReader("content")),
Bucket: aws.String(bucket),
Key: aws.String(object),
ObjectLockLegalHoldStatus: aws.String(uploads[i].legalhold),
}
output, err := s3Client.PutObject(putInput)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal()
return
}
uploads[i].versionId = *output.VersionId
}
// In all cases, we can remove an object by creating a delete marker
// First delete without version ID
deleteInput := &s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
}
deleteOutput, err := s3Client.DeleteObject(deleteInput)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("DELETE expected to succeed but got %v", err), err).Fatal()
return
}
uploads = append(uploads, uploadedObject{versionId: *deleteOutput.VersionId, deleteMarker: true})
// Put tagging on each version
for i := range uploads {
if uploads[i].deleteMarker {
continue
}
deleteInput := &s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
VersionId: aws.String(uploads[i].versionId),
}
_, err = s3Client.DeleteObject(deleteInput)
if err == nil && uploads[i].legalhold == "ON" {
failureLog(function, args, startTime, "", "DELETE expected to fail but succeed instead", nil).Fatal()
return
}
if err != nil && uploads[i].legalhold == "OFF" {
failureLog(function, args, startTime, "", fmt.Sprintf("DELETE expected to succeed but got %v", err), err).Fatal()
return
}
}
for i := range uploads {
if uploads[i].deleteMarker || uploads[i].legalhold == "OFF" {
continue
}
input := &s3.PutObjectLegalHoldInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
LegalHold: &s3.ObjectLockLegalHold{Status: aws.String("OFF")},
VersionId: aws.String(uploads[i].versionId),
}
_, err := s3Client.PutObjectLegalHold(input)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("Turning off legalhold failed with %v", err), err).Fatal()
return
}
}
successLogger(function, args, startTime).Info()
}

670
mint/build/versioning/list.go Normal file
View File

@ -0,0 +1,670 @@
/*
*
* Mint, (C) 2021 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package main
import (
"fmt"
"math/rand"
"reflect"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
)
// Test regular listing result with simple use cases:
// Upload an object ten times, delete it once (delete marker)
// and check listing result
func testListObjectVersionsSimple() {
startTime := time.Now()
function := "testListObjectVersionsSimple"
bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-")
object := "testObject"
expiry := 1 * time.Minute
args := map[string]interface{}{
"bucketName": bucket,
"objectName": object,
"expiry": expiry,
}
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
return
}
defer cleanupBucket(bucket, function, args, startTime)
putVersioningInput := &s3.PutBucketVersioningInput{
Bucket: aws.String(bucket),
VersioningConfiguration: &s3.VersioningConfiguration{
Status: aws.String("Enabled"),
},
}
_, err = s3Client.PutBucketVersioning(putVersioningInput)
if err != nil {
if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") {
ignoreLog(function, args, startTime, "Versioning is not implemented").Info()
return
}
failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal()
return
}
for i := 0; i < 10; i++ {
putInput1 := &s3.PutObjectInput{
Body: aws.ReadSeekCloser(strings.NewReader("my content 1")),
Bucket: aws.String(bucket),
Key: aws.String(object),
}
_, err = s3Client.PutObject(putInput1)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal()
return
}
}
input := &s3.ListObjectVersionsInput{
Bucket: aws.String(bucket),
}
deleteInput := &s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
}
_, err = s3Client.DeleteObject(deleteInput)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("Delete expected to succeed but got %v", err), err).Fatal()
return
}
// Accumulate all versions IDs
var versionIDs = make(map[string]struct{})
result, err := s3Client.ListObjectVersions(input)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal()
return
}
// Check the delete marker entries
if len(result.DeleteMarkers) != 1 {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected result", nil).Fatal()
return
}
dm := *result.DeleteMarkers[0]
if !*dm.IsLatest {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected result", nil).Fatal()
return
}
if *dm.Key != object {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected result", nil).Fatal()
return
}
if time.Since(*dm.LastModified) > time.Hour {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected result", nil).Fatal()
return
}
if *dm.VersionId == "" {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected result", nil).Fatal()
return
}
versionIDs[*dm.VersionId] = struct{}{}
// Check versions entries
if len(result.Versions) != 10 {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected result", nil).Fatal()
return
}
for _, version := range result.Versions {
v := *version
if *v.IsLatest {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected IsLatest field", nil).Fatal()
return
}
if *v.Key != object {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected Key field", nil).Fatal()
return
}
if time.Since(*v.LastModified) > time.Hour {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected LastModified field", nil).Fatal()
return
}
if *v.VersionId == "" {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected VersionId field", nil).Fatal()
return
}
if *v.ETag != "\"094459df8fcebffc70d9aa08d75f9944\"" {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected ETag field", nil).Fatal()
return
}
if *v.Size != 12 {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected Size field", nil).Fatal()
return
}
if *v.StorageClass != "STANDARD" {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected StorageClass field", nil).Fatal()
return
}
versionIDs[*v.VersionId] = struct{}{}
}
// Ensure that we have 11 distinct versions IDs
if len(versionIDs) != 11 {
failureLog(function, args, startTime, "", "ListObjectVersions didn't return 11 different version IDs", nil).Fatal()
return
}
successLogger(function, args, startTime).Info()
}
func testListObjectVersionsWithPrefixAndDelimiter() {
startTime := time.Now()
function := "testListObjectVersionsWithPrefixAndDelimiter"
bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-")
object := "testObject"
expiry := 1 * time.Minute
args := map[string]interface{}{
"bucketName": bucket,
"objectName": object,
"expiry": expiry,
}
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
return
}
defer cleanupBucket(bucket, function, args, startTime)
putVersioningInput := &s3.PutBucketVersioningInput{
Bucket: aws.String(bucket),
VersioningConfiguration: &s3.VersioningConfiguration{
Status: aws.String("Enabled"),
},
}
_, err = s3Client.PutBucketVersioning(putVersioningInput)
if err != nil {
if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") {
ignoreLog(function, args, startTime, "Versioning is not implemented").Info()
return
}
failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal()
return
}
for _, objectName := range []string{"dir/object", "dir/dir/object", "object"} {
putInput := &s3.PutObjectInput{
Body: aws.ReadSeekCloser(strings.NewReader("my content 1")),
Bucket: aws.String(bucket),
Key: aws.String(objectName),
}
_, err = s3Client.PutObject(putInput)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal()
return
}
}
type objectResult struct {
name string
isLatest bool
}
type listResult struct {
versions []objectResult
commonPrefixes []string
}
simplifyListingResult := func(out *s3.ListObjectVersionsOutput) (result listResult) {
for _, commonPrefix := range out.CommonPrefixes {
result.commonPrefixes = append(result.commonPrefixes, *commonPrefix.Prefix)
}
for _, version := range out.Versions {
result.versions = append(result.versions, objectResult{name: *version.Key, isLatest: *version.IsLatest})
}
return
}
// Recursive listing
input := &s3.ListObjectVersionsInput{
Bucket: aws.String(bucket),
}
result, err := s3Client.ListObjectVersions(input)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal()
return
}
gotResult := simplifyListingResult(result)
expectedResult := listResult{
versions: []objectResult{
objectResult{name: "dir/dir/object", isLatest: true},
objectResult{name: "dir/object", isLatest: true},
objectResult{name: "object", isLatest: true},
}}
if !reflect.DeepEqual(gotResult, expectedResult) {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected listing result", nil).Fatal()
return
}
// Listing with delimiter
input = &s3.ListObjectVersionsInput{
Bucket: aws.String(bucket),
Delimiter: aws.String("/"),
}
result, err = s3Client.ListObjectVersions(input)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal()
return
}
gotResult = simplifyListingResult(result)
expectedResult = listResult{
versions: []objectResult{
objectResult{name: "object", isLatest: true},
},
commonPrefixes: []string{"dir/"}}
if !reflect.DeepEqual(gotResult, expectedResult) {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected listing result", nil).Fatal()
return
}
// Listing with prefix and delimiter
input = &s3.ListObjectVersionsInput{
Bucket: aws.String(bucket),
Delimiter: aws.String("/"),
Prefix: aws.String("dir/"),
}
result, err = s3Client.ListObjectVersions(input)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal()
return
}
gotResult = simplifyListingResult(result)
expectedResult = listResult{
versions: []objectResult{
objectResult{name: "dir/object", isLatest: true},
},
commonPrefixes: []string{"dir/dir/"}}
if !reflect.DeepEqual(gotResult, expectedResult) {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected listing result", nil).Fatal()
return
}
successLogger(function, args, startTime).Info()
}
// Test if key marker continuation works in listing works well
func testListObjectVersionsKeysContinuation() {
startTime := time.Now()
function := "testListObjectKeysContinuation"
bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-")
object := "testObject"
expiry := 1 * time.Minute
args := map[string]interface{}{
"bucketName": bucket,
"objectName": object,
"expiry": expiry,
}
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
return
}
defer cleanupBucket(bucket, function, args, startTime)
putVersioningInput := &s3.PutBucketVersioningInput{
Bucket: aws.String(bucket),
VersioningConfiguration: &s3.VersioningConfiguration{
Status: aws.String("Enabled"),
},
}
_, err = s3Client.PutBucketVersioning(putVersioningInput)
if err != nil {
if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") {
ignoreLog(function, args, startTime, "Versioning is not implemented").Info()
return
}
failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal()
return
}
for i := 0; i < 10; i++ {
putInput1 := &s3.PutObjectInput{
Body: aws.ReadSeekCloser(strings.NewReader("my content 1")),
Bucket: aws.String(bucket),
Key: aws.String(fmt.Sprintf("testobject-%d", i)),
}
_, err = s3Client.PutObject(putInput1)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal()
return
}
}
input := &s3.ListObjectVersionsInput{
Bucket: aws.String(bucket),
MaxKeys: aws.Int64(5),
}
type resultPage struct {
versions []string
nextKeyMarker string
lastPage bool
}
var gotResult []resultPage
var numPages int
err = s3Client.ListObjectVersionsPages(input,
func(page *s3.ListObjectVersionsOutput, lastPage bool) bool {
numPages++
resultPage := resultPage{lastPage: lastPage}
if page.NextKeyMarker != nil {
resultPage.nextKeyMarker = *page.NextKeyMarker
}
for _, v := range page.Versions {
resultPage.versions = append(resultPage.versions, *v.Key)
}
gotResult = append(gotResult, resultPage)
return true
})
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal()
return
}
if numPages != 2 {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected number of pages", nil).Fatal()
return
}
expectedResult := []resultPage{
resultPage{versions: []string{"testobject-0", "testobject-1", "testobject-2", "testobject-3", "testobject-4"}, nextKeyMarker: "testobject-4", lastPage: false},
resultPage{versions: []string{"testobject-5", "testobject-6", "testobject-7", "testobject-8", "testobject-9"}, nextKeyMarker: "", lastPage: true},
}
if !reflect.DeepEqual(expectedResult, gotResult) {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected listing result", nil).Fatal()
return
}
successLogger(function, args, startTime).Info()
}
// Test if version id marker continuation works in listing works well
func testListObjectVersionsVersionIDContinuation() {
startTime := time.Now()
function := "testListObjectVersionIDContinuation"
bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-")
object := "testObject"
expiry := 1 * time.Minute
args := map[string]interface{}{
"bucketName": bucket,
"objectName": object,
"expiry": expiry,
}
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
return
}
defer cleanupBucket(bucket, function, args, startTime)
putVersioningInput := &s3.PutBucketVersioningInput{
Bucket: aws.String(bucket),
VersioningConfiguration: &s3.VersioningConfiguration{
Status: aws.String("Enabled"),
},
}
_, err = s3Client.PutBucketVersioning(putVersioningInput)
if err != nil {
if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") {
ignoreLog(function, args, startTime, "Versioning is not implemented").Info()
return
}
failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal()
return
}
for i := 0; i < 10; i++ {
putInput1 := &s3.PutObjectInput{
Body: aws.ReadSeekCloser(strings.NewReader("my content 1")),
Bucket: aws.String(bucket),
Key: aws.String("testobject"),
}
_, err = s3Client.PutObject(putInput1)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal()
return
}
}
input := &s3.ListObjectVersionsInput{
Bucket: aws.String(bucket),
MaxKeys: aws.Int64(5),
}
type resultPage struct {
versions []string
nextVersionIDMarker string
lastPage bool
}
var gotResult []resultPage
var gotNextVersionIDMarker string
var numPages int
err = s3Client.ListObjectVersionsPages(input,
func(page *s3.ListObjectVersionsOutput, lastPage bool) bool {
numPages++
resultPage := resultPage{lastPage: lastPage}
if page.NextVersionIdMarker != nil {
resultPage.nextVersionIDMarker = *page.NextVersionIdMarker
}
for _, v := range page.Versions {
resultPage.versions = append(resultPage.versions, *v.Key)
}
if !lastPage {
// There is only two pages, so here we are saving the version id
// of the last element in the first page of listing
gotNextVersionIDMarker = *(*page.Versions[len(page.Versions)-1]).VersionId
}
gotResult = append(gotResult, resultPage)
return true
})
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal()
return
}
if numPages != 2 {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected number of pages", nil).Fatal()
return
}
expectedResult := []resultPage{
resultPage{versions: []string{"testobject", "testobject", "testobject", "testobject", "testobject"}, nextVersionIDMarker: gotNextVersionIDMarker, lastPage: false},
resultPage{versions: []string{"testobject", "testobject", "testobject", "testobject", "testobject"}, lastPage: true},
}
if !reflect.DeepEqual(expectedResult, gotResult) {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected listing result", nil).Fatal()
return
}
successLogger(function, args, startTime).Info()
}
// Test listing object when there is some empty directory object
func testListObjectsVersionsWithEmptyDirObject() {
startTime := time.Now()
function := "testListObjectsVersionsWithEmptyDirObject"
bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-")
object := "testObject"
expiry := 1 * time.Minute
args := map[string]interface{}{
"bucketName": bucket,
"objectName": object,
"expiry": expiry,
}
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
return
}
defer cleanupBucket(bucket, function, args, startTime)
putVersioningInput := &s3.PutBucketVersioningInput{
Bucket: aws.String(bucket),
VersioningConfiguration: &s3.VersioningConfiguration{
Status: aws.String("Enabled"),
},
}
_, err = s3Client.PutBucketVersioning(putVersioningInput)
if err != nil {
if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") {
ignoreLog(function, args, startTime, "Versioning is not implemented").Info()
return
}
failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal()
return
}
for _, objectName := range []string{"dir/object", "dir/"} {
putInput := &s3.PutObjectInput{
Body: aws.ReadSeekCloser(strings.NewReader("")),
Bucket: aws.String(bucket),
Key: aws.String(objectName),
}
_, err = s3Client.PutObject(putInput)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal()
return
}
}
type objectResult struct {
name string
etag string
isLatest bool
}
type listResult struct {
versions []objectResult
commonPrefixes []string
}
simplifyListingResult := func(out *s3.ListObjectVersionsOutput) (result listResult) {
for _, commonPrefix := range out.CommonPrefixes {
result.commonPrefixes = append(result.commonPrefixes, *commonPrefix.Prefix)
}
for _, version := range out.Versions {
result.versions = append(result.versions, objectResult{
name: *version.Key,
etag: *version.ETag,
isLatest: *version.IsLatest,
})
}
return
}
// Recursive listing
input := &s3.ListObjectVersionsInput{
Bucket: aws.String(bucket),
}
result, err := s3Client.ListObjectVersions(input)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal()
return
}
gotResult := simplifyListingResult(result)
expectedResult := listResult{
versions: []objectResult{
objectResult{name: "dir/", etag: "\"d41d8cd98f00b204e9800998ecf8427e\"", isLatest: true},
objectResult{name: "dir/object", etag: "\"d41d8cd98f00b204e9800998ecf8427e\"", isLatest: true},
}}
if !reflect.DeepEqual(gotResult, expectedResult) {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected listing result", nil).Fatal()
return
}
// Listing with delimiter
input = &s3.ListObjectVersionsInput{
Bucket: aws.String(bucket),
Delimiter: aws.String("/"),
}
result, err = s3Client.ListObjectVersions(input)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal()
return
}
gotResult = simplifyListingResult(result)
expectedResult = listResult{
commonPrefixes: []string{"dir/"}}
if !reflect.DeepEqual(gotResult, expectedResult) {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected listing result", nil).Fatal()
return
}
// Listing with prefix and delimiter
input = &s3.ListObjectVersionsInput{
Bucket: aws.String(bucket),
Delimiter: aws.String("/"),
Prefix: aws.String("dir/"),
}
result, err = s3Client.ListObjectVersions(input)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal()
return
}
gotResult = simplifyListingResult(result)
expectedResult = listResult{
versions: []objectResult{
{name: "dir/", etag: "\"d41d8cd98f00b204e9800998ecf8427e\"", isLatest: true},
{name: "dir/object", etag: "\"d41d8cd98f00b204e9800998ecf8427e\"", isLatest: true},
},
}
if !reflect.DeepEqual(gotResult, expectedResult) {
failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected listing result", nil).Fatal()
return
}
successLogger(function, args, startTime).Info()
}

122
mint/build/versioning/main.go Normal file
View File

@ -0,0 +1,122 @@
// Mint, (C) 2021 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
//
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"os"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
log "github.com/sirupsen/logrus"
)
// S3 client for testing
var s3Client *s3.S3
func cleanupBucket(bucket string, function string, args map[string]interface{}, startTime time.Time) {
input := &s3.ListObjectVersionsInput{
Bucket: aws.String(bucket),
}
err := s3Client.ListObjectVersionsPages(input,
func(page *s3.ListObjectVersionsOutput, lastPage bool) bool {
for _, v := range page.Versions {
input := &s3.DeleteObjectInput{
Bucket: &bucket,
Key: v.Key,
VersionId: v.VersionId,
BypassGovernanceRetention: aws.Bool(true),
}
_, err := s3Client.DeleteObject(input)
if err != nil {
log.Fatalln("cleanupBucket:", err)
return true
}
}
for _, v := range page.DeleteMarkers {
input := &s3.DeleteObjectInput{
Bucket: &bucket,
Key: v.Key,
VersionId: v.VersionId,
}
_, err := s3Client.DeleteObject(input)
if err != nil {
log.Fatalln("cleanupBucket:", err)
return true
}
}
return true
})
_, err = s3Client.DeleteBucket(&s3.DeleteBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
failureLog(function, args, startTime, "", "Cleanup bucket Failed", err).Fatal()
return
}
}
func main() {
endpoint := os.Getenv("SERVER_ENDPOINT")
accessKey := os.Getenv("ACCESS_KEY")
secretKey := os.Getenv("SECRET_KEY")
secure := os.Getenv("ENABLE_HTTPS")
sdkEndpoint := "http://" + endpoint
if secure == "1" {
sdkEndpoint = "https://" + endpoint
}
creds := credentials.NewStaticCredentials(accessKey, secretKey, "")
newSession := session.New()
s3Config := &aws.Config{
Credentials: creds,
Endpoint: aws.String(sdkEndpoint),
Region: aws.String("us-east-1"),
S3ForcePathStyle: aws.Bool(true),
}
// Create an S3 service object in the default region.
s3Client = s3.New(newSession, s3Config)
// Output to stdout instead of the default stderr
log.SetOutput(os.Stdout)
// create custom formatter
mintFormatter := mintJSONFormatter{}
// set custom formatter
log.SetFormatter(&mintFormatter)
// log Info or above -- success cases are Info level, failures are Fatal level
log.SetLevel(log.InfoLevel)
testMakeBucket()
testPutObject()
testPutObjectWithTaggingAndMetadata()
testGetObject()
testStatObject()
testDeleteObject()
testListObjectVersionsSimple()
testListObjectVersionsWithPrefixAndDelimiter()
testListObjectVersionsKeysContinuation()
testListObjectVersionsVersionIDContinuation()
testListObjectsVersionsWithEmptyDirObject()
testTagging()
testLockingLegalhold()
testLockingRetentionGovernance()
testLockingRetentionCompliance()
}

294
mint/build/versioning/put.go Normal file
View File

@ -0,0 +1,294 @@
/*
*
* Mint, (C) 2021 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package main
import (
"errors"
"fmt"
"math/rand"
"net/url"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
)
// Put two objects with the same name but with different content
func testPutObject() {
startTime := time.Now()
function := "testPutObject"
bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-")
object := "testObject"
expiry := 1 * time.Minute
args := map[string]interface{}{
"bucketName": bucket,
"objectName": object,
"expiry": expiry,
}
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
return
}
defer cleanupBucket(bucket, function, args, startTime)
putVersioningInput := &s3.PutBucketVersioningInput{
Bucket: aws.String(bucket),
VersioningConfiguration: &s3.VersioningConfiguration{
Status: aws.String("Enabled"),
},
}
_, err = s3Client.PutBucketVersioning(putVersioningInput)
if err != nil {
if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") {
ignoreLog(function, args, startTime, "Versioning is not implemented").Info()
return
}
failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal()
return
}
putInput1 := &s3.PutObjectInput{
Body: aws.ReadSeekCloser(strings.NewReader("my content 1")),
Bucket: aws.String(bucket),
Key: aws.String(object),
}
_, err = s3Client.PutObject(putInput1)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal()
return
}
putInput2 := &s3.PutObjectInput{
Body: aws.ReadSeekCloser(strings.NewReader("content file 2")),
Bucket: aws.String(bucket),
Key: aws.String(object),
}
_, err = s3Client.PutObject(putInput2)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal()
return
}
input := &s3.ListObjectVersionsInput{
Bucket: aws.String(bucket),
}
result, err := s3Client.ListObjectVersions(input)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal()
return
}
if len(result.Versions) != 2 {
failureLog(function, args, startTime, "", "Unexpected list content", errors.New("unexpected number of versions")).Fatal()
return
}
vid1 := *result.Versions[0]
vid2 := *result.Versions[1]
if *vid1.VersionId == "" || *vid2.VersionId == "" || *vid1.VersionId == *vid2.VersionId {
failureLog(function, args, startTime, "", "Unexpected list content", errors.New("unexpected VersionId field")).Fatal()
return
}
if *vid1.IsLatest == false || *vid2.IsLatest == true {
failureLog(function, args, startTime, "", "Unexpected list content", errors.New("unexpected IsLatest field")).Fatal()
return
}
if *vid1.Size != 14 || *vid2.Size != 12 {
failureLog(function, args, startTime, "", "Unexpected list content", errors.New("unexpected Size field")).Fatal()
return
}
if *vid1.ETag != "\"e847032b45d3d76230058a80d8ca909b\"" || *vid2.ETag != "\"094459df8fcebffc70d9aa08d75f9944\"" {
failureLog(function, args, startTime, "", "Unexpected list content", errors.New("unexpected ETag field")).Fatal()
return
}
if *vid1.Key != "testObject" || *vid2.Key != "testObject" {
failureLog(function, args, startTime, "", "Unexpected list content", errors.New("unexpected Key field")).Fatal()
return
}
if (*vid1.LastModified).Before(*vid2.LastModified) {
failureLog(function, args, startTime, "", "Unexpected list content", errors.New("unexpected Last modified field")).Fatal()
return
}
successLogger(function, args, startTime).Info()
}
// Upload object versions with tagging and metadata and check them
func testPutObjectWithTaggingAndMetadata() {
startTime := time.Now()
function := "testPutObjectWithTaggingAndMetadata"
bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-")
object := "testObject"
expiry := 1 * time.Minute
args := map[string]interface{}{
"bucketName": bucket,
"objectName": object,
"expiry": expiry,
}
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
return
}
defer cleanupBucket(bucket, function, args, startTime)
putVersioningInput := &s3.PutBucketVersioningInput{
Bucket: aws.String(bucket),
VersioningConfiguration: &s3.VersioningConfiguration{
Status: aws.String("Enabled"),
},
}
_, err = s3Client.PutBucketVersioning(putVersioningInput)
if err != nil {
if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") {
ignoreLog(function, args, startTime, "Versioning is not implemented").Info()
return
}
failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal()
return
}
type objectUpload struct {
tags string
metadata map[string]string
versionId string
}
uploads := []objectUpload{
{tags: "key=value"},
{},
{metadata: map[string]string{"My-Metadata-Key": "my-metadata-val"}},
{tags: "key1=value1&key2=value2", metadata: map[string]string{"Foo-Key": "foo-val"}},
}
for i := range uploads {
putInput := &s3.PutObjectInput{
Body: aws.ReadSeekCloser(strings.NewReader("foocontent")),
Bucket: aws.String(bucket),
Key: aws.String(object),
}
if uploads[i].tags != "" {
putInput.Tagging = aws.String(uploads[i].tags)
}
if uploads[i].metadata != nil {
putInput.Metadata = make(map[string]*string)
for k, v := range uploads[i].metadata {
putInput.Metadata[k] = aws.String(v)
}
}
result, err := s3Client.PutObject(putInput)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("PUT object expected to succeed but got %v", err), err).Fatal()
return
}
uploads[i].versionId = *result.VersionId
}
for i := range uploads {
putInput := &s3.PutObjectInput{
Body: aws.ReadSeekCloser(strings.NewReader("foocontent")),
Bucket: aws.String(bucket),
Key: aws.String(object),
}
if uploads[i].tags != "" {
putInput.Tagging = aws.String(uploads[i].tags)
}
if uploads[i].metadata != nil {
putInput.Metadata = make(map[string]*string)
for k, v := range uploads[i].metadata {
putInput.Metadata[k] = aws.String(v)
}
}
result, err := s3Client.PutObject(putInput)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("PUT object expected to succeed but got %v", err), err).Fatal()
return
}
uploads[i].versionId = *result.VersionId
}
// Check for tagging after removal
for i := range uploads {
if uploads[i].tags != "" {
input := &s3.GetObjectTaggingInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
VersionId: aws.String(uploads[i].versionId),
}
tagResult, err := s3Client.GetObjectTagging(input)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("GET Object tagging expected to succeed but got %v", err), err).Fatal()
return
}
var vals = make(url.Values)
for _, tag := range tagResult.TagSet {
vals.Add(*tag.Key, *tag.Value)
}
if uploads[i].tags != vals.Encode() {
failureLog(function, args, startTime, "", "PUT Object with tagging header returned unexpected result", nil).Fatal()
return
}
}
if uploads[i].metadata != nil {
input := &s3.HeadObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
VersionId: aws.String(uploads[i].versionId),
}
result, err := s3Client.HeadObject(input)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("HEAD Object expected to succeed but got %v", err), err).Fatal()
return
}
for expectedKey, expectedVal := range uploads[i].metadata {
gotValue, ok := result.Metadata[expectedKey]
if !ok {
failureLog(function, args, startTime, "", "HEAD Object returned unexpected metadata key result", nil).Fatal()
return
}
if expectedVal != *gotValue {
failureLog(function, args, startTime, "", "HEAD Object returned unexpected metadata value result", nil).Fatal()
return
}
}
}
}
successLogger(function, args, startTime).Info()
}

277
mint/build/versioning/retention.go Normal file
View File

@ -0,0 +1,277 @@
/*
*
* Mint, (C) 2021 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package main
import (
"fmt"
"math/rand"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
)
// Test locking retention governance
func testLockingRetentionGovernance() {
startTime := time.Now()
function := "testLockingRetentionGovernance"
bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-")
object := "testObject"
expiry := 1 * time.Minute
args := map[string]interface{}{
"bucketName": bucket,
"objectName": object,
"expiry": expiry,
}
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucket),
ObjectLockEnabledForBucket: aws.Bool(true),
})
if err != nil {
if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") {
ignoreLog(function, args, startTime, "Versioning is not implemented").Info()
return
}
failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
return
}
defer cleanupBucket(bucket, function, args, startTime)
type uploadedObject struct {
retention string
retentionUntil time.Time
successfulRemove bool
versionId string
deleteMarker bool
}
uploads := []uploadedObject{
{},
{retention: "GOVERNANCE", retentionUntil: time.Now().UTC().Add(time.Hour)},
{},
}
// Upload versions and save their version IDs
for i := range uploads {
putInput := &s3.PutObjectInput{
Body: aws.ReadSeekCloser(strings.NewReader("content")),
Bucket: aws.String(bucket),
Key: aws.String(object),
}
if uploads[i].retention != "" {
putInput.ObjectLockMode = aws.String(uploads[i].retention)
putInput.ObjectLockRetainUntilDate = aws.Time(uploads[i].retentionUntil)
}
output, err := s3Client.PutObject(putInput)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal()
return
}
uploads[i].versionId = *output.VersionId
}
// In all cases, we can remove an object by creating a delete marker
// First delete without version ID
deleteInput := &s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
}
deleteOutput, err := s3Client.DeleteObject(deleteInput)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("DELETE expected to succeed but got %v", err), err).Fatal()
return
}
uploads = append(uploads, uploadedObject{versionId: *deleteOutput.VersionId, deleteMarker: true})
// Put tagging on each version
for i := range uploads {
if uploads[i].deleteMarker {
continue
}
deleteInput := &s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
VersionId: aws.String(uploads[i].versionId),
}
_, err = s3Client.DeleteObject(deleteInput)
if err == nil && uploads[i].retention != "" {
failureLog(function, args, startTime, "", "DELETE expected to fail but succeed instead", nil).Fatal()
return
}
if err != nil && uploads[i].retention == "" {
failureLog(function, args, startTime, "", fmt.Sprintf("DELETE expected to succeed but got %v", err), err).Fatal()
return
}
}
successLogger(function, args, startTime).Info()
}
// Test locking retention compliance
func testLockingRetentionCompliance() {
startTime := time.Now()
function := "testLockingRetentionCompliance"
bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-")
object := "testObject"
expiry := 1 * time.Minute
args := map[string]interface{}{
"bucketName": bucket,
"objectName": object,
"expiry": expiry,
}
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucket),
ObjectLockEnabledForBucket: aws.Bool(true),
})
if err != nil {
if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") {
ignoreLog(function, args, startTime, "Versioning is not implemented").Info()
return
}
failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
return
}
defer func() {
start := time.Now()
input := &s3.ListObjectVersionsInput{
Bucket: aws.String(bucket),
}
for time.Since(start) < 30*time.Minute {
err := s3Client.ListObjectVersionsPages(input,
func(page *s3.ListObjectVersionsOutput, lastPage bool) bool {
for _, v := range page.Versions {
input := &s3.DeleteObjectInput{
Bucket: &bucket,
Key: v.Key,
VersionId: v.VersionId,
}
_, err := s3Client.DeleteObject(input)
if err != nil {
return true
}
}
for _, v := range page.DeleteMarkers {
input := &s3.DeleteObjectInput{
Bucket: &bucket,
Key: v.Key,
VersionId: v.VersionId,
}
_, err := s3Client.DeleteObject(input)
if err != nil {
return true
}
}
return true
})
_, err = s3Client.DeleteBucket(&s3.DeleteBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
time.Sleep(30 * time.Second)
continue
}
return
}
failureLog(function, args, startTime, "", "Unable to cleanup bucket after compliance tests", nil).Fatal()
return
}()
type uploadedObject struct {
retention string
retentionUntil time.Time
successfulRemove bool
versionId string
deleteMarker bool
}
uploads := []uploadedObject{
{},
{retention: "COMPLIANCE", retentionUntil: time.Now().UTC().Add(time.Minute)},
{},
}
// Upload versions and save their version IDs
for i := range uploads {
putInput := &s3.PutObjectInput{
Body: aws.ReadSeekCloser(strings.NewReader("content")),
Bucket: aws.String(bucket),
Key: aws.String(object),
}
if uploads[i].retention != "" {
putInput.ObjectLockMode = aws.String(uploads[i].retention)
putInput.ObjectLockRetainUntilDate = aws.Time(uploads[i].retentionUntil)
}
output, err := s3Client.PutObject(putInput)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal()
return
}
uploads[i].versionId = *output.VersionId
}
// In all cases, we can remove an object by creating a delete marker
// First delete without version ID
deleteInput := &s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
}
deleteOutput, err := s3Client.DeleteObject(deleteInput)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("DELETE expected to succeed but got %v", err), err).Fatal()
return
}
uploads = append(uploads, uploadedObject{versionId: *deleteOutput.VersionId, deleteMarker: true})
// Put tagging on each version
for i := range uploads {
if uploads[i].deleteMarker {
continue
}
deleteInput := &s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
VersionId: aws.String(uploads[i].versionId),
}
_, err = s3Client.DeleteObject(deleteInput)
if err == nil && uploads[i].retention != "" {
failureLog(function, args, startTime, "", "DELETE expected to fail but succeed instead", nil).Fatal()
return
}
if err != nil && uploads[i].retention == "" {
failureLog(function, args, startTime, "", fmt.Sprintf("DELETE expected to succeed but got %v", err), err).Fatal()
return
}
}
successLogger(function, args, startTime).Info()
}

183
mint/build/versioning/stat.go Normal file
View File

@ -0,0 +1,183 @@
/*
*
* Mint, (C) 2021 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package main
import (
"fmt"
"math/rand"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/s3"
)
func testStatObject() {
startTime := time.Now()
function := "testStatObject"
bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-")
object := "testObject"
expiry := 1 * time.Minute
args := map[string]interface{}{
"bucketName": bucket,
"objectName": object,
"expiry": expiry,
}
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
return
}
defer cleanupBucket(bucket, function, args, startTime)
putVersioningInput := &s3.PutBucketVersioningInput{
Bucket: aws.String(bucket),
VersioningConfiguration: &s3.VersioningConfiguration{
Status: aws.String("Enabled"),
},
}
_, err = s3Client.PutBucketVersioning(putVersioningInput)
if err != nil {
if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") {
ignoreLog(function, args, startTime, "Versioning is not implemented").Info()
return
}
failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal()
return
}
putInput1 := &s3.PutObjectInput{
Body: aws.ReadSeekCloser(strings.NewReader("my content 1")),
Bucket: aws.String(bucket),
Key: aws.String(object),
}
_, err = s3Client.PutObject(putInput1)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal()
return
}
putInput2 := &s3.PutObjectInput{
Body: aws.ReadSeekCloser(strings.NewReader("content file 2")),
Bucket: aws.String(bucket),
Key: aws.String(object),
}
_, err = s3Client.PutObject(putInput2)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal()
return
}
deleteInput := &s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
}
_, err = s3Client.DeleteObject(deleteInput)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("Delete expected to succeed but got %v", err), err).Fatal()
return
}
input := &s3.ListObjectVersionsInput{
Bucket: aws.String(bucket),
}
result, err := s3Client.ListObjectVersions(input)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal()
return
}
testCases := []struct {
size int64
versionId string
etag string
contentType string
deleteMarker bool
}{
{0, *(*result.DeleteMarkers[0]).VersionId, "", "", true},
{14, *(*result.Versions[0]).VersionId, "\"e847032b45d3d76230058a80d8ca909b\"", "binary/octet-stream", false},
{12, *(*result.Versions[1]).VersionId, "\"094459df8fcebffc70d9aa08d75f9944\"", "binary/octet-stream", false},
}
for i, testCase := range testCases {
headInput := &s3.HeadObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
VersionId: aws.String(testCase.versionId),
}
result, err := s3Client.HeadObject(headInput)
if testCase.deleteMarker && err == nil {
failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) expected to fail but succeeded", i+1), nil).Fatal()
return
}
if !testCase.deleteMarker && err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) expected to succeed but failed", i+1), err).Fatal()
return
}
if testCase.deleteMarker {
aerr, ok := err.(awserr.Error)
if !ok {
failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) unexpected error with delete marker", i+1), err).Fatal()
return
}
if aerr.Code() != "MethodNotAllowed" {
failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) unexpected error code with delete marker", i+1), err).Fatal()
return
}
continue
}
if *result.ContentLength != testCase.size {
failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) unexpected Content-Length", i+1), err).Fatal()
return
}
if *result.ETag != testCase.etag {
failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) unexpected ETag", i+1), err).Fatal()
return
}
if *result.ContentType != testCase.contentType {
failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) unexpected Content-Type", i+1), err).Fatal()
return
}
if result.DeleteMarker != nil && *result.DeleteMarker {
failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) unexpected DeleteMarker", i+1), err).Fatal()
return
}
if time.Since(*result.LastModified) > time.Hour {
failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) unexpected LastModified", i+1), err).Fatal()
return
}
}
successLogger(function, args, startTime).Info()
}

203
mint/build/versioning/tagging.go Normal file
View File

@ -0,0 +1,203 @@
/*
*
* Mint, (C) 2021 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package main
import (
"fmt"
"math/rand"
"reflect"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
)
// Test PUT/GET/DELETE tagging for separate versions
func testTagging() {
startTime := time.Now()
function := "testTagging"
bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-")
object := "testObject"
expiry := 1 * time.Minute
args := map[string]interface{}{
"bucketName": bucket,
"objectName": object,
"expiry": expiry,
}
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
return
}
defer cleanupBucket(bucket, function, args, startTime)
putVersioningInput := &s3.PutBucketVersioningInput{
Bucket: aws.String(bucket),
VersioningConfiguration: &s3.VersioningConfiguration{
Status: aws.String("Enabled"),
},
}
_, err = s3Client.PutBucketVersioning(putVersioningInput)
if err != nil {
if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") {
ignoreLog(function, args, startTime, "Versioning is not implemented").Info()
return
}
failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal()
return
}
type uploadedObject struct {
content string
tagging []*s3.Tag
versionId string
deleteMarker bool
}
uploads := []uploadedObject{
{content: "my content 1", tagging: []*s3.Tag{{Key: aws.String("type"), Value: aws.String("text")}}},
{content: "content file 2"},
{content: "\"%32&é", tagging: []*s3.Tag{{Key: aws.String("type"), Value: aws.String("garbage")}}},
{deleteMarker: true},
}
// Upload versions and save their version IDs
for i := range uploads {
if uploads[i].deleteMarker {
// Delete the current object to create a delete marker)
deleteInput := &s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
}
deleteOutput, err := s3Client.DeleteObject(deleteInput)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("DELETE object expected to succeed but got %v", err), err).Fatal()
return
}
uploads[i].versionId = *deleteOutput.VersionId
continue
}
putInput := &s3.PutObjectInput{
Body: aws.ReadSeekCloser(strings.NewReader(uploads[i].content)),
Bucket: aws.String(bucket),
Key: aws.String(object),
}
output, err := s3Client.PutObject(putInput)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal()
return
}
uploads[i].versionId = *output.VersionId
}
// Put tagging on each version
for i := range uploads {
if uploads[i].tagging == nil {
continue
}
putTaggingInput := &s3.PutObjectTaggingInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
Tagging: &s3.Tagging{TagSet: uploads[i].tagging},
VersionId: aws.String(uploads[i].versionId),
}
_, err = s3Client.PutObjectTagging(putTaggingInput)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("PUT Object tagging expected to succeed but got %v", err), err).Fatal()
return
}
}
// Check versions tagging
for i := range uploads {
input := &s3.GetObjectTaggingInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
VersionId: aws.String(uploads[i].versionId),
}
result, err := s3Client.GetObjectTagging(input)
if err == nil && uploads[i].deleteMarker {
failureLog(function, args, startTime, "", "GET Object tagging expected to fail with delete marker but succeded", err).Fatal()
return
}
if err != nil && !uploads[i].deleteMarker {
failureLog(function, args, startTime, "", fmt.Sprintf("GET Object tagging expected to succeed but got %v", err), err).Fatal()
return
}
if uploads[i].deleteMarker {
continue
}
if !reflect.DeepEqual(result.TagSet, uploads[i].tagging) {
failureLog(function, args, startTime, "", "GET Object tagging returned unexpected result", nil).Fatal()
return
}
}
// Remove all tagging for all objects
for i := range uploads {
input := &s3.DeleteObjectTaggingInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
VersionId: aws.String(uploads[i].versionId),
}
_, err := s3Client.DeleteObjectTagging(input)
if err == nil && uploads[i].deleteMarker {
failureLog(function, args, startTime, "", "DELETE Object tagging expected to fail with delete marker but succeded", err).Fatal()
return
}
if err != nil && !uploads[i].deleteMarker {
failureLog(function, args, startTime, "", fmt.Sprintf("GET Object tagging expected to succeed but got %v", err), err).Fatal()
return
}
}
// Check for tagging after removal
for i := range uploads {
if uploads[i].deleteMarker {
// Avoid testing this use case since already tested earlier
continue
}
input := &s3.GetObjectTaggingInput{
Bucket: aws.String(bucket),
Key: aws.String(object),
VersionId: aws.String(uploads[i].versionId),
}
result, err := s3Client.GetObjectTagging(input)
if err != nil {
failureLog(function, args, startTime, "", fmt.Sprintf("GET Object tagging expected to succeed but got %v", err), err).Fatal()
return
}
var nilTagSet []*s3.Tag
if !reflect.DeepEqual(result.TagSet, nilTagSet) {
failureLog(function, args, startTime, "", "GET Object tagging after DELETE returned unexpected result", nil).Fatal()
return
}
}
successLogger(function, args, startTime).Info()
}

137
mint/build/versioning/utils.go Normal file
View File

@ -0,0 +1,137 @@
/*
*
* Mint, (C) 2021 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package main
import (
"encoding/json"
"encoding/xml"
"fmt"
"math/rand"
"net/http"
"strings"
"time"
log "github.com/sirupsen/logrus"
)
const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569"
const (
letterIdxBits = 6 // 6 bits to represent a letter index
letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits
)
// different kinds of test failures
const (
PASS = "PASS" // Indicate that a test passed
FAIL = "FAIL" // Indicate that a test failed
)
type errorResponse struct {
XMLName xml.Name `xml:"Error" json:"-"`
Code string
Message string
BucketName string
Key string
RequestID string `xml:"RequestId"`
HostID string `xml:"HostId"`
// Region where the bucket is located. This header is returned
// only in HEAD bucket and ListObjects response.
Region string
// Headers of the returned S3 XML error
Headers http.Header `xml:"-" json:"-"`
}
type mintJSONFormatter struct {
}
func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) {
data := make(log.Fields, len(entry.Data))
for k, v := range entry.Data {
switch v := v.(type) {
case error:
// Otherwise errors are ignored by `encoding/json`
// https://github.com/sirupsen/logrus/issues/137
data[k] = v.Error()
default:
data[k] = v
}
}
serialized, err := json.Marshal(data)
if err != nil {
return nil, fmt.Errorf("Failed to marshal fields to JSON, %w", err)
}
return append(serialized, '\n'), nil
}
// log successful test runs
func successLogger(function string, args map[string]interface{}, startTime time.Time) *log.Entry {
// calculate the test case duration
duration := time.Since(startTime)
// log with the fields as per mint
fields := log.Fields{"name": "versioning", "function": function, "args": args, "duration": duration.Nanoseconds() / 1000000, "status": PASS}
return log.WithFields(fields)
}
// log not applicable test runs
func ignoreLog(function string, args map[string]interface{}, startTime time.Time, alert string) *log.Entry {
// calculate the test case duration
duration := time.Since(startTime)
// log with the fields as per mint
fields := log.Fields{"name": "versioning", "function": function, "args": args,
"duration": duration.Nanoseconds() / 1000000, "status": "NA", "alert": strings.Split(alert, " ")[0] + " is NotImplemented"}
return log.WithFields(fields)
}
// log failed test runs
func failureLog(function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry {
// calculate the test case duration
duration := time.Since(startTime)
var fields log.Fields
// log with the fields as per mint
if err != nil {
fields = log.Fields{"name": "versioning", "function": function, "args": args,
"duration": duration.Nanoseconds() / 1000000, "status": FAIL, "alert": alert, "message": message, "error": err}
} else {
fields = log.Fields{"name": "versioning", "function": function, "args": args,
"duration": duration.Nanoseconds() / 1000000, "status": FAIL, "alert": alert, "message": message}
}
return log.WithFields(fields)
}
func randString(n int, src rand.Source, prefix string) string {
b := make([]byte, n)
// A rand.Int63() generates 63 random bits, enough for letterIdxMax letters!
for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
if remain == 0 {
cache, remain = src.Int63(), letterIdxMax
}
if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
b[i] = letterBytes[idx]
i--
}
cache >>= letterIdxBits
remain--
}
return prefix + string(b[0:30-len(prefix)])
}

1
mint/release.sh
View File

@ -17,6 +17,7 @@
export MINT_ROOT_DIR=${MINT_ROOT_DIR:-/mint} export MINT_ROOT_DIR=${MINT_ROOT_DIR:-/mint}
export MINT_RUN_CORE_DIR="$MINT_ROOT_DIR/run/core" export MINT_RUN_CORE_DIR="$MINT_ROOT_DIR/run/core"
export MINT_RUN_BUILD_DIR="$MINT_ROOT_DIR/build"
export MINT_RUN_SECURITY_DIR="$MINT_ROOT_DIR/run/security" export MINT_RUN_SECURITY_DIR="$MINT_ROOT_DIR/run/security"
export WGET="wget --quiet --no-check-certificate" export WGET="wget --quiet --no-check-certificate"

28
mint/run/core/versioning/run.sh Executable file
View File

@ -0,0 +1,28 @@
#!/bin/bash
#
# Mint (C) 2021 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# handle command line arguments
if [ $# -ne 2 ]; then
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
exit 1
fi
output_log_file="$1"
error_log_file="$2"
# run tests
/mint/run/core/versioning/tests 1>>"$output_log_file" 2>"$error_log_file"