diff --git a/.github/workflows/replication.yaml b/.github/workflows/replication.yaml index 753ecf8b9..71d4e1465 100644 --- a/.github/workflows/replication.yaml +++ b/.github/workflows/replication.yaml @@ -40,6 +40,7 @@ jobs: sudo sysctl net.ipv6.conf.all.disable_ipv6=0 sudo sysctl net.ipv6.conf.default.disable_ipv6=0 make test-ilm + make test-ilm-transition - name: Test PBAC run: | diff --git a/Makefile b/Makefile index a68987747..4d65c39c5 100644 --- a/Makefile +++ b/Makefile @@ -60,6 +60,10 @@ test-ilm: install-race @echo "Running ILM tests" @env bash $(PWD)/docs/bucket/replication/setup_ilm_expiry_replication.sh +test-ilm-transition: install-race + @echo "Running ILM tiering tests with healing" + @env bash $(PWD)/docs/bucket/lifecycle/setup_ilm_transition.sh + test-pbac: install-race @echo "Running bucket policies tests" @env bash $(PWD)/docs/iam/policies/pbac-tests.sh diff --git a/cmd/erasure-healing-common.go b/cmd/erasure-healing-common.go index 1e39c2c26..7c3c8a4d8 100644 --- a/cmd/erasure-healing-common.go +++ b/cmd/erasure-healing-common.go @@ -392,8 +392,8 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad if metaErrs[i] != nil { continue } - meta := partsMetadata[i] + meta := partsMetadata[i] if meta.Deleted || meta.IsRemote() { continue } @@ -442,13 +442,17 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad } for i, onlineDisk := range onlineDisks { - if metaErrs[i] == nil && !hasPartErr(dataErrsByDisk[i]) { - // All parts verified, mark it as all data available. - availableDisks[i] = onlineDisk - } else { - // upon errors just make that disk's fileinfo invalid - partsMetadata[i] = FileInfo{} + if metaErrs[i] == nil { + meta := partsMetadata[i] + if meta.Deleted || meta.IsRemote() || !hasPartErr(dataErrsByDisk[i]) { + // All parts verified, mark it as all data available. + availableDisks[i] = onlineDisk + continue + } } + + // upon errors just make that disk's fileinfo invalid + partsMetadata[i] = FileInfo{} } return diff --git a/cmd/erasure-metadata.go b/cmd/erasure-metadata.go index 885c42c0d..ce053a07d 100644 --- a/cmd/erasure-metadata.go +++ b/cmd/erasure-metadata.go @@ -514,8 +514,10 @@ func listObjectParities(partsMetadata []FileInfo, errs []error) (parities []int) if metadata.Deleted || metadata.Size == 0 { parities[index] = totalShards / 2 } else if metadata.TransitionStatus == lifecycle.TransitionComplete { - // For tiered objects, read quorum is N/2+1 to ensure simple majority on xl.meta. It is not equal to EcM because the data integrity is entrusted with the warm tier. - parities[index] = totalShards - (totalShards/2 + 1) + // For tiered objects, read quorum is N/2+1 to ensure simple majority on xl.meta. + // It is not equal to EcM because the data integrity is entrusted with the warm tier. + // However, we never go below EcM, in case of a EcM=EcN setup. + parities[index] = max(totalShards-(totalShards/2+1), metadata.Erasure.ParityBlocks) } else { parities[index] = metadata.Erasure.ParityBlocks } diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 8ee2ae6dd..80c819680 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -2376,7 +2376,11 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st }() var rv remoteVersionID - rv, err = tgtClient.Put(ctx, destObj, pr, fi.Size) + rv, err = tgtClient.PutWithMeta(ctx, destObj, pr, fi.Size, map[string]string{ + "name": object, // preserve the original name of the object on the remote tier object metadata. + // this is just for future reverse lookup() purposes (applies only for new objects) + // does not apply retro-actively on already transitioned objects. + }) pr.CloseWithError(err) if err != nil { traceFn(ILMTransition, nil, err) diff --git a/cmd/warm-backend-azure.go b/cmd/warm-backend-azure.go index c39f7f4d5..bb087e749 100644 --- a/cmd/warm-backend-azure.go +++ b/cmd/warm-backend-azure.go @@ -26,6 +26,7 @@ import ( "strings" "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" @@ -59,10 +60,15 @@ func (az *warmBackendAzure) getDest(object string) string { return destObj } -func (az *warmBackendAzure) Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) { +func (az *warmBackendAzure) PutWithMeta(ctx context.Context, object string, r io.Reader, length int64, meta map[string]string) (remoteVersionID, error) { + azMeta := map[string]*string{} + for k, v := range meta { + azMeta[k] = to.Ptr(v) + } resp, err := az.clnt.UploadStream(ctx, az.Bucket, az.getDest(object), io.LimitReader(r, length), &azblob.UploadStreamOptions{ Concurrency: 4, AccessTier: az.tier(), // set tier if specified + Metadata: azMeta, }) if err != nil { return "", azureToObjectError(err, az.Bucket, az.getDest(object)) @@ -74,6 +80,10 @@ func (az *warmBackendAzure) Put(ctx context.Context, object string, r io.Reader, return remoteVersionID(vid), nil } +func (az *warmBackendAzure) Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) { + return az.PutWithMeta(ctx, object, r, length, map[string]string{}) +} + func (az *warmBackendAzure) Get(ctx context.Context, object string, rv remoteVersionID, opts WarmBackendGetOpts) (r io.ReadCloser, err error) { if opts.startOffset < 0 { return nil, InvalidRange{} diff --git a/cmd/warm-backend-gcs.go b/cmd/warm-backend-gcs.go index 3246fd1c8..8cacb71c0 100644 --- a/cmd/warm-backend-gcs.go +++ b/cmd/warm-backend-gcs.go @@ -47,16 +47,17 @@ func (gcs *warmBackendGCS) getDest(object string) string { return destObj } -// FIXME: add support for remote version ID in GCS remote tier and remove this. -// Currently it's a no-op. - -func (gcs *warmBackendGCS) Put(ctx context.Context, key string, data io.Reader, length int64) (remoteVersionID, error) { +func (gcs *warmBackendGCS) PutWithMeta(ctx context.Context, key string, data io.Reader, length int64, meta map[string]string) (remoteVersionID, error) { object := gcs.client.Bucket(gcs.Bucket).Object(gcs.getDest(key)) - // TODO: set storage class w := object.NewWriter(ctx) if gcs.StorageClass != "" { w.ObjectAttrs.StorageClass = gcs.StorageClass } + w.ObjectAttrs.Metadata = meta + if _, err := xioutil.Copy(w, data); err != nil { + return "", gcsToObjectError(err, gcs.Bucket, key) + } + if _, err := xioutil.Copy(w, data); err != nil { return "", gcsToObjectError(err, gcs.Bucket, key) } @@ -64,6 +65,12 @@ func (gcs *warmBackendGCS) Put(ctx context.Context, key string, data io.Reader, return "", w.Close() } +// FIXME: add support for remote version ID in GCS remote tier and remove this. +// Currently it's a no-op. +func (gcs *warmBackendGCS) Put(ctx context.Context, key string, data io.Reader, length int64) (remoteVersionID, error) { + return gcs.PutWithMeta(ctx, key, data, length, map[string]string{}) +} + func (gcs *warmBackendGCS) Get(ctx context.Context, key string, rv remoteVersionID, opts WarmBackendGetOpts) (r io.ReadCloser, err error) { // GCS storage decompresses a gzipped object by default and returns the data. // Refer to https://cloud.google.com/storage/docs/transcoding#decompressive_transcoding diff --git a/cmd/warm-backend-minio.go b/cmd/warm-backend-minio.go index 94c09f226..005f7e341 100644 --- a/cmd/warm-backend-minio.go +++ b/cmd/warm-backend-minio.go @@ -78,7 +78,7 @@ func optimalPartSize(objectSize int64) (partSize int64, err error) { return partSize, nil } -func (m *warmBackendMinIO) Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) { +func (m *warmBackendMinIO) PutWithMeta(ctx context.Context, object string, r io.Reader, length int64, meta map[string]string) (remoteVersionID, error) { partSize, err := optimalPartSize(length) if err != nil { return remoteVersionID(""), err @@ -87,10 +87,15 @@ func (m *warmBackendMinIO) Put(ctx context.Context, object string, r io.Reader, StorageClass: m.StorageClass, PartSize: uint64(partSize), DisableContentSha256: true, + UserMetadata: meta, }) return remoteVersionID(res.VersionID), m.ToObjectError(err, object) } +func (m *warmBackendMinIO) Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) { + return m.PutWithMeta(ctx, object, r, length, map[string]string{}) +} + func newWarmBackendMinIO(conf madmin.TierMinIO, tier string) (*warmBackendMinIO, error) { // Validation of credentials if conf.AccessKey == "" || conf.SecretKey == "" { diff --git a/cmd/warm-backend-s3.go b/cmd/warm-backend-s3.go index de7e7d96d..f46b88fb6 100644 --- a/cmd/warm-backend-s3.go +++ b/cmd/warm-backend-s3.go @@ -56,14 +56,19 @@ func (s3 *warmBackendS3) getDest(object string) string { return destObj } -func (s3 *warmBackendS3) Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) { +func (s3 *warmBackendS3) PutWithMeta(ctx context.Context, object string, r io.Reader, length int64, meta map[string]string) (remoteVersionID, error) { res, err := s3.client.PutObject(ctx, s3.Bucket, s3.getDest(object), r, length, minio.PutObjectOptions{ SendContentMd5: true, StorageClass: s3.StorageClass, + UserMetadata: meta, }) return remoteVersionID(res.VersionID), s3.ToObjectError(err, object) } +func (s3 *warmBackendS3) Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) { + return s3.PutWithMeta(ctx, object, r, length, map[string]string{}) +} + func (s3 *warmBackendS3) Get(ctx context.Context, object string, rv remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) { gopts := minio.GetObjectOptions{} diff --git a/cmd/warm-backend.go b/cmd/warm-backend.go index 2389f3b1f..91a936004 100644 --- a/cmd/warm-backend.go +++ b/cmd/warm-backend.go @@ -38,6 +38,7 @@ type WarmBackendGetOpts struct { // WarmBackend provides interface to be implemented by remote tier backends type WarmBackend interface { Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) + PutWithMeta(ctx context.Context, object string, r io.Reader, length int64, meta map[string]string) (remoteVersionID, error) Get(ctx context.Context, object string, rv remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) Remove(ctx context.Context, object string, rv remoteVersionID) error InUse(ctx context.Context) (bool, error) diff --git a/docs/bucket/lifecycle/setup_ilm_transition.sh b/docs/bucket/lifecycle/setup_ilm_transition.sh new file mode 100755 index 000000000..975ea81f8 --- /dev/null +++ b/docs/bucket/lifecycle/setup_ilm_transition.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +set -x + +trap 'catch $LINENO' ERR + +# shellcheck disable=SC2120 +catch() { + if [ $# -ne 0 ]; then + echo "error on line $1" + for site in sitea siteb; do + echo "$site server logs =========" + cat "/tmp/${site}_1.log" + echo "===========================" + cat "/tmp/${site}_2.log" + done + fi + + echo "Cleaning up instances of MinIO" + pkill minio + pkill -9 minio + rm -rf /tmp/multisitea + rm -rf /tmp/multisiteb + if [ $# -ne 0 ]; then + exit $# + fi +} + +catch + +export MINIO_CI_CD=1 +export MINIO_BROWSER=off +export MINIO_KMS_AUTO_ENCRYPTION=off +export MINIO_PROMETHEUS_AUTH_TYPE=public +export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= +unset MINIO_KMS_KES_CERT_FILE +unset MINIO_KMS_KES_KEY_FILE +unset MINIO_KMS_KES_ENDPOINT +unset MINIO_KMS_KES_KEY_NAME + +if [ ! -f ./mc ]; then + wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc +fi + +minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 & +minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 & + +minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 & +minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 & + +# Wait to make sure all MinIO instances are up + +export MC_HOST_sitea=http://minioadmin:minioadmin@127.0.0.1:9001 +export MC_HOST_siteb=http://minioadmin:minioadmin@127.0.0.1:9004 + +./mc ready sitea +./mc ready siteb + +./mc mb --ignore-existing sitea/bucket +./mc mb --ignore-existing siteb/bucket + +sleep 10s + +## Add warm tier +./mc ilm tier add minio sitea WARM-TIER --endpoint http://localhost:9004 --access-key minioadmin --secret-key minioadmin --bucket bucket + +## Add ILM rules +./mc ilm add sitea/bucket --transition-days 0 --transition-tier WARM-TIER +./mc ilm rule list sitea/bucket + +./mc cp README.md sitea/bucket/README.md + +until $(./mc stat sitea/bucket/README.md --json | jq -r '.metadata."X-Amz-Storage-Class"' | grep -q WARM-TIER); do + echo "waiting until the object is tiered to run heal" + sleep 1s +done +./mc stat sitea/bucket/README.md + +success=$(./mc admin heal -r sitea/bucket/README.md --json --force | jq -r 'select((.name == "bucket/README.md") and (.after.color == "green")) | .after.color == "green"') +if [ "${success}" != "true" ]; then + echo "Found bug expected transitioned object to report 'green'" + exit 1 +fi + +catch