add tests for ILM transition and healing (#166) (#20601)

This PR fixes a regression introduced in https://github.com/minio/minio/pull/19797
by restoring the healing ability of transitioned objects

Bonus: support for transitioned objects to carry original
The object name is for future reverse lookups if necessary.

Also fix parity calculation for tiered objects to n/2 for n/2 == (parity)

.github/workflows/replication.yaml

@@ -40,6 +40,7 @@ jobs:
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make test-ilm
+          make test-ilm-transition
 
       - name: Test PBAC
         run: |

Makefile

@@ -60,6 +60,10 @@ test-ilm: install-race
 	@echo "Running ILM tests"
 	@env bash $(PWD)/docs/bucket/replication/setup_ilm_expiry_replication.sh
 
+test-ilm-transition: install-race
+	@echo "Running ILM tiering tests with healing"
+	@env bash $(PWD)/docs/bucket/lifecycle/setup_ilm_transition.sh
+
 test-pbac: install-race
 	@echo "Running bucket policies tests"
 	@env bash $(PWD)/docs/iam/policies/pbac-tests.sh

cmd/erasure-healing-common.go

@@ -392,8 +392,8 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 		if metaErrs[i] != nil {
 			continue
 		}
-		meta := partsMetadata[i]
 
+		meta := partsMetadata[i]
 		if meta.Deleted || meta.IsRemote() {
 			continue
 		}
@@ -442,14 +442,18 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 	}
 
 	for i, onlineDisk := range onlineDisks {
-		if metaErrs[i] == nil && !hasPartErr(dataErrsByDisk[i]) {
+		if metaErrs[i] == nil {
+			meta := partsMetadata[i]
+			if meta.Deleted || meta.IsRemote() || !hasPartErr(dataErrsByDisk[i]) {
 				// All parts verified, mark it as all data available.
 				availableDisks[i] = onlineDisk
-		} else {
+				continue
+			}
+		}
+
 		// upon errors just make that disk's fileinfo invalid
 		partsMetadata[i] = FileInfo{}
 	}
-	}
 
 	return
 }

cmd/erasure-metadata.go

@@ -514,8 +514,10 @@ func listObjectParities(partsMetadata []FileInfo, errs []error) (parities []int)
 		if metadata.Deleted || metadata.Size == 0 {
 			parities[index] = totalShards / 2
 		} else if metadata.TransitionStatus == lifecycle.TransitionComplete {
-			// For tiered objects, read quorum is N/2+1 to ensure simple majority on xl.meta. It is not equal to EcM because the data integrity is entrusted with the warm tier.
+			// For tiered objects, read quorum is N/2+1 to ensure simple majority on xl.meta.
-			parities[index] = totalShards - (totalShards/2 + 1)
+			// It is not equal to EcM because the data integrity is entrusted with the warm tier.
+			// However, we never go below EcM, in case of a EcM=EcN setup.
+			parities[index] = max(totalShards-(totalShards/2+1), metadata.Erasure.ParityBlocks)
 		} else {
 			parities[index] = metadata.Erasure.ParityBlocks
 		}

cmd/erasure-object.go

@@ -2376,7 +2376,11 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 	}()
 
 	var rv remoteVersionID
-	rv, err = tgtClient.Put(ctx, destObj, pr, fi.Size)
+	rv, err = tgtClient.PutWithMeta(ctx, destObj, pr, fi.Size, map[string]string{
+		"name": object, // preserve the original name of the object on the remote tier object metadata.
+		// this is just for future reverse lookup() purposes (applies only for new objects)
+		// does not apply retro-actively on already transitioned objects.
+	})
 	pr.CloseWithError(err)
 	if err != nil {
 		traceFn(ILMTransition, nil, err)

cmd/warm-backend-azure.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
@@ -59,10 +60,15 @@ func (az *warmBackendAzure) getDest(object string) string {
 	return destObj
 }
 
-func (az *warmBackendAzure) Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) {
+func (az *warmBackendAzure) PutWithMeta(ctx context.Context, object string, r io.Reader, length int64, meta map[string]string) (remoteVersionID, error) {
+	azMeta := map[string]*string{}
+	for k, v := range meta {
+		azMeta[k] = to.Ptr(v)
+	}
 	resp, err := az.clnt.UploadStream(ctx, az.Bucket, az.getDest(object), io.LimitReader(r, length), &azblob.UploadStreamOptions{
 		Concurrency: 4,
 		AccessTier:  az.tier(), // set tier if specified
+		Metadata:    azMeta,
 	})
 	if err != nil {
 		return "", azureToObjectError(err, az.Bucket, az.getDest(object))
@@ -74,6 +80,10 @@ func (az *warmBackendAzure) Put(ctx context.Context, object string, r io.Reader,
 	return remoteVersionID(vid), nil
 }
 
+func (az *warmBackendAzure) Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) {
+	return az.PutWithMeta(ctx, object, r, length, map[string]string{})
+}
+
 func (az *warmBackendAzure) Get(ctx context.Context, object string, rv remoteVersionID, opts WarmBackendGetOpts) (r io.ReadCloser, err error) {
 	if opts.startOffset < 0 {
 		return nil, InvalidRange{}

cmd/warm-backend-gcs.go

@@ -47,16 +47,17 @@ func (gcs *warmBackendGCS) getDest(object string) string {
 	return destObj
 }
 
-// FIXME: add support for remote version ID in GCS remote tier and remove this.
+func (gcs *warmBackendGCS) PutWithMeta(ctx context.Context, key string, data io.Reader, length int64, meta map[string]string) (remoteVersionID, error) {
-// Currently it's a no-op.
-
-func (gcs *warmBackendGCS) Put(ctx context.Context, key string, data io.Reader, length int64) (remoteVersionID, error) {
 	object := gcs.client.Bucket(gcs.Bucket).Object(gcs.getDest(key))
-	// TODO: set storage class
 	w := object.NewWriter(ctx)
 	if gcs.StorageClass != "" {
 		w.ObjectAttrs.StorageClass = gcs.StorageClass
 	}
+	w.ObjectAttrs.Metadata = meta
+	if _, err := xioutil.Copy(w, data); err != nil {
+		return "", gcsToObjectError(err, gcs.Bucket, key)
+	}
+
 	if _, err := xioutil.Copy(w, data); err != nil {
 		return "", gcsToObjectError(err, gcs.Bucket, key)
 	}
@@ -64,6 +65,12 @@ func (gcs *warmBackendGCS) Put(ctx context.Context, key string, data io.Reader,
 	return "", w.Close()
 }
 
+// FIXME: add support for remote version ID in GCS remote tier and remove this.
+// Currently it's a no-op.
+func (gcs *warmBackendGCS) Put(ctx context.Context, key string, data io.Reader, length int64) (remoteVersionID, error) {
+	return gcs.PutWithMeta(ctx, key, data, length, map[string]string{})
+}
+
 func (gcs *warmBackendGCS) Get(ctx context.Context, key string, rv remoteVersionID, opts WarmBackendGetOpts) (r io.ReadCloser, err error) {
 	// GCS storage decompresses a gzipped object by default and returns the data.
 	// Refer to https://cloud.google.com/storage/docs/transcoding#decompressive_transcoding

cmd/warm-backend-minio.go

@@ -78,7 +78,7 @@ func optimalPartSize(objectSize int64) (partSize int64, err error) {
 	return partSize, nil
 }
 
-func (m *warmBackendMinIO) Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) {
+func (m *warmBackendMinIO) PutWithMeta(ctx context.Context, object string, r io.Reader, length int64, meta map[string]string) (remoteVersionID, error) {
 	partSize, err := optimalPartSize(length)
 	if err != nil {
 		return remoteVersionID(""), err
@@ -87,10 +87,15 @@ func (m *warmBackendMinIO) Put(ctx context.Context, object string, r io.Reader,
 		StorageClass:         m.StorageClass,
 		PartSize:             uint64(partSize),
 		DisableContentSha256: true,
+		UserMetadata:         meta,
 	})
 	return remoteVersionID(res.VersionID), m.ToObjectError(err, object)
 }
 
+func (m *warmBackendMinIO) Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) {
+	return m.PutWithMeta(ctx, object, r, length, map[string]string{})
+}
+
 func newWarmBackendMinIO(conf madmin.TierMinIO, tier string) (*warmBackendMinIO, error) {
 	// Validation of credentials
 	if conf.AccessKey == "" || conf.SecretKey == "" {

cmd/warm-backend-s3.go

@@ -56,14 +56,19 @@ func (s3 *warmBackendS3) getDest(object string) string {
 	return destObj
 }
 
-func (s3 *warmBackendS3) Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) {
+func (s3 *warmBackendS3) PutWithMeta(ctx context.Context, object string, r io.Reader, length int64, meta map[string]string) (remoteVersionID, error) {
 	res, err := s3.client.PutObject(ctx, s3.Bucket, s3.getDest(object), r, length, minio.PutObjectOptions{
 		SendContentMd5: true,
 		StorageClass:   s3.StorageClass,
+		UserMetadata:   meta,
 	})
 	return remoteVersionID(res.VersionID), s3.ToObjectError(err, object)
 }
 
+func (s3 *warmBackendS3) Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) {
+	return s3.PutWithMeta(ctx, object, r, length, map[string]string{})
+}
+
 func (s3 *warmBackendS3) Get(ctx context.Context, object string, rv remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) {
 	gopts := minio.GetObjectOptions{}
 

cmd/warm-backend.go

@@ -38,6 +38,7 @@ type WarmBackendGetOpts struct {
 // WarmBackend provides interface to be implemented by remote tier backends
 type WarmBackend interface {
 	Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error)
+	PutWithMeta(ctx context.Context, object string, r io.Reader, length int64, meta map[string]string) (remoteVersionID, error)
 	Get(ctx context.Context, object string, rv remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error)
 	Remove(ctx context.Context, object string, rv remoteVersionID) error
 	InUse(ctx context.Context) (bool, error)

docs/bucket/lifecycle/setup_ilm_transition.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+
+set -x
+
+trap 'catch $LINENO' ERR
+
+# shellcheck disable=SC2120
+catch() {
+	if [ $# -ne 0 ]; then
+		echo "error on line $1"
+		for site in sitea siteb; do
+			echo "$site server logs ========="
+			cat "/tmp/${site}_1.log"
+			echo "==========================="
+			cat "/tmp/${site}_2.log"
+		done
+	fi
+
+	echo "Cleaning up instances of MinIO"
+	pkill minio
+	pkill -9 minio
+	rm -rf /tmp/multisitea
+	rm -rf /tmp/multisiteb
+	if [ $# -ne 0 ]; then
+		exit $#
+	fi
+}
+
+catch
+
+export MINIO_CI_CD=1
+export MINIO_BROWSER=off
+export MINIO_KMS_AUTO_ENCRYPTION=off
+export MINIO_PROMETHEUS_AUTH_TYPE=public
+export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw=
+unset MINIO_KMS_KES_CERT_FILE
+unset MINIO_KMS_KES_KEY_FILE
+unset MINIO_KMS_KES_ENDPOINT
+unset MINIO_KMS_KES_KEY_NAME
+
+if [ ! -f ./mc ]; then
+	wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc &&
+		chmod +x mc
+fi
+
+minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
+	"http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 &
+minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
+	"http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 &
+
+minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
+	"http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 &
+minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
+	"http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 &
+
+# Wait to make sure all MinIO instances are up
+
+export MC_HOST_sitea=http://minioadmin:minioadmin@127.0.0.1:9001
+export MC_HOST_siteb=http://minioadmin:minioadmin@127.0.0.1:9004
+
+./mc ready sitea
+./mc ready siteb
+
+./mc mb --ignore-existing sitea/bucket
+./mc mb --ignore-existing siteb/bucket
+
+sleep 10s
+
+## Add warm tier
+./mc ilm tier add minio sitea WARM-TIER --endpoint http://localhost:9004 --access-key minioadmin --secret-key minioadmin --bucket bucket
+
+## Add ILM rules
+./mc ilm add sitea/bucket --transition-days 0 --transition-tier WARM-TIER
+./mc ilm rule list sitea/bucket
+
+./mc cp README.md sitea/bucket/README.md
+
+until $(./mc stat sitea/bucket/README.md --json | jq -r '.metadata."X-Amz-Storage-Class"' | grep -q WARM-TIER); do
+	echo "waiting until the object is tiered to run heal"
+	sleep 1s
+done
+./mc stat sitea/bucket/README.md
+
+success=$(./mc admin heal -r sitea/bucket/README.md --json --force | jq -r 'select((.name == "bucket/README.md") and (.after.color == "green")) | .after.color == "green"')
+if [ "${success}" != "true" ]; then
+	echo "Found bug expected transitioned object to report 'green'"
+	exit 1
+fi
+
+catch