Enable replication of SSE-C objects (#19107)

If site replication enabled across sites, replicate the SSE-C
objects as well. These objects could be read from target sites
using the same client encryption keys.

Signed-off-by: Shubhendu Ram Tripathi <shubhendu@minio.io>

Makefile

@@ -107,6 +107,12 @@ test-site-replication-oidc: install-race ## verify automatic site replication
 test-site-replication-minio: install-race ## verify automatic site replication
 	@echo "Running tests for automatic site replication of IAM (with MinIO IDP)"
 	@(env bash $(PWD)/docs/site-replication/run-multi-site-minio-idp.sh)
+	@echo "Running tests for automatic site replication of SSE-C objects"
+	@(env bash $(PWD)/docs/site-replication/run-ssec-object-replication.sh)
+	@echo "Running tests for automatic site replication of SSE-C objects with SSE-KMS enabled for bucket"
+	@(env bash $(PWD)/docs/site-replication/run-sse-kms-object-replication.sh)
+	@echo "Running tests for automatic site replication of SSE-C objects with compression enabled for site"
+	@(env bash $(PWD)/docs/site-replication/run-ssec-object-replication-with-compression.sh)
 
 verify: ## verify minio various setups
 	@echo "Verifying build with race"

cmd/bucket-handlers.go

@@ -1218,11 +1218,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 			return
 		}
 
-		if crypto.SSEC.IsRequested(r.Header) && isReplicationEnabled(ctx, bucket) {
-			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParametersSSEC), r.URL)
-			return
-		}
-
 		var (
 			reader io.Reader
 			keyID  string

cmd/bucket-replication.go

@@ -51,6 +51,8 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/tinylib/msgp/msgp"
 	"github.com/zeebo/xxh3"
+	"golang.org/x/exp/maps"
+	"golang.org/x/exp/slices"
 )
 
 const (
@@ -76,11 +78,6 @@ const (
 	ReplicationWorkerMultiplier = 1.5
 )
 
-func isReplicationEnabled(ctx context.Context, bucketName string) bool {
-	rc, _ := getReplicationConfig(ctx, bucketName)
-	return rc != nil
-}
-
 // gets replication config associated to a given bucket name.
 func getReplicationConfig(ctx context.Context, bucketName string) (rc *replication.Config, err error) {
 	rCfg, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucketName)
@@ -764,13 +761,20 @@ func (m caseInsensitiveMap) Lookup(key string) (string, bool) {
 func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (putOpts minio.PutObjectOptions, err error) {
 	meta := make(map[string]string)
 	for k, v := range objInfo.UserDefined {
-		if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
+		// In case of SSE-C objects copy the allowed internal headers as well
-			continue
+		if !crypto.SSEC.IsEncrypted(objInfo.UserDefined) || !slices.Contains(maps.Keys(validSSEReplicationHeaders), k) {
+			if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
+				continue
+			}
+			if isStandardHeader(k) {
+				continue
+			}
 		}
-		if isStandardHeader(k) {
+		if slices.Contains(maps.Keys(validSSEReplicationHeaders), k) {
-			continue
+			meta[validSSEReplicationHeaders[k]] = v
+		} else {
+			meta[k] = v
 		}
-		meta[k] = v
 	}
 
 	if sc == "" && (objInfo.StorageClass == storageclass.STANDARD || objInfo.StorageClass == storageclass.RRS) {
@@ -1166,9 +1170,10 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object)
 
 	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, ObjectOptions{
-		VersionID:        ri.VersionID,
+		VersionID:          ri.VersionID,
-		Versioned:        versioned,
+		Versioned:          versioned,
-		VersionSuspended: versionSuspended,
+		VersionSuspended:   versionSuspended,
+		ReplicationRequest: true,
 	})
 	if err != nil {
 		if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
@@ -1322,11 +1327,13 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 	versioned := globalBucketVersioningSys.PrefixEnabled(bucket, object)
 	versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object)
 
-	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, ObjectOptions{
+	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{},
-		VersionID:        ri.VersionID,
+		ObjectOptions{
-		Versioned:        versioned,
+			VersionID:          ri.VersionID,
-		VersionSuspended: versionSuspended,
+			Versioned:          versioned,
-	})
+			VersionSuspended:   versionSuspended,
+			ReplicationRequest: true,
+		})
 	if err != nil {
 		if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
 			objInfo := ri.ToObjectInfo()
@@ -1344,6 +1351,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 	defer gr.Close()
 
 	objInfo := gr.ObjInfo
+
 	// make sure we have the latest metadata for metrics calculation
 	rinfo.PrevReplicationStatus = objInfo.TargetReplicationStatus(tgt.ARN)
 
@@ -1367,6 +1375,11 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		return
 	}
 
+	// Set the encrypted size for SSE-C objects
+	if crypto.SSEC.IsEncrypted(objInfo.UserDefined) {
+		size = objInfo.Size
+	}
+
 	if tgt.Bucket == "" {
 		logger.LogIf(ctx, fmt.Errorf("unable to replicate object %s(%s) to %s, target bucket is missing", objInfo.Name, objInfo.VersionID, tgt.EndpointURL()))
 		sendEvent(eventArgs{
@@ -1599,21 +1612,35 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 		pInfo minio.ObjectPart
 	)
 
+	var objectSize int64
 	for _, partInfo := range objInfo.Parts {
-		hr, err = hash.NewReader(ctx, io.LimitReader(r, partInfo.ActualSize), partInfo.ActualSize, "", "", partInfo.ActualSize)
+		if crypto.SSEC.IsEncrypted(objInfo.UserDefined) {
+			hr, err = hash.NewReader(ctx, io.LimitReader(r, partInfo.Size), partInfo.Size, "", "", partInfo.ActualSize)
+		} else {
+			hr, err = hash.NewReader(ctx, io.LimitReader(r, partInfo.ActualSize), partInfo.ActualSize, "", "", partInfo.ActualSize)
+		}
 		if err != nil {
 			return err
 		}
 
+		cHeader := http.Header{}
+		cHeader.Add(xhttp.MinIOSourceReplicationRequest, "true")
 		popts := minio.PutObjectPartOptions{
-			SSE: opts.ServerSideEncryption,
+			SSE:          opts.ServerSideEncryption,
+			CustomHeader: cHeader,
 		}
 
-		pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.ActualSize, popts)
+		if crypto.SSEC.IsEncrypted(objInfo.UserDefined) {
+			objectSize += partInfo.Size
+			pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.Size, popts)
+		} else {
+			objectSize += partInfo.ActualSize
+			pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.ActualSize, popts)
+		}
 		if err != nil {
 			return err
 		}
-		if pInfo.Size != partInfo.ActualSize {
+		if !crypto.SSEC.IsEncrypted(objInfo.UserDefined) && pInfo.Size != partInfo.ActualSize {
 			return fmt.Errorf("Part size mismatch: got %d, want %d", pInfo.Size, partInfo.ActualSize)
 		}
 		uploadedParts = append(uploadedParts, minio.CompletePart{
@@ -1624,6 +1651,7 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 	cctx, ccancel := context.WithTimeout(ctx, 10*time.Minute)
 	defer ccancel()
 	_, err = c.CompleteMultipartUpload(cctx, bucket, object, uploadID, uploadedParts, minio.PutObjectOptions{
+		UserMetadata: map[string]string{validSSEReplicationHeaders[ReservedMetadataPrefix+"Actual-Object-Size"]: objInfo.UserDefined[ReservedMetadataPrefix+"actual-size"]},
 		Internal: minio.AdvancedPutOptions{
 			SourceMTime: objInfo.ModTime,
 			// always set this to distinguish between `mc mirror` replication and serverside

cmd/erasure-multipart.go

@@ -1255,7 +1255,11 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	}
 
 	// Save the consolidated actual size.
-	fi.Metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(objectActualSize, 10)
+	if opts.ReplicationRequest {
+		fi.Metadata[ReservedMetadataPrefix+"actual-size"] = opts.UserDefined["X-Minio-Internal-Actual-Object-Size"]
+	} else {
+		fi.Metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(objectActualSize, 10)
+	}
 
 	if opts.DataMovement {
 		fi.SetDataMov()

cmd/erasure-object.go

@@ -245,6 +245,11 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 		}, toObjectErr(errMethodNotAllowed, bucket, object)
 	}
 
+	// Set NoDecryption for SSE-C objects and if replication request
+	if crypto.SSEC.IsEncrypted(objInfo.UserDefined) && opts.ReplicationRequest {
+		opts.NoDecryption = true
+	}
+
 	if objInfo.IsRemote() {
 		gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, h, objInfo, opts)
 		if err != nil {

cmd/generic-handlers.go

@@ -33,6 +33,8 @@ import (
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/grid"
 	xnet "github.com/minio/pkg/v2/net"
+	"golang.org/x/exp/maps"
+	"golang.org/x/exp/slices"
 
 	"github.com/minio/minio/internal/amztime"
 	"github.com/minio/minio/internal/config/dns"
@@ -73,6 +75,9 @@ const (
 // and must not set by clients
 func containsReservedMetadata(header http.Header) bool {
 	for key := range header {
+		if slices.Contains(maps.Keys(validSSEReplicationHeaders), key) {
+			return false
+		}
 		if stringsHasPrefixFold(key, ReservedMetadataPrefix) {
 			return true
 		}

cmd/generic-handlers_test.go

@@ -108,15 +108,15 @@ var containsReservedMetadataTests = []struct {
 	},
 	{
 		header:     http.Header{crypto.MetaIV: []string{"iv"}},
-		shouldFail: true,
+		shouldFail: false,
 	},
 	{
 		header:     http.Header{crypto.MetaAlgorithm: []string{crypto.InsecureSealAlgorithm}},
-		shouldFail: true,
+		shouldFail: false,
 	},
 	{
 		header:     http.Header{crypto.MetaSealedKeySSEC: []string{"mac"}},
-		shouldFail: true,
+		shouldFail: false,
 	},
 	{
 		header:     http.Header{ReservedMetadataPrefix + "Key": []string{"value"}},

cmd/handler-utils.go

@@ -33,6 +33,8 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/mcontext"
 	xnet "github.com/minio/pkg/v2/net"
+	"golang.org/x/exp/maps"
+	"golang.org/x/exp/slices"
 )
 
 const (
@@ -82,6 +84,31 @@ var supportedHeaders = []string{
 	xhttp.AmzObjectTagging,
 	"expires",
 	xhttp.AmzBucketReplicationStatus,
+	"X-Minio-Replication-Server-Side-Encryption-Sealed-Key",
+	"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm",
+	"X-Minio-Replication-Server-Side-Encryption-Iv",
+	"X-Minio-Replication-Encrypted-Multipart",
+	"X-Minio-Replication-Actual-Object-Size",
+	// Add more supported headers here.
+}
+
+// mapping of internal headers to allowed replication headers
+var validSSEReplicationHeaders = map[string]string{
+	"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "X-Minio-Replication-Server-Side-Encryption-Sealed-Key",
+	"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm",
+	"X-Minio-Internal-Server-Side-Encryption-Iv":             "X-Minio-Replication-Server-Side-Encryption-Iv",
+	"X-Minio-Internal-Encrypted-Multipart":                   "X-Minio-Replication-Encrypted-Multipart",
+	"X-Minio-Internal-Actual-Object-Size":                    "X-Minio-Replication-Actual-Object-Size",
+	// Add more supported headers here.
+}
+
+// mapping of replication headers to internal headers
+var replicationToInternalHeaders = map[string]string{
+	"X-Minio-Replication-Server-Side-Encryption-Sealed-Key":     "X-Minio-Internal-Server-Side-Encryption-Sealed-Key",
+	"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm": "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm",
+	"X-Minio-Replication-Server-Side-Encryption-Iv":             "X-Minio-Internal-Server-Side-Encryption-Iv",
+	"X-Minio-Replication-Encrypted-Multipart":                   "X-Minio-Internal-Encrypted-Multipart",
+	"X-Minio-Replication-Actual-Object-Size":                    "X-Minio-Internal-Actual-Object-Size",
 	// Add more supported headers here.
 }
 
@@ -178,7 +205,11 @@ func extractMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[
 	for _, supportedHeader := range supportedHeaders {
 		value, ok := nv[http.CanonicalHeaderKey(supportedHeader)]
 		if ok {
-			m[supportedHeader] = strings.Join(value, ",")
+			if slices.Contains(maps.Keys(replicationToInternalHeaders), supportedHeader) {
+				m[replicationToInternalHeaders[supportedHeader]] = strings.Join(value, ",")
+			} else {
+				m[supportedHeader] = strings.Join(value, ",")
+			}
 		}
 	}
 

cmd/object-api-options.go

@@ -480,7 +480,7 @@ func completeMultipartOpts(ctx context.Context, r *http.Request, bucket, object
 	}
 	opts.MTime = mtime
 	opts.UserDefined = make(map[string]string)
-
+	opts.UserDefined[ReservedMetadataPrefix+"Actual-Object-Size"] = r.Header.Get(xhttp.MinIOReplicationActualObjectSize)
 	// Transfer SSEC key in opts.EncryptFn
 	if crypto.SSEC.IsRequested(r.Header) {
 		key, err := ParseSSECustomerRequest(r)
@@ -491,5 +491,8 @@ func completeMultipartOpts(ctx context.Context, r *http.Request, bucket, object
 			}
 		}
 	}
+	if _, ok := r.Header[xhttp.MinIOSourceReplicationRequest]; ok {
+		opts.ReplicationRequest = true
+	}
 	return opts, nil
 }

cmd/object-handlers.go

@@ -2234,8 +2234,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 			return
 		}
 
-		if crypto.SSEC.IsRequested(r.Header) && isReplicationEnabled(ctx, bucket) {
+		if crypto.SSEC.IsRequested(r.Header) && isCompressible(r.Header, object) {
-			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParametersSSEC), r.URL)
+			writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionWithCompression), r.URL)
 			return
 		}
 
@@ -2649,10 +2649,6 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 				return errInvalidEncryptionParameters
 			}
 
-			if crypto.SSEC.IsRequested(r.Header) && isReplicationEnabled(ctx, bucket) {
-				return errInvalidEncryptionParametersSSEC
-			}
-
 			reader, objectEncryptionKey, err = EncryptRequest(hashReader, r, bucket, object, metadata)
 			if err != nil {
 				return err

cmd/object-multipart-handlers.go

@@ -116,14 +116,29 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 			return
 		}
 
-		if crypto.SSEC.IsRequested(r.Header) && isReplicationEnabled(ctx, bucket) {
+		if crypto.SSEC.IsRequested(r.Header) && isCompressible(r.Header, object) {
-			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParametersSSEC), r.URL)
+			writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionWithCompression), r.URL)
 			return
 		}
 
-		if err = setEncryptionMetadata(r, bucket, object, encMetadata); err != nil {
+		_, sourceReplReq := r.Header[xhttp.MinIOSourceReplicationRequest]
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		ssecRepHeaders := []string{
-			return
+			"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm",
+			"X-Minio-Replication-Server-Side-Encryption-Sealed-Key",
+			"X-Minio-Replication-Server-Side-Encryption-Iv",
+		}
+		ssecRep := false
+		for _, header := range ssecRepHeaders {
+			if val := r.Header.Get(header); val != "" {
+				ssecRep = true
+				break
+			}
+		}
+		if !(ssecRep && sourceReplReq) {
+			if err = setEncryptionMetadata(r, bucket, object, encMetadata); err != nil {
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+				return
+			}
 		}
 		// Set this for multipart only operations, we need to differentiate during
 		// decryption if the file was actually multipart or not.
@@ -757,9 +772,10 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 	pReader := NewPutObjReader(hashReader)
 
 	_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
+	_, replicationStatus := mi.UserDefined[xhttp.AmzBucketReplicationStatus]
 	var objectEncryptionKey crypto.ObjectKey
 	if isEncrypted {
-		if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(mi.UserDefined) {
+		if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(mi.UserDefined) && !replicationStatus {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrSSEMultipartEncrypted), r.URL)
 			return
 		}
@@ -779,52 +795,55 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 			}
 		}
 
-		// Calculating object encryption key
+		_, sourceReplReq := r.Header[xhttp.MinIOSourceReplicationRequest]
-		key, err = decryptObjectMeta(key, bucket, object, mi.UserDefined)
+		if !(sourceReplReq && crypto.SSEC.IsEncrypted(mi.UserDefined)) {
-		if err != nil {
+			// Calculating object encryption key
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			key, err = decryptObjectMeta(key, bucket, object, mi.UserDefined)
-			return
+			if err != nil {
-		}
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		copy(objectEncryptionKey[:], key)
+				return
+			}
+			copy(objectEncryptionKey[:], key)
 
-		partEncryptionKey := objectEncryptionKey.DerivePartKey(uint32(partID))
+			partEncryptionKey := objectEncryptionKey.DerivePartKey(uint32(partID))
-		in := io.Reader(hashReader)
+			in := io.Reader(hashReader)
-		if size > encryptBufferThreshold {
+			if size > encryptBufferThreshold {
-			// The encryption reads in blocks of 64KB.
+				// The encryption reads in blocks of 64KB.
-			// We add a buffer on bigger files to reduce the number of syscalls upstream.
+				// We add a buffer on bigger files to reduce the number of syscalls upstream.
-			in = bufio.NewReaderSize(hashReader, encryptBufferSize)
+				in = bufio.NewReaderSize(hashReader, encryptBufferSize)
-		}
+			}
-		reader, err = sio.EncryptReader(in, sio.Config{Key: partEncryptionKey[:], CipherSuites: fips.DARECiphers()})
+			reader, err = sio.EncryptReader(in, sio.Config{Key: partEncryptionKey[:], CipherSuites: fips.DARECiphers()})
-		if err != nil {
+			if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-			return
+				return
-		}
+			}
-		wantSize := int64(-1)
+			wantSize := int64(-1)
-		if size >= 0 {
+			if size >= 0 {
-			info := ObjectInfo{Size: size}
+				info := ObjectInfo{Size: size}
-			wantSize = info.EncryptedSize()
+				wantSize = info.EncryptedSize()
-		}
+			}
-		// do not try to verify encrypted content
+			// do not try to verify encrypted content
-		hashReader, err = hash.NewReader(ctx, etag.Wrap(reader, hashReader), wantSize, "", "", actualSize)
+			hashReader, err = hash.NewReader(ctx, etag.Wrap(reader, hashReader), wantSize, "", "", actualSize)
-		if err != nil {
+			if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-			return
+				return
-		}
+			}
-		if err := hashReader.AddChecksum(r, true); err != nil {
+			if err := hashReader.AddChecksum(r, true); err != nil {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
-			return
+				return
-		}
+			}
 
-		pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
+			pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
-		if err != nil {
+			if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-			return
+				return
-		}
+			}
 
-		if idxCb != nil {
+			if idxCb != nil {
-			idxCb = compressionIndexEncrypter(objectEncryptionKey, idxCb)
+				idxCb = compressionIndexEncrypter(objectEncryptionKey, idxCb)
+			}
+			opts.EncryptFn = metadataEncrypter(objectEncryptionKey)
 		}
-		opts.EncryptFn = metadataEncrypter(objectEncryptionKey)
 	}
 	opts.IndexCB = idxCb
 

docs/bucket/versioning/versioning-tests.sh

@@ -10,10 +10,10 @@ trap 'catch $LINENO' ERR
 catch() {
 	if [ $# -ne 0 ]; then
 		echo "error on line $1"
-		echo "$site server logs ========="
+		echo "server logs ========="
-		cat "/tmp/${site}_1.log"
+		cat "/tmp/sitea_1.log"
 		echo "==========================="
-		cat "/tmp/${site}_2.log"
+		cat "/tmp/sitea_2.log"
 	fi
 
 	echo "Cleaning up instances of MinIO"
@@ -42,32 +42,34 @@ if [ ! -f ./mc ]; then
 		chmod +x mc
 fi
 
-minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
+minio server --address ":9001" "https://localhost:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
-	"http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 &
+	"https://localhost:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 &
-minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
+minio server --address ":9002" "https://localhost:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
-	"http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 &
+	"https://localhost:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 &
 
-export MC_HOST_sitea=http://minio:minio123@127.0.0.1:9001
+sleep 60
 
-./mc mb sitea/delissue
+export MC_HOST_sitea=https://minio:minio123@localhost:9001
 
-./mc version enable sitea/delissue
+./mc mb sitea/delissue --insecure
 
-echo hello | ./mc pipe sitea/delissue/hello
+./mc version enable sitea/delissue --insecure
 
-./mc version suspend sitea/delissue
+echo hello | ./mc pipe sitea/delissue/hello --insecure
 
-./mc rm sitea/delissue/hello
+./mc version suspend sitea/delissue --insecure
 
-./mc version enable sitea/delissue
+./mc rm sitea/delissue/hello --insecure
 
-echo hello | ./mc pipe sitea/delissue/hello
+./mc version enable sitea/delissue --insecure
 
-./mc version suspend sitea/delissue
+echo hello | ./mc pipe sitea/delissue/hello --insecure
 
-./mc rm sitea/delissue/hello
+./mc version suspend sitea/delissue --insecure
 
-count=$(./mc ls --versions sitea/delissue | wc -l)
+./mc rm sitea/delissue/hello --insecure
+
+count=$(./mc ls --versions sitea/delissue --insecure | wc -l)
 
 if [ ${count} -ne 3 ]; then
 	echo "BUG: expected number of versions to be '3' found ${count}"
@@ -76,6 +78,6 @@ if [ ${count} -ne 3 ]; then
 fi
 
 echo "SUCCESS:"
-./mc ls --versions sitea/delissue
+./mc ls --versions sitea/delissue --insecure
 
 catch

docs/site-replication/run-sse-kms-object-replication.sh

@@ -0,0 +1,230 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC2120
+exit_1() {
+	cleanup
+
+	echo "minio1 ============"
+	cat /tmp/minio1_1.log
+	echo "minio2 ============"
+	cat /tmp/minio2_1.log
+
+	exit 1
+}
+
+cleanup() {
+	echo -n "Cleaning up instances of MinIO ..."
+	pkill -9 minio || sudo pkill -9 minio
+	pkill -9 kes || sudo pkill -9 kes
+	rm -rf ${PWD}/keys
+	rm -rf /tmp/minio{1,2}
+	echo "done"
+}
+
+cleanup
+
+export MINIO_CI_CD=1
+export MINIO_BROWSER=off
+export MINIO_ROOT_USER="minio"
+export MINIO_ROOT_PASSWORD="minio123"
+
+# Create certificates for TLS enabled MinIO
+echo -n "Setup certs for MinIO instances ..."
+wget -O certgen https://github.com/minio/certgen/releases/latest/download/certgen-linux-amd64 && chmod +x certgen
+./certgen --host localhost
+mkdir -p ~/.minio/certs
+mv public.crt ~/.minio/certs || sudo mv public.crt ~/.minio/certs
+mv private.key ~/.minio/certs || sudo mv private.key ~/.minio/certs
+echo "done"
+
+# Start MinIO instances
+echo -n "Starting MinIO instances ..."
+CI=on MINIO_KMS_SECRET_KEY=minio-default-key:IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g= MINIO_ROOT_USER=minio MINIO_ROOT_PASSWORD=minio123 minio server --address ":9001" --console-address ":10000" /tmp/minio1/{1...4}/disk{1...4} /tmp/minio1/{5...8}/disk{1...4} >/tmp/minio1_1.log 2>&1 &
+CI=on MINIO_KMS_SECRET_KEY=minio-default-key:IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g= MINIO_ROOT_USER=minio MINIO_ROOT_PASSWORD=minio123 minio server --address ":9002" --console-address ":11000" /tmp/minio2/{1...4}/disk{1...4} /tmp/minio2/{5...8}/disk{1...4} >/tmp/minio2_1.log 2>&1 &
+echo "done"
+
+if [ ! -f ./mc ]; then
+	echo -n "Downloading MinIO client ..."
+	wget -O mc https://dl.min.io/client/mc/release/linux-amd64/mc &&
+		chmod +x mc
+	echo "done"
+fi
+
+sleep 10
+
+export MC_HOST_minio1=https://minio:minio123@localhost:9001
+export MC_HOST_minio2=https://minio:minio123@localhost:9002
+
+# Prepare data for tests
+echo -n "Preparing test data ..."
+mkdir -p /tmp/data
+echo "Hello from encrypted world" >/tmp/data/encrypted
+touch /tmp/data/mpartobj
+shred -s 500M /tmp/data/mpartobj
+touch /tmp/data/defpartsize
+shred -s 500M /tmp/data/defpartsize
+touch /tmp/data/custpartsize
+shred -s 500M /tmp/data/custpartsize
+echo "done"
+
+# Add replication site
+./mc admin replicate add minio1 minio2 --insecure
+# sleep for replication to complete
+sleep 30
+
+# Create bucket in source cluster
+echo "Create bucket in source MinIO instance"
+./mc mb minio1/test-bucket --insecure
+
+# Enable SSE KMS for the bucket
+./mc encrypt set sse-kms minio-default-key minio1/test-bucket --insecure
+
+# Load objects to source site
+echo "Loading objects to source MinIO instance"
+./mc cp /tmp/data/encrypted minio1/test-bucket --insecure
+./mc cp /tmp/data/mpartobj minio1/test-bucket --encrypt-key "minio1/test-bucket/mpartobj=iliketobecrazybutnotsomuchreally" --insecure
+./mc cp /tmp/data/defpartsize minio1/test-bucket --insecure
+./mc put /tmp/data/custpartsize minio1/test-bucket --insecure --part-size 50MiB
+sleep 120
+
+# List the objects from source site
+echo "Objects from source instance"
+./mc ls minio1/test-bucket --insecure
+count1=$(./mc ls minio1/test-bucket/encrypted --insecure | wc -l)
+if [ "${count1}" -ne 1 ]; then
+	echo "BUG: object minio1/test-bucket/encrypted not found"
+	exit_1
+fi
+count2=$(./mc ls minio1/test-bucket/mpartobj --insecure | wc -l)
+if [ "${count2}" -ne 1 ]; then
+	echo "BUG: object minio1/test-bucket/mpartobj not found"
+	exit_1
+fi
+count3=$(./mc ls minio1/test-bucket/defpartsize --insecure | wc -l)
+if [ "${count3}" -ne 1 ]; then
+	echo "BUG: object minio1/test-bucket/defpartsize not found"
+	exit_1
+fi
+count4=$(./mc ls minio1/test-bucket/custpartsize --insecure | wc -l)
+if [ "${count4}" -ne 1 ]; then
+	echo "BUG: object minio1/test-bucket/custpartsize not found"
+	exit_1
+fi
+
+# List the objects from replicated site
+echo "Objects from replicated instance"
+./mc ls minio2/test-bucket --insecure
+repcount1=$(./mc ls minio2/test-bucket/encrypted --insecure | wc -l)
+if [ "${repcount1}" -ne 1 ]; then
+	echo "BUG: object test-bucket/encrypted not replicated"
+	exit_1
+fi
+repcount2=$(./mc ls minio2/test-bucket/mpartobj --insecure | wc -l)
+if [ "${repcount2}" -ne 1 ]; then
+	echo "BUG: object test-bucket/mpartobj not replicated"
+	exit_1
+fi
+repcount3=$(./mc ls minio2/test-bucket/defpartsize --insecure | wc -l)
+if [ "${repcount3}" -ne 1 ]; then
+	echo "BUG: object test-bucket/defpartsize not replicated"
+	exit_1
+fi
+repcount4=$(./mc ls minio2/test-bucket/custpartsize --insecure | wc -l)
+if [ "${repcount4}" -ne 1 ]; then
+	echo "BUG: object test-bucket/custpartsize not replicated"
+	exit_1
+fi
+
+# Stat the objects from source site
+echo "Stat minio1/test-bucket/encrypted"
+./mc stat minio1/test-bucket/encrypted --insecure --json
+stat_out1=$(./mc stat minio1/test-bucket/encrypted --insecure --json)
+src_obj1_algo=$(echo "${stat_out1}" | jq '.metadata."X-Amz-Server-Side-Encryption"')
+src_obj1_keyid=$(echo "${stat_out1}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"')
+echo "Stat minio1/test-bucket/defpartsize"
+./mc stat minio1/test-bucket/defpartsize --insecure --json
+stat_out2=$(./mc stat minio1/test-bucket/defpartsize --insecure --json)
+src_obj2_algo=$(echo "${stat_out2}" | jq '.metadata."X-Amz-Server-Side-Encryption"')
+src_obj2_keyid=$(echo "${stat_out2}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"')
+echo "Stat minio1/test-bucket/custpartsize"
+./mc stat minio1/test-bucket/custpartsize --insecure --json
+stat_out3=$(./mc stat minio1/test-bucket/custpartsize --insecure --json)
+src_obj3_algo=$(echo "${stat_out3}" | jq '.metadata."X-Amz-Server-Side-Encryption"')
+src_obj3_keyid=$(echo "${stat_out3}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"')
+echo "Stat minio1/test-bucket/mpartobj"
+./mc stat minio1/test-bucket/mpartobj --encrypt-key "minio1/test-bucket/mpartobj=iliketobecrazybutnotsomuchreally" --insecure --json
+stat_out4=$(./mc stat minio1/test-bucket/mpartobj --encrypt-key "minio1/test-bucket/mpartobj=iliketobecrazybutnotsomuchreally" --insecure --json)
+src_obj4_etag=$(echo "${stat_out4}" | jq '.etag')
+src_obj4_size=$(echo "${stat_out4}" | jq '.size')
+src_obj4_md5=$(echo "${stat_out4}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
+
+# Stat the objects from replicated site
+echo "Stat minio2/test-bucket/encrypted"
+./mc stat minio2/test-bucket/encrypted --insecure --json
+stat_out1_rep=$(./mc stat minio2/test-bucket/encrypted --insecure --json)
+rep_obj1_algo=$(echo "${stat_out1_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption"')
+rep_obj1_keyid=$(echo "${stat_out1_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"')
+echo "Stat minio2/test-bucket/defpartsize"
+./mc stat minio2/test-bucket/defpartsize --insecure --json
+stat_out2_rep=$(./mc stat minio2/test-bucket/defpartsize --insecure --json)
+rep_obj2_algo=$(echo "${stat_out2_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption"')
+rep_obj2_keyid=$(echo "${stat_out2_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"')
+echo "Stat minio2/test-bucket/custpartsize"
+./mc stat minio2/test-bucket/custpartsize --insecure --json
+stat_out3_rep=$(./mc stat minio2/test-bucket/custpartsize --insecure --json)
+rep_obj3_algo=$(echo "${stat_out3_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption"')
+rep_obj3_keyid=$(echo "${stat_out3_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"')
+echo "Stat minio2/test-bucket/mpartobj"
+./mc stat minio2/test-bucket/mpartobj --encrypt-key "minio2/test-bucket/mpartobj=iliketobecrazybutnotsomuchreally" --insecure --json
+stat_out4_rep=$(./mc stat minio2/test-bucket/mpartobj --encrypt-key "minio2/test-bucket/mpartobj=iliketobecrazybutnotsomuchreally" --insecure --json)
+rep_obj4_etag=$(echo "${stat_out4}" | jq '.etag')
+rep_obj4_size=$(echo "${stat_out4}" | jq '.size')
+rep_obj4_md5=$(echo "${stat_out4}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
+
+# Check the algo and keyId of replicated objects
+if [ "${rep_obj1_algo}" != "${src_obj1_algo}" ]; then
+	echo "BUG: Algorithm: '${rep_obj1_algo}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_algo}'"
+	exit_1
+fi
+if [ "${rep_obj1_keyid}" != "${src_obj1_keyid}" ]; then
+	echo "BUG: KeyId: '${rep_obj1_keyid}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_keyid}'"
+	exit_1
+fi
+if [ "${rep_obj2_algo}" != "${src_obj2_algo}" ]; then
+	echo "BUG: Algorithm: '${rep_obj2_algo}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_algo}'"
+	exit_1
+fi
+if [ "${rep_obj2_keyid}" != "${src_obj2_keyid}" ]; then
+	echo "BUG: KeyId: '${rep_obj2_keyid}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_keyid}'"
+	exit_1
+fi
+if [ "${rep_obj3_algo}" != "${src_obj3_algo}" ]; then
+	echo "BUG: Algorithm: '${rep_obj3_algo}' of replicated object: 'minio2/test-bucket/custpartsize' doesn't match with source value: '${src_obj3_algo}'"
+	exit_1
+fi
+if [ "${rep_obj3_keyid}" != "${src_obj3_keyid}" ]; then
+	echo "BUG: KeyId: '${rep_obj3_keyid}' of replicated object: 'minio2/test-bucket/custpartsize' doesn't match with source value: '${src_obj3_keyid}'"
+	exit_1
+fi
+
+# Check the etag, size and md5 of replicated SSEC object
+if [ "${rep_obj4_etag}" != "${src_obj4_etag}" ]; then
+	echo "BUG: Etag: '${rep_obj4_etag}' of replicated object: 'minio2/test-bucket/mpartobj' doesn't match with source value: '${src_obj4_etag}'"
+	exit_1
+fi
+if [ "${rep_obj4_size}" != "${src_obj4_size}" ]; then
+	echo "BUG: Size: '${rep_obj4_size}' of replicated object: 'minio2/test-bucket/mpartobj' doesn't match with source value: '${src_obj4_size}'"
+	exit_1
+fi
+if [ "${src_obj4_md5}" != "${rep_obj4_md5}" ]; then
+	echo "BUG: MD5 checksum of object 'minio2/test-bucket/mpartobj' doesn't match with source. Expected: '${src_obj4_md5}', Found: '${rep_obj4_md5}'"
+	exit_1
+fi
+
+# Check content of replicated objects
+./mc cat minio2/test-bucket/encrypted --insecure
+./mc cat minio2/test-bucket/mpartobj --encrypt-key "minio2/test-bucket/mpartobj=iliketobecrazybutnotsomuchreally" --insecure >/dev/null || exit_1
+./mc cat minio2/test-bucket/defpartsize --insecure >/dev/null || exit_1
+./mc cat minio2/test-bucket/custpartsize --insecure >/dev/null || exit_1
+
+cleanup

docs/site-replication/run-ssec-object-replication-with-compression.sh

@@ -0,0 +1,193 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC2120
+exit_1() {
+	cleanup
+
+	echo "minio1 ============"
+	cat /tmp/minio1_1.log
+	echo "minio2 ============"
+	cat /tmp/minio2_1.log
+
+	exit 1
+}
+
+cleanup() {
+	echo -n "Cleaning up instances of MinIO ..."
+	pkill minio || sudo pkill minio
+	pkill -9 minio || sudo pkill -9 minio
+	rm -rf /tmp/minio{1,2}
+	echo "done"
+}
+
+cleanup
+
+export MINIO_CI_CD=1
+export MINIO_BROWSER=off
+export MINIO_ROOT_USER="minio"
+export MINIO_ROOT_PASSWORD="minio123"
+
+# Create certificates for TLS enabled MinIO
+echo -n "Setup certs for MinIO instances ..."
+wget -O certgen https://github.com/minio/certgen/releases/latest/download/certgen-linux-amd64 && chmod +x certgen
+./certgen --host localhost
+mkdir -p ~/.minio/certs
+mv public.crt ~/.minio/certs || sudo mv public.crt ~/.minio/certs
+mv private.key ~/.minio/certs || sudo mv private.key ~/.minio/certs
+echo "done"
+
+# Start MinIO instances
+echo -n "Starting MinIO instances ..."
+minio server --address ":9001" --console-address ":10000" /tmp/minio1/{1...4}/disk{1...4} /tmp/minio1/{5...8}/disk{1...4} >/tmp/minio1_1.log 2>&1 &
+minio server --address ":9002" --console-address ":11000" /tmp/minio2/{1...4}/disk{1...4} /tmp/minio2/{5...8}/disk{1...4} >/tmp/minio2_1.log 2>&1 &
+echo "done"
+
+if [ ! -f ./mc ]; then
+	echo -n "Downloading MinIO client ..."
+	wget -O mc https://dl.min.io/client/mc/release/linux-amd64/mc &&
+		chmod +x mc
+	echo "done"
+fi
+
+sleep 10
+
+export MC_HOST_minio1=https://minio:minio123@localhost:9001
+export MC_HOST_minio2=https://minio:minio123@localhost:9002
+
+# Prepare data for tests
+echo -n "Preparing test data ..."
+mkdir -p /tmp/data
+echo "Hello world" >/tmp/data/plainfile
+echo "Hello from encrypted world" >/tmp/data/encrypted
+touch /tmp/data/defpartsize
+shred -s 500M /tmp/data/defpartsize
+touch /tmp/data/mpartobj.txt
+shred -s 500M /tmp/data/mpartobj.txt
+echo "done"
+
+# Enable compression for site minio1
+./mc admin config set minio1 compression enable=on extensions=".txt" --insecure
+./mc admin config set minio1 compression allow_encryption=on --insecure
+
+# Create bucket in source cluster
+echo "Create bucket in source MinIO instance"
+./mc mb minio1/test-bucket --insecure
+
+# Load objects to source site
+echo "Loading objects to source MinIO instance"
+./mc cp /tmp/data/plainfile minio1/test-bucket --insecure
+./mc cp /tmp/data/encrypted minio1/test-bucket --encrypt-key "minio1/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure
+./mc cp /tmp/data/defpartsize minio1/test-bucket --encrypt-key "minio1/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure
+
+# Below should fail as compression and SSEC used at the same time
+RESULT=$({ ./mc put /tmp/data/mpartobj.txt minio1/test-bucket --encrypt-key "minio1/test-bucket/mpartobj.txt=iliketobecrazybutnotsomuchreally" --insecure; } 2>&1)
+if [[ ${RESULT} != *"Server side encryption specified with SSE-C with compression not allowed"* ]]; then
+	echo "BUG: Loading an SSE-C object to site with compression should fail. Succeeded though."
+	exit_1
+fi
+
+# Add replication site
+./mc admin replicate add minio1 minio2 --insecure
+# sleep for replication to complete
+sleep 30
+
+# List the objects from source site
+echo "Objects from source instance"
+./mc ls minio1/test-bucket --insecure
+count1=$(./mc ls minio1/test-bucket/plainfile --insecure | wc -l)
+if [ "${count1}" -ne 1 ]; then
+	echo "BUG: object minio1/test-bucket/plainfile not found"
+	exit_1
+fi
+count2=$(./mc ls minio1/test-bucket/encrypted --insecure | wc -l)
+if [ "${count2}" -ne 1 ]; then
+	echo "BUG: object minio1/test-bucket/encrypted not found"
+	exit_1
+fi
+count3=$(./mc ls minio1/test-bucket/defpartsize --insecure | wc -l)
+if [ "${count3}" -ne 1 ]; then
+	echo "BUG: object minio1/test-bucket/defpartsize not found"
+	exit_1
+fi
+sleep 120
+
+# List the objects from replicated site
+echo "Objects from replicated instance"
+./mc ls minio2/test-bucket --insecure
+repcount1=$(./mc ls minio2/test-bucket/plainfile --insecure | wc -l)
+if [ "${repcount1}" -ne 1 ]; then
+	echo "BUG: object test-bucket/plainfile not replicated"
+	exit_1
+fi
+repcount2=$(./mc ls minio2/test-bucket/encrypted --insecure | wc -l)
+if [ "${repcount2}" -ne 1 ]; then
+	echo "BUG: object test-bucket/encrypted not replicated"
+	exit_1
+fi
+repcount3=$(./mc ls minio2/test-bucket/defpartsize --insecure | wc -l)
+if [ "${repcount3}" -ne 1 ]; then
+	echo "BUG: object test-bucket/defpartsize not replicated"
+	exit_1
+fi
+
+# Stat the SSEC objects from source site
+echo "Stat minio1/test-bucket/encrypted"
+./mc stat minio1/test-bucket/encrypted --encrypt-key "minio1/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json
+stat_out1=$(./mc stat minio1/test-bucket/encrypted --encrypt-key "minio1/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json)
+src_obj1_etag=$(echo "${stat_out1}" | jq '.etag')
+src_obj1_size=$(echo "${stat_out1}" | jq '.size')
+src_obj1_md5=$(echo "${stat_out1}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
+echo "Stat minio1/test-bucket/defpartsize"
+./mc stat minio1/test-bucket/defpartsize --encrypt-key "minio1/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json
+stat_out2=$(./mc stat minio1/test-bucket/defpartsize --encrypt-key "minio1/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json)
+src_obj2_etag=$(echo "${stat_out2}" | jq '.etag')
+src_obj2_size=$(echo "${stat_out2}" | jq '.size')
+src_obj2_md5=$(echo "${stat_out2}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
+
+# Stat the SSEC objects from replicated site
+echo "Stat minio2/test-bucket/encrypted"
+./mc stat minio2/test-bucket/encrypted --encrypt-key "minio2/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json
+stat_out1_rep=$(./mc stat minio2/test-bucket/encrypted --encrypt-key "minio2/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json)
+rep_obj1_etag=$(echo "${stat_out1_rep}" | jq '.etag')
+rep_obj1_size=$(echo "${stat_out1_rep}" | jq '.size')
+rep_obj1_md5=$(echo "${stat_out1_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
+echo "Stat minio2/test-bucket/defpartsize"
+./mc stat minio2/test-bucket/defpartsize --encrypt-key "minio2/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json
+stat_out2_rep=$(./mc stat minio2/test-bucket/defpartsize --encrypt-key "minio2/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json)
+rep_obj2_etag=$(echo "${stat_out2_rep}" | jq '.etag')
+rep_obj2_size=$(echo "${stat_out2_rep}" | jq '.size')
+rep_obj2_md5=$(echo "${stat_out2_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
+
+# Check the etag and size of replicated SSEC objects
+if [ "${rep_obj1_etag}" != "${src_obj1_etag}" ]; then
+	echo "BUG: Etag: '${rep_obj1_etag}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_etag}'"
+	exit_1
+fi
+if [ "${rep_obj1_size}" != "${src_obj1_size}" ]; then
+	echo "BUG: Size: '${rep_obj1_size}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_size}'"
+	exit_1
+fi
+if [ "${rep_obj2_etag}" != "${src_obj2_etag}" ]; then
+	echo "BUG: Etag: '${rep_obj2_etag}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_etag}'"
+	exit_1
+fi
+if [ "${rep_obj2_size}" != "${src_obj2_size}" ]; then
+	echo "BUG: Size: '${rep_obj2_size}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_size}'"
+	exit_1
+fi
+
+# Check content of replicated SSEC objects
+./mc cat minio2/test-bucket/encrypted --encrypt-key "minio2/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure
+./mc cat minio2/test-bucket/defpartsize --encrypt-key "minio2/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure >/dev/null || exit_1
+
+# Check the MD5 checksums of encrypted objects from source and target
+if [ "${src_obj1_md5}" != "${rep_obj1_md5}" ]; then
+	echo "BUG: MD5 checksum of object 'minio2/test-bucket/encrypted' doesn't match with source. Expected: '${src_obj1_md5}', Found: '${rep_obj1_md5}'"
+	exit_1
+fi
+if [ "${src_obj2_md5}" != "${rep_obj2_md5}" ]; then
+	echo "BUG: MD5 checksum of object 'minio2/test-bucket/defpartsize' doesn't match with source. Expected: '${src_obj2_md5}', Found: '${rep_obj2_md5}'"
+	exit_1
+fi
+
+cleanup

docs/site-replication/run-ssec-object-replication.sh

@@ -0,0 +1,219 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC2120
+exit_1() {
+	cleanup
+
+	echo "minio1 ============"
+	cat /tmp/minio1_1.log
+	echo "minio2 ============"
+	cat /tmp/minio2_1.log
+
+	exit 1
+}
+
+cleanup() {
+	echo -n "Cleaning up instances of MinIO ..."
+	pkill minio || sudo pkill minio
+	pkill -9 minio || sudo pkill -9 minio
+	rm -rf /tmp/minio{1,2}
+	echo "done"
+}
+
+cleanup
+
+export MINIO_CI_CD=1
+export MINIO_BROWSER=off
+export MINIO_ROOT_USER="minio"
+export MINIO_ROOT_PASSWORD="minio123"
+
+# Create certificates for TLS enabled MinIO
+echo -n "Setup certs for MinIO instances ..."
+wget -O certgen https://github.com/minio/certgen/releases/latest/download/certgen-linux-amd64 && chmod +x certgen
+./certgen --host localhost
+mkdir -p ~/.minio/certs
+mv public.crt ~/.minio/certs || sudo mv public.crt ~/.minio/certs
+mv private.key ~/.minio/certs || sudo mv private.key ~/.minio/certs
+echo "done"
+
+# Start MinIO instances
+echo -n "Starting MinIO instances ..."
+minio server --address ":9001" --console-address ":10000" /tmp/minio1/{1...4}/disk{1...4} /tmp/minio1/{5...8}/disk{1...4} >/tmp/minio1_1.log 2>&1 &
+minio server --address ":9002" --console-address ":11000" /tmp/minio2/{1...4}/disk{1...4} /tmp/minio2/{5...8}/disk{1...4} >/tmp/minio2_1.log 2>&1 &
+echo "done"
+
+if [ ! -f ./mc ]; then
+	echo -n "Downloading MinIO client ..."
+	wget -O mc https://dl.min.io/client/mc/release/linux-amd64/mc &&
+		chmod +x mc
+	echo "done"
+fi
+
+sleep 10
+
+export MC_HOST_minio1=https://minio:minio123@localhost:9001
+export MC_HOST_minio2=https://minio:minio123@localhost:9002
+
+# Prepare data for tests
+echo -n "Preparing test data ..."
+mkdir -p /tmp/data
+echo "Hello world" >/tmp/data/plainfile
+echo "Hello from encrypted world" >/tmp/data/encrypted
+touch /tmp/data/defpartsize
+shred -s 500M /tmp/data/defpartsize
+touch /tmp/data/custpartsize
+shred -s 500M /tmp/data/custpartsize
+echo "done"
+
+# Add replication site
+./mc admin replicate add minio1 minio2 --insecure
+# sleep for replication to complete
+sleep 30
+
+# Create bucket in source cluster
+echo "Create bucket in source MinIO instance"
+./mc mb minio1/test-bucket --insecure
+
+# Load objects to source site
+echo "Loading objects to source MinIO instance"
+./mc cp /tmp/data/plainfile minio1/test-bucket --insecure
+./mc cp /tmp/data/encrypted minio1/test-bucket --encrypt-key "minio1/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure
+./mc cp /tmp/data/defpartsize minio1/test-bucket --encrypt-key "minio1/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure
+./mc put /tmp/data/custpartsize minio1/test-bucket --encrypt-key "minio1/test-bucket/custpartsize=iliketobecrazybutnotsomuchreally" --insecure --part-size 50MiB
+sleep 120
+
+# List the objects from source site
+echo "Objects from source instance"
+./mc ls minio1/test-bucket --insecure
+count1=$(./mc ls minio1/test-bucket/plainfile --insecure | wc -l)
+if [ "${count1}" -ne 1 ]; then
+	echo "BUG: object minio1/test-bucket/plainfile not found"
+	exit_1
+fi
+count2=$(./mc ls minio1/test-bucket/encrypted --insecure | wc -l)
+if [ "${count2}" -ne 1 ]; then
+	echo "BUG: object minio1/test-bucket/encrypted not found"
+	exit_1
+fi
+count3=$(./mc ls minio1/test-bucket/defpartsize --insecure | wc -l)
+if [ "${count3}" -ne 1 ]; then
+	echo "BUG: object minio1/test-bucket/defpartsize not found"
+	exit_1
+fi
+count4=$(./mc ls minio1/test-bucket/custpartsize --insecure | wc -l)
+if [ "${count4}" -ne 1 ]; then
+	echo "BUG: object minio1/test-bucket/custpartsize not found"
+	exit_1
+fi
+
+# List the objects from replicated site
+echo "Objects from replicated instance"
+./mc ls minio2/test-bucket --insecure
+repcount1=$(./mc ls minio2/test-bucket/plainfile --insecure | wc -l)
+if [ "${repcount1}" -ne 1 ]; then
+	echo "BUG: object test-bucket/plainfile not replicated"
+	exit_1
+fi
+repcount2=$(./mc ls minio2/test-bucket/encrypted --insecure | wc -l)
+if [ "${repcount2}" -ne 1 ]; then
+	echo "BUG: object test-bucket/encrypted not replicated"
+	exit_1
+fi
+repcount3=$(./mc ls minio2/test-bucket/defpartsize --insecure | wc -l)
+if [ "${repcount3}" -ne 1 ]; then
+	echo "BUG: object test-bucket/defpartsize not replicated"
+	exit_1
+fi
+
+repcount4=$(./mc ls minio2/test-bucket/custpartsize --insecure | wc -l)
+if [ "${repcount4}" -ne 1 ]; then
+	echo "BUG: object test-bucket/custpartsize not replicated"
+	exit_1
+fi
+
+# Stat the SSEC objects from source site
+echo "Stat minio1/test-bucket/encrypted"
+./mc stat minio1/test-bucket/encrypted --encrypt-key "minio1/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json
+stat_out1=$(./mc stat minio1/test-bucket/encrypted --encrypt-key "minio1/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json)
+src_obj1_etag=$(echo "${stat_out1}" | jq '.etag')
+src_obj1_size=$(echo "${stat_out1}" | jq '.size')
+src_obj1_md5=$(echo "${stat_out1}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
+echo "Stat minio1/test-bucket/defpartsize"
+./mc stat minio1/test-bucket/defpartsize --encrypt-key "minio1/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json
+stat_out2=$(./mc stat minio1/test-bucket/defpartsize --encrypt-key "minio1/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json)
+src_obj2_etag=$(echo "${stat_out2}" | jq '.etag')
+src_obj2_size=$(echo "${stat_out2}" | jq '.size')
+src_obj2_md5=$(echo "${stat_out2}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
+echo "Stat minio1/test-bucket/custpartsize"
+./mc stat minio1/test-bucket/custpartsize --encrypt-key "minio1/test-bucket/custpartsize=iliketobecrazybutnotsomuchreally" --insecure --json
+stat_out3=$(./mc stat minio1/test-bucket/custpartsize --encrypt-key "minio1/test-bucket/custpartsize=iliketobecrazybutnotsomuchreally" --insecure --json)
+src_obj3_etag=$(echo "${stat_out3}" | jq '.etag')
+src_obj3_size=$(echo "${stat_out3}" | jq '.size')
+src_obj3_md5=$(echo "${stat_out3}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
+
+# Stat the SSEC objects from replicated site
+echo "Stat minio2/test-bucket/encrypted"
+./mc stat minio2/test-bucket/encrypted --encrypt-key "minio2/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json
+stat_out1_rep=$(./mc stat minio2/test-bucket/encrypted --encrypt-key "minio2/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json)
+rep_obj1_etag=$(echo "${stat_out1_rep}" | jq '.etag')
+rep_obj1_size=$(echo "${stat_out1_rep}" | jq '.size')
+rep_obj1_md5=$(echo "${stat_out1_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
+echo "Stat minio2/test-bucket/defpartsize"
+./mc stat minio2/test-bucket/defpartsize --encrypt-key "minio2/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json
+stat_out2_rep=$(./mc stat minio2/test-bucket/defpartsize --encrypt-key "minio2/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json)
+rep_obj2_etag=$(echo "${stat_out2_rep}" | jq '.etag')
+rep_obj2_size=$(echo "${stat_out2_rep}" | jq '.size')
+rep_obj2_md5=$(echo "${stat_out2_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
+echo "Stat minio2/test-bucket/custpartsize"
+./mc stat minio2/test-bucket/custpartsize --encrypt-key "minio2/test-bucket/custpartsize=iliketobecrazybutnotsomuchreally" --insecure --json
+stat_out3_rep=$(./mc stat minio2/test-bucket/custpartsize --encrypt-key "minio2/test-bucket/custpartsize=iliketobecrazybutnotsomuchreally" --insecure --json)
+rep_obj3_etag=$(echo "${stat_out3_rep}" | jq '.etag')
+rep_obj3_size=$(echo "${stat_out3_rep}" | jq '.size')
+rep_obj3_md5=$(echo "${stat_out3_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
+
+# Check the etag and size of replicated SSEC objects
+if [ "${rep_obj1_etag}" != "${src_obj1_etag}" ]; then
+	echo "BUG: Etag: '${rep_obj1_etag}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_etag}'"
+	exit_1
+fi
+if [ "${rep_obj1_size}" != "${src_obj1_size}" ]; then
+	echo "BUG: Size: '${rep_obj1_size}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_size}'"
+	exit_1
+fi
+if [ "${rep_obj2_etag}" != "${src_obj2_etag}" ]; then
+	echo "BUG: Etag: '${rep_obj2_etag}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_etag}'"
+	exit_1
+fi
+if [ "${rep_obj2_size}" != "${src_obj2_size}" ]; then
+	echo "BUG: Size: '${rep_obj2_size}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_size}'"
+	exit_1
+fi
+if [ "${rep_obj3_etag}" != "${src_obj3_etag}" ]; then
+	echo "BUG: Etag: '${rep_obj3_etag}' of replicated object: 'minio2/test-bucket/custpartsize' doesn't match with source value: '${src_obj3_etag}'"
+	exit_1
+fi
+if [ "${rep_obj3_size}" != "${src_obj3_size}" ]; then
+	echo "BUG: Size: '${rep_obj3_size}' of replicated object: 'minio2/test-bucket/custpartsize' doesn't match with source value: '${src_obj3_size}'"
+	exit_1
+fi
+
+# Check content of replicated SSEC objects
+./mc cat minio2/test-bucket/encrypted --encrypt-key "minio2/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure
+./mc cat minio2/test-bucket/defpartsize --encrypt-key "minio2/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure >/dev/null || exit_1
+./mc cat minio2/test-bucket/custpartsize --encrypt-key "minio2/test-bucket/custpartsize=iliketobecrazybutnotsomuchreally" --insecure >/dev/null || exit_1
+
+# Check the MD5 checksums of encrypted objects from source and target
+if [ "${src_obj1_md5}" != "${rep_obj1_md5}" ]; then
+	echo "BUG: MD5 checksum of object 'minio2/test-bucket/encrypted' doesn't match with source. Expected: '${src_obj1_md5}', Found: '${rep_obj1_md5}'"
+	exit_1
+fi
+if [ "${src_obj2_md5}" != "${rep_obj2_md5}" ]; then
+	echo "BUG: MD5 checksum of object 'minio2/test-bucket/defpartsize' doesn't match with source. Expected: '${src_obj2_md5}', Found: '${rep_obj2_md5}'"
+	exit_1
+fi
+if [ "${src_obj3_md5}" != "${rep_obj3_md5}" ]; then
+	echo "BUG: MD5 checksum of object 'minio2/test-bucket/custpartsize' doesn't match with source. Expected: '${src_obj3_md5}', Found: '${rep_obj3_md5}'"
+	exit_1
+fi
+
+cleanup

internal/bucket/replication/replication.go

@@ -220,9 +220,6 @@ func (c Config) GetDestination() Destination {
 
 // Replicate returns true if the object should be replicated.
 func (c Config) Replicate(obj ObjectOpts) bool {
-	if obj.SSEC {
-		return false
-	}
 	for _, rule := range c.FilterActionableRules(obj) {
 		if rule.Status == Disabled {
 			continue

internal/bucket/replication/replication_test.go

@@ -250,12 +250,12 @@ func TestReplicate(t *testing.T) {
 		{ObjectOpts{Name: "c1test"}, cfgs[0], true},                   // 2. valid ObjectOpts passing empty Filter
 		{ObjectOpts{Name: "c1test", VersionID: "vid"}, cfgs[0], true}, // 3. valid ObjectOpts passing empty Filter
 
-		{ObjectOpts{Name: "c1test", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[0], true},                                // 4. DeleteMarker version replication valid case - matches DeleteMarkerReplication status
+		{ObjectOpts{Name: "c1test", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[0], true},                               // 4. DeleteMarker version replication valid case - matches DeleteMarkerReplication status
-		{ObjectOpts{Name: "c1test", VersionID: "vid", OpType: DeleteReplicationType}, cfgs[0], true},                                  // 5. permanent delete of version, matches DeleteReplication status - valid case
+		{ObjectOpts{Name: "c1test", VersionID: "vid", OpType: DeleteReplicationType}, cfgs[0], true},                                 // 5. permanent delete of version, matches DeleteReplication status - valid case
-		{ObjectOpts{Name: "c1test", VersionID: "vid", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[0], true},              // 6. permanent delete of version, matches DeleteReplication status
+		{ObjectOpts{Name: "c1test", VersionID: "vid", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[0], true},             // 6. permanent delete of version, matches DeleteReplication status
-		{ObjectOpts{Name: "c1test", VersionID: "vid", DeleteMarker: true, SSEC: true, OpType: DeleteReplicationType}, cfgs[0], false}, // 7. permanent delete of version, disqualified by SSE-C
+		{ObjectOpts{Name: "c1test", VersionID: "vid", DeleteMarker: true, SSEC: true, OpType: DeleteReplicationType}, cfgs[0], true}, // 7. permanent delete of version
-		{ObjectOpts{Name: "c1test", DeleteMarker: true, SSEC: true, OpType: DeleteReplicationType}, cfgs[0], false},                   // 8. setting DeleteMarker on SSE-C encrypted object, disqualified by SSE-C
+		{ObjectOpts{Name: "c1test", DeleteMarker: true, SSEC: true, OpType: DeleteReplicationType}, cfgs[0], true},                   // 8. setting DeleteMarker on SSE-C encrypted object
-		{ObjectOpts{Name: "c1test", SSEC: true}, cfgs[0], false},                                                                      // 9. replication of SSE-C encrypted object, disqualified
+		{ObjectOpts{Name: "c1test", SSEC: true}, cfgs[0], true},                                                                      // 9. replication of SSE-C encrypted object
 
 		//  using config 2 - no filters, only replication of object, metadata enabled
 		{ObjectOpts{Name: "c2test"}, cfgs[1], true},                                                                                   // 10. valid ObjectOpts passing empty Filter
@@ -264,7 +264,7 @@ func TestReplicate(t *testing.T) {
 		{ObjectOpts{Name: "c2test", VersionID: "vid", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[1], false},             // 13. permanent delete of DeleteMarker version, disallowed by DeleteReplication status
 		{ObjectOpts{Name: "c2test", VersionID: "vid", DeleteMarker: true, SSEC: true, OpType: DeleteReplicationType}, cfgs[1], false}, // 14. permanent delete of version, disqualified by SSE-C & DeleteReplication status
 		{ObjectOpts{Name: "c2test", DeleteMarker: true, SSEC: true, OpType: DeleteReplicationType}, cfgs[1], false},                   // 15. setting DeleteMarker on SSE-C encrypted object, disqualified by SSE-C & DeleteMarkerReplication status
-		{ObjectOpts{Name: "c2test", SSEC: true}, cfgs[1], false},                                                                      // 16. replication of SSE-C encrypted object, disqualified by default
+		{ObjectOpts{Name: "c2test", SSEC: true}, cfgs[1], true},                                                                       // 16. replication of SSE-C encrypted object
 		// using config 2 - has more than one rule with overlapping prefixes
 		{ObjectOpts{Name: "xy/c3test", UserTags: "k1=v1"}, cfgs[2], true},                                                                       // 17. matches rule 1 for replication of content/metadata
 		{ObjectOpts{Name: "xyz/c3test", UserTags: "k1=v1"}, cfgs[2], true},                                                                      // 18. matches rule 1 for replication of content/metadata

internal/crypto/error.go

@@ -82,6 +82,8 @@ var (
 	// ErrIncompatibleEncryptionMethod indicates that both SSE-C headers and SSE-S3 headers were specified, and are incompatible
 	// The client needs to remove the SSE-S3 header or the SSE-C headers
 	ErrIncompatibleEncryptionMethod = Errorf("Server side encryption specified with both SSE-C and SSE-S3 headers")
+	// ErrIncompatibleEncryptionWithCompression indicates that both data compression and SSE-C not allowed at the same time
+	ErrIncompatibleEncryptionWithCompression = Errorf("Server side encryption specified with SSE-C with compression not allowed")
 
 	// ErrInvalidEncryptionKeyID returns error when KMS key id contains invalid characters
 	ErrInvalidEncryptionKeyID = Errorf("KMS KeyID contains unsupported characters")

internal/http/headers.go

@@ -233,6 +233,9 @@ const (
 	// Header indicates a Tag operation was performed on one/more peers successfully, though the
 	// current cluster does not have the object yet. This is in a site/bucket replication scenario.
 	MinIOTaggingProxied = "X-Minio-Tagging-Proxied"
+	// Header indicates the actual replicated object size
+	// In case of SSEC objects getting replicated (multipart) actual size would be needed at target
+	MinIOReplicationActualObjectSize = "X-Minio-Replication-Actual-Object-Size"
 
 	// predicted date/time of transition
 	MinIOTransition            = "X-Minio-Transition"