do not save plain-text ETag when encryption is requested (#17427)

fixes an issue under bucket replication could cause
ETags for replicated SSE-S3 single part PUT objects,
to fail as we would attempt a decryption while listing,
or stat() operation.

Makefile

@@ -74,6 +74,9 @@ test-iam: build ## verify IAM (external IDP, etcd backends)
 	@echo "Running tests for IAM (external IDP, etcd backends) with -race"
 	@MINIO_API_REQUESTS_MAX=10000 GORACE=history_size=7 CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd
 
+test-sio-error:
+	@(env bash $(PWD)/docs/bucket/replication/sio-error.sh)
+
 test-replication-2site:
 	@(env bash $(PWD)/docs/bucket/replication/setup_2site_existing_replication.sh)
 
@@ -83,7 +86,7 @@ test-replication-3site:
 test-delete-replication:
 	@(env bash $(PWD)/docs/bucket/replication/delete-replication.sh)
 
-test-replication: install test-replication-2site test-replication-3site test-delete-replication ## verify multi site replication
+test-replication: install test-replication-2site test-replication-3site test-delete-replication test-sio-error ## verify multi site replication
 	@echo "Running tests for replicating three sites"
 
 test-site-replication-ldap: install ## verify automatic site replication

cmd/erasure-object.go

@@ -37,6 +37,7 @@ import (
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
@@ -1263,8 +1264,16 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	}
 
 	userDefined["etag"] = r.MD5CurrentHexString()
+	kind, _ := crypto.IsEncrypted(userDefined)
 	if opts.PreserveETag != "" {
+		if !opts.ReplicationRequest {
 			userDefined["etag"] = opts.PreserveETag
+		} else if kind != crypto.S3 {
+			// if we have a replication request
+			// and SSE-S3 is specified do not preserve
+			// the incoming etag.
+			userDefined["etag"] = opts.PreserveETag
+		}
 	}
 
 	// Guess content-type from the extension if possible.

docs/bucket/replication/sio-error.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -x
+
+export CI=1
+
+make || exit -1
+
+killall -9 minio
+
+rm -rf /tmp/xl/
+mkdir -p /tmp/xl/1/ /tmp/xl/2/
+
+export MINIO_KMS_SECRET_KEY="my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
+
+NODES=4
+
+args1=()
+args2=()
+for i in $(seq 1 $NODES); do
+	args1+=("http://localhost:$((9000 + i))/tmp/xl/1/$i ")
+	args2+=("http://localhost:$((9100 + i))/tmp/xl/2/$i ")
+done
+
+for i in $(seq 1 $NODES); do
+	./minio server --address "127.0.0.1:$((9000 + i))" ${args1[@]} & # | tee /tmp/minio/node.$i &
+	./minio server --address "127.0.0.1:$((9100 + i))" ${args2[@]} & # | tee /tmp/minio/node.$i &
+done
+
+sleep 10
+
+./mc alias set myminio1 http://localhost:9001 minioadmin minioadmin
+./mc alias set myminio2 http://localhost:9101 minioadmin minioadmin
+
+sleep 1
+
+./mc mb myminio1/testbucket/ --with-lock
+./mc mb myminio2/testbucket/ --with-lock
+
+./mc encrypt set sse-s3 my-minio-key myminio1/testbucket/
+./mc encrypt set sse-s3 my-minio-key myminio2/testbucket/
+
+./mc replicate add myminio1/testbucket --remote-bucket http://minioadmin:minioadmin@localhost:9101/testbucket --priority 1
+./mc replicate add myminio2/testbucket --remote-bucket http://minioadmin:minioadmin@localhost:9001/testbucket --priority 1
+
+sleep 1
+
+./mc cp internal.tar myminio1/testbucket/dir/1.tar
+./mc cp internal.tar myminio2/testbucket/dir/2.tar
+
+sleep 1
+
+./mc ls -r --versions myminio1/testbucket/dir/ >/tmp/dir_1.txt
+./mc ls -r --versions myminio2/testbucket/dir/ >/tmp/dir_2.txt
+
+out=$(diff -qpruN /tmp/dir_1.txt /tmp/dir_2.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+	echo "BUG: expected no 'diff' after replication: $out"
+	exit 1
+fi