MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-24 06:05:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

Enable replication of SSE-C objects (#19107)

Browse Source 
If site replication enabled across sites, replicate the SSE-C
objects as well. These objects could be read from target sites
using the same client encryption keys.

Signed-off-by: Shubhendu Ram Tripathi <shubhendu@minio.io>
This commit is contained in:
Shubhendu 2024-03-28 23:14:56 +05:30 committed by GitHub
parent d87f91720b
commit 468a9fae83
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 854 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Makefile
View File

@ -107,6 +107,12 @@ test-site-replication-oidc: install-race ## verify automatic site replication
test-site-replication-minio: install-race ## verify automatic site replication
@echo "Running tests for automatic site replication of IAM (with MinIO IDP)"
@(env bash $(PWD)/docs/site-replication/run-multi-site-minio-idp.sh)
@echo "Running tests for automatic site replication of SSE-C objects"
@(env bash $(PWD)/docs/site-replication/run-ssec-object-replication.sh)
@echo "Running tests for automatic site replication of SSE-C objects with SSE-KMS enabled for bucket"
@(env bash $(PWD)/docs/site-replication/run-sse-kms-object-replication.sh)
@echo "Running tests for automatic site replication of SSE-C objects with compression enabled for site"
@(env bash $(PWD)/docs/site-replication/run-ssec-object-replication-with-compression.sh)
verify: ## verify minio various setups
@echo "Verifying build with race"

5
cmd/bucket-handlers.go
View File

@ -1218,11 +1218,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
return
}
if crypto.SSEC.IsRequested(r.Header) && isReplicationEnabled(ctx, bucket) {
writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParametersSSEC), r.URL)
return
}
var (
reader io.Reader
keyID string

72
cmd/bucket-replication.go
View File

@ -51,6 +51,8 @@ import (
"github.com/minio/minio/internal/logger"
"github.com/tinylib/msgp/msgp"
"github.com/zeebo/xxh3"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
)
const (
@ -76,11 +78,6 @@ const (
ReplicationWorkerMultiplier = 1.5
)
func isReplicationEnabled(ctx context.Context, bucketName string) bool {
rc, _ := getReplicationConfig(ctx, bucketName)
return rc != nil
}
// gets replication config associated to a given bucket name.
func getReplicationConfig(ctx context.Context, bucketName string) (rc *replication.Config, err error) {
rCfg, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucketName)
@ -764,13 +761,20 @@ func (m caseInsensitiveMap) Lookup(key string) (string, bool) {
func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (putOpts minio.PutObjectOptions, err error) {
meta := make(map[string]string)
for k, v := range objInfo.UserDefined {
if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
continue
// In case of SSE-C objects copy the allowed internal headers as well
if !crypto.SSEC.IsEncrypted(objInfo.UserDefined) || !slices.Contains(maps.Keys(validSSEReplicationHeaders), k) {
if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
continue
}
if isStandardHeader(k) {
continue
}
}
if isStandardHeader(k) {
continue
if slices.Contains(maps.Keys(validSSEReplicationHeaders), k) {
meta[validSSEReplicationHeaders[k]] = v
} else {
meta[k] = v
}
meta[k] = v
}
if sc == "" && (objInfo.StorageClass == storageclass.STANDARD || objInfo.StorageClass == storageclass.RRS) {
@ -1166,9 +1170,10 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object)
gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, ObjectOptions{
VersionID: ri.VersionID,
Versioned: versioned,
VersionSuspended: versionSuspended,
VersionID: ri.VersionID,
Versioned: versioned,
VersionSuspended: versionSuspended,
ReplicationRequest: true,
})
if err != nil {
if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
@ -1322,11 +1327,13 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
versioned := globalBucketVersioningSys.PrefixEnabled(bucket, object)
versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object)
gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, ObjectOptions{
VersionID: ri.VersionID,
Versioned: versioned,
VersionSuspended: versionSuspended,
})
gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{},
ObjectOptions{
VersionID: ri.VersionID,
Versioned: versioned,
VersionSuspended: versionSuspended,
ReplicationRequest: true,
})
if err != nil {
if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
objInfo := ri.ToObjectInfo()
@ -1344,6 +1351,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
defer gr.Close()
objInfo := gr.ObjInfo
// make sure we have the latest metadata for metrics calculation
rinfo.PrevReplicationStatus = objInfo.TargetReplicationStatus(tgt.ARN)
@ -1367,6 +1375,11 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
return
}
// Set the encrypted size for SSE-C objects
if crypto.SSEC.IsEncrypted(objInfo.UserDefined) {
size = objInfo.Size
}
if tgt.Bucket == "" {
logger.LogIf(ctx, fmt.Errorf("unable to replicate object %s(%s) to %s, target bucket is missing", objInfo.Name, objInfo.VersionID, tgt.EndpointURL()))
sendEvent(eventArgs{
@ -1599,21 +1612,35 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
pInfo minio.ObjectPart
)
var objectSize int64
for _, partInfo := range objInfo.Parts {
hr, err = hash.NewReader(ctx, io.LimitReader(r, partInfo.ActualSize), partInfo.ActualSize, "", "", partInfo.ActualSize)
if crypto.SSEC.IsEncrypted(objInfo.UserDefined) {
hr, err = hash.NewReader(ctx, io.LimitReader(r, partInfo.Size), partInfo.Size, "", "", partInfo.ActualSize)
} else {
hr, err = hash.NewReader(ctx, io.LimitReader(r, partInfo.ActualSize), partInfo.ActualSize, "", "", partInfo.ActualSize)
}
if err != nil {
return err
}
cHeader := http.Header{}
cHeader.Add(xhttp.MinIOSourceReplicationRequest, "true")
popts := minio.PutObjectPartOptions{
SSE: opts.ServerSideEncryption,
SSE: opts.ServerSideEncryption,
CustomHeader: cHeader,
}
pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.ActualSize, popts)
if crypto.SSEC.IsEncrypted(objInfo.UserDefined) {
objectSize += partInfo.Size
pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.Size, popts)
} else {
objectSize += partInfo.ActualSize
pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.ActualSize, popts)
}
if err != nil {
return err
}
if pInfo.Size != partInfo.ActualSize {
if !crypto.SSEC.IsEncrypted(objInfo.UserDefined) && pInfo.Size != partInfo.ActualSize {
return fmt.Errorf("Part size mismatch: got %d, want %d", pInfo.Size, partInfo.ActualSize)
}
uploadedParts = append(uploadedParts, minio.CompletePart{
@ -1624,6 +1651,7 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
cctx, ccancel := context.WithTimeout(ctx, 10*time.Minute)
defer ccancel()
_, err = c.CompleteMultipartUpload(cctx, bucket, object, uploadID, uploadedParts, minio.PutObjectOptions{
UserMetadata: map[string]string{validSSEReplicationHeaders[ReservedMetadataPrefix+"Actual-Object-Size"]: objInfo.UserDefined[ReservedMetadataPrefix+"actual-size"]},
Internal: minio.AdvancedPutOptions{
SourceMTime: objInfo.ModTime,
// always set this to distinguish between `mc mirror` replication and serverside

6
cmd/erasure-multipart.go
View File

@ -1255,7 +1255,11 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
}
// Save the consolidated actual size.
fi.Metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(objectActualSize, 10)
if opts.ReplicationRequest {
fi.Metadata[ReservedMetadataPrefix+"actual-size"] = opts.UserDefined["X-Minio-Internal-Actual-Object-Size"]
} else {
fi.Metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(objectActualSize, 10)
}
if opts.DataMovement {
fi.SetDataMov()

5
cmd/erasure-object.go
View File

@ -245,6 +245,11 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
}, toObjectErr(errMethodNotAllowed, bucket, object)
}
// Set NoDecryption for SSE-C objects and if replication request
if crypto.SSEC.IsEncrypted(objInfo.UserDefined) && opts.ReplicationRequest {
opts.NoDecryption = true
}
if objInfo.IsRemote() {
gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, h, objInfo, opts)
if err != nil {

5
cmd/generic-handlers.go
View File

@ -33,6 +33,8 @@ import (
"github.com/minio/minio-go/v7/pkg/set"
"github.com/minio/minio/internal/grid"
xnet "github.com/minio/pkg/v2/net"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
"github.com/minio/minio/internal/amztime"
"github.com/minio/minio/internal/config/dns"
@ -73,6 +75,9 @@ const (
// and must not set by clients
func containsReservedMetadata(header http.Header) bool {
for key := range header {
if slices.Contains(maps.Keys(validSSEReplicationHeaders), key) {
return false
}
if stringsHasPrefixFold(key, ReservedMetadataPrefix) {
return true
}

6
cmd/generic-handlers_test.go
View File

@ -108,15 +108,15 @@ var containsReservedMetadataTests = []struct {
},
{
header: http.Header{crypto.MetaIV: []string{"iv"}},
shouldFail: true,
shouldFail: false,
},
{
header: http.Header{crypto.MetaAlgorithm: []string{crypto.InsecureSealAlgorithm}},
shouldFail: true,
shouldFail: false,
},
{
header: http.Header{crypto.MetaSealedKeySSEC: []string{"mac"}},
shouldFail: true,
shouldFail: false,
},
{
header: http.Header{ReservedMetadataPrefix + "Key": []string{"value"}},

33
cmd/handler-utils.go
View File

@ -33,6 +33,8 @@ import (
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/mcontext"
xnet "github.com/minio/pkg/v2/net"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
)
const (
@ -82,6 +84,31 @@ var supportedHeaders = []string{
xhttp.AmzObjectTagging,
"expires",
xhttp.AmzBucketReplicationStatus,
"X-Minio-Replication-Server-Side-Encryption-Sealed-Key",
"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm",
"X-Minio-Replication-Server-Side-Encryption-Iv",
"X-Minio-Replication-Encrypted-Multipart",
"X-Minio-Replication-Actual-Object-Size",
// Add more supported headers here.
}
// mapping of internal headers to allowed replication headers
var validSSEReplicationHeaders = map[string]string{
"X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "X-Minio-Replication-Server-Side-Encryption-Sealed-Key",
"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm",
"X-Minio-Internal-Server-Side-Encryption-Iv": "X-Minio-Replication-Server-Side-Encryption-Iv",
"X-Minio-Internal-Encrypted-Multipart": "X-Minio-Replication-Encrypted-Multipart",
"X-Minio-Internal-Actual-Object-Size": "X-Minio-Replication-Actual-Object-Size",
// Add more supported headers here.
}
// mapping of replication headers to internal headers
var replicationToInternalHeaders = map[string]string{
"X-Minio-Replication-Server-Side-Encryption-Sealed-Key": "X-Minio-Internal-Server-Side-Encryption-Sealed-Key",
"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm": "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm",
"X-Minio-Replication-Server-Side-Encryption-Iv": "X-Minio-Internal-Server-Side-Encryption-Iv",
"X-Minio-Replication-Encrypted-Multipart": "X-Minio-Internal-Encrypted-Multipart",
"X-Minio-Replication-Actual-Object-Size": "X-Minio-Internal-Actual-Object-Size",
// Add more supported headers here.
}
@ -178,7 +205,11 @@ func extractMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[
for _, supportedHeader := range supportedHeaders {
value, ok := nv[http.CanonicalHeaderKey(supportedHeader)]
if ok {
m[supportedHeader] = strings.Join(value, ",")
if slices.Contains(maps.Keys(replicationToInternalHeaders), supportedHeader) {
m[replicationToInternalHeaders[supportedHeader]] = strings.Join(value, ",")
} else {
m[supportedHeader] = strings.Join(value, ",")
}
}
}

5
cmd/object-api-options.go
View File

@ -480,7 +480,7 @@ func completeMultipartOpts(ctx context.Context, r *http.Request, bucket, object
}
opts.MTime = mtime
opts.UserDefined = make(map[string]string)
opts.UserDefined[ReservedMetadataPrefix+"Actual-Object-Size"] = r.Header.Get(xhttp.MinIOReplicationActualObjectSize)
// Transfer SSEC key in opts.EncryptFn
if crypto.SSEC.IsRequested(r.Header) {
key, err := ParseSSECustomerRequest(r)
@ -491,5 +491,8 @@ func completeMultipartOpts(ctx context.Context, r *http.Request, bucket, object
}
}
}
if _, ok := r.Header[xhttp.MinIOSourceReplicationRequest]; ok {
opts.ReplicationRequest = true
}
return opts, nil
}

8
cmd/object-handlers.go
View File

@ -2234,8 +2234,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
return
}
if crypto.SSEC.IsRequested(r.Header) && isReplicationEnabled(ctx, bucket) {
writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParametersSSEC), r.URL)
if crypto.SSEC.IsRequested(r.Header) && isCompressible(r.Header, object) {
writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionWithCompression), r.URL)
return
}
@ -2649,10 +2649,6 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
return errInvalidEncryptionParameters
}
if crypto.SSEC.IsRequested(r.Header) && isReplicationEnabled(ctx, bucket) {
return errInvalidEncryptionParametersSSEC
}
reader, objectEncryptionKey, err = EncryptRequest(hashReader, r, bucket, object, metadata)
if err != nil {
return err

115
cmd/object-multipart-handlers.go
View File

@ -116,14 +116,29 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
return
}
if crypto.SSEC.IsRequested(r.Header) && isReplicationEnabled(ctx, bucket) {
writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParametersSSEC), r.URL)
if crypto.SSEC.IsRequested(r.Header) && isCompressible(r.Header, object) {
writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionWithCompression), r.URL)
return
}
if err = setEncryptionMetadata(r, bucket, object, encMetadata); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
_, sourceReplReq := r.Header[xhttp.MinIOSourceReplicationRequest]
ssecRepHeaders := []string{
"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm",
"X-Minio-Replication-Server-Side-Encryption-Sealed-Key",
"X-Minio-Replication-Server-Side-Encryption-Iv",
}
ssecRep := false
for _, header := range ssecRepHeaders {
if val := r.Header.Get(header); val != "" {
ssecRep = true
break
}
}
if !(ssecRep && sourceReplReq) {
if err = setEncryptionMetadata(r, bucket, object, encMetadata); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
}
// Set this for multipart only operations, we need to differentiate during
// decryption if the file was actually multipart or not.
@ -757,9 +772,10 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
pReader := NewPutObjReader(hashReader)
_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
_, replicationStatus := mi.UserDefined[xhttp.AmzBucketReplicationStatus]
var objectEncryptionKey crypto.ObjectKey
if isEncrypted {
if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(mi.UserDefined) {
if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(mi.UserDefined) && !replicationStatus {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrSSEMultipartEncrypted), r.URL)
return
}
@ -779,52 +795,55 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
}
}
// Calculating object encryption key
key, err = decryptObjectMeta(key, bucket, object, mi.UserDefined)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
copy(objectEncryptionKey[:], key)
_, sourceReplReq := r.Header[xhttp.MinIOSourceReplicationRequest]
if !(sourceReplReq && crypto.SSEC.IsEncrypted(mi.UserDefined)) {
// Calculating object encryption key
key, err = decryptObjectMeta(key, bucket, object, mi.UserDefined)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
copy(objectEncryptionKey[:], key)
partEncryptionKey := objectEncryptionKey.DerivePartKey(uint32(partID))
in := io.Reader(hashReader)
if size > encryptBufferThreshold {
// The encryption reads in blocks of 64KB.
// We add a buffer on bigger files to reduce the number of syscalls upstream.
in = bufio.NewReaderSize(hashReader, encryptBufferSize)
}
reader, err = sio.EncryptReader(in, sio.Config{Key: partEncryptionKey[:], CipherSuites: fips.DARECiphers()})
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
wantSize := int64(-1)
if size >= 0 {
info := ObjectInfo{Size: size}
wantSize = info.EncryptedSize()
}
// do not try to verify encrypted content
hashReader, err = hash.NewReader(ctx, etag.Wrap(reader, hashReader), wantSize, "", "", actualSize)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if err := hashReader.AddChecksum(r, true); err != nil {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
return
}
partEncryptionKey := objectEncryptionKey.DerivePartKey(uint32(partID))
in := io.Reader(hashReader)
if size > encryptBufferThreshold {
// The encryption reads in blocks of 64KB.
// We add a buffer on bigger files to reduce the number of syscalls upstream.
in = bufio.NewReaderSize(hashReader, encryptBufferSize)
}
reader, err = sio.EncryptReader(in, sio.Config{Key: partEncryptionKey[:], CipherSuites: fips.DARECiphers()})
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
wantSize := int64(-1)
if size >= 0 {
info := ObjectInfo{Size: size}
wantSize = info.EncryptedSize()
}
// do not try to verify encrypted content
hashReader, err = hash.NewReader(ctx, etag.Wrap(reader, hashReader), wantSize, "", "", actualSize)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if err := hashReader.AddChecksum(r, true); err != nil {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
return
}
pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if idxCb != nil {
idxCb = compressionIndexEncrypter(objectEncryptionKey, idxCb)
if idxCb != nil {
idxCb = compressionIndexEncrypter(objectEncryptionKey, idxCb)
}
opts.EncryptFn = metadataEncrypter(objectEncryptionKey)
}
opts.EncryptFn = metadataEncrypter(objectEncryptionKey)
}
opts.IndexCB = idxCb

40
docs/bucket/versioning/versioning-tests.sh
View File

@ -10,10 +10,10 @@ trap 'catch $LINENO' ERR
catch() {
if [ $# -ne 0 ]; then
echo "error on line $1"
echo "$site server logs ========="
cat "/tmp/${site}_1.log"
echo "server logs ========="
cat "/tmp/sitea_1.log"
echo "==========================="
cat "/tmp/${site}_2.log"
cat "/tmp/sitea_2.log"
fi
echo "Cleaning up instances of MinIO"
@ -42,32 +42,34 @@ if [ ! -f ./mc ]; then
chmod +x mc
fi
minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
"http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 &
minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
"http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 &
minio server --address ":9001" "https://localhost:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
"https://localhost:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 &
minio server --address ":9002" "https://localhost:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
"https://localhost:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 &
export MC_HOST_sitea=http://minio:minio123@127.0.0.1:9001
sleep 60
./mc mb sitea/delissue
export MC_HOST_sitea=https://minio:minio123@localhost:9001
./mc version enable sitea/delissue
./mc mb sitea/delissue --insecure
echo hello | ./mc pipe sitea/delissue/hello
./mc version enable sitea/delissue --insecure
./mc version suspend sitea/delissue
echo hello | ./mc pipe sitea/delissue/hello --insecure
./mc rm sitea/delissue/hello
./mc version suspend sitea/delissue --insecure
./mc version enable sitea/delissue
./mc rm sitea/delissue/hello --insecure
echo hello | ./mc pipe sitea/delissue/hello
./mc version enable sitea/delissue --insecure
./mc version suspend sitea/delissue
echo hello | ./mc pipe sitea/delissue/hello --insecure
./mc rm sitea/delissue/hello
./mc version suspend sitea/delissue --insecure
count=$(./mc ls --versions sitea/delissue | wc -l)
./mc rm sitea/delissue/hello --insecure
count=$(./mc ls --versions sitea/delissue --insecure | wc -l)
if [ ${count} -ne 3 ]; then
echo "BUG: expected number of versions to be '3' found ${count}"
@ -76,6 +78,6 @@ if [ ${count} -ne 3 ]; then
fi
echo "SUCCESS:"
./mc ls --versions sitea/delissue
./mc ls --versions sitea/delissue --insecure
catch

230
docs/site-replication/run-sse-kms-object-replication.sh Executable file
View File

@ -0,0 +1,230 @@
#!/usr/bin/env bash
# shellcheck disable=SC2120
exit_1() {
cleanup
echo "minio1 ============"
cat /tmp/minio1_1.log
echo "minio2 ============"
cat /tmp/minio2_1.log
exit 1
}
cleanup() {
echo -n "Cleaning up instances of MinIO ..."
pkill -9 minio || sudo pkill -9 minio
pkill -9 kes || sudo pkill -9 kes
rm -rf ${PWD}/keys
rm -rf /tmp/minio{1,2}
echo "done"
}
cleanup
export MINIO_CI_CD=1
export MINIO_BROWSER=off
export MINIO_ROOT_USER="minio"
export MINIO_ROOT_PASSWORD="minio123"
# Create certificates for TLS enabled MinIO
echo -n "Setup certs for MinIO instances ..."
wget -O certgen https://github.com/minio/certgen/releases/latest/download/certgen-linux-amd64 && chmod +x certgen
./certgen --host localhost
mkdir -p ~/.minio/certs
mv public.crt ~/.minio/certs || sudo mv public.crt ~/.minio/certs
mv private.key ~/.minio/certs || sudo mv private.key ~/.minio/certs
echo "done"
# Start MinIO instances
echo -n "Starting MinIO instances ..."
CI=on MINIO_KMS_SECRET_KEY=minio-default-key:IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g= MINIO_ROOT_USER=minio MINIO_ROOT_PASSWORD=minio123 minio server --address ":9001" --console-address ":10000" /tmp/minio1/{1...4}/disk{1...4} /tmp/minio1/{5...8}/disk{1...4} >/tmp/minio1_1.log 2>&1 &
CI=on MINIO_KMS_SECRET_KEY=minio-default-key:IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g= MINIO_ROOT_USER=minio MINIO_ROOT_PASSWORD=minio123 minio server --address ":9002" --console-address ":11000" /tmp/minio2/{1...4}/disk{1...4} /tmp/minio2/{5...8}/disk{1...4} >/tmp/minio2_1.log 2>&1 &
echo "done"
if [ ! -f ./mc ]; then
echo -n "Downloading MinIO client ..."
wget -O mc https://dl.min.io/client/mc/release/linux-amd64/mc &&
chmod +x mc
echo "done"
fi
sleep 10
export MC_HOST_minio1=https://minio:minio123@localhost:9001
export MC_HOST_minio2=https://minio:minio123@localhost:9002
# Prepare data for tests
echo -n "Preparing test data ..."
mkdir -p /tmp/data
echo "Hello from encrypted world" >/tmp/data/encrypted
touch /tmp/data/mpartobj
shred -s 500M /tmp/data/mpartobj
touch /tmp/data/defpartsize
shred -s 500M /tmp/data/defpartsize
touch /tmp/data/custpartsize
shred -s 500M /tmp/data/custpartsize
echo "done"
# Add replication site
./mc admin replicate add minio1 minio2 --insecure
# sleep for replication to complete
sleep 30
# Create bucket in source cluster
echo "Create bucket in source MinIO instance"
./mc mb minio1/test-bucket --insecure
# Enable SSE KMS for the bucket
./mc encrypt set sse-kms minio-default-key minio1/test-bucket --insecure
# Load objects to source site
echo "Loading objects to source MinIO instance"
./mc cp /tmp/data/encrypted minio1/test-bucket --insecure
./mc cp /tmp/data/mpartobj minio1/test-bucket --encrypt-key "minio1/test-bucket/mpartobj=iliketobecrazybutnotsomuchreally" --insecure
./mc cp /tmp/data/defpartsize minio1/test-bucket --insecure
./mc put /tmp/data/custpartsize minio1/test-bucket --insecure --part-size 50MiB
sleep 120
# List the objects from source site
echo "Objects from source instance"
./mc ls minio1/test-bucket --insecure
count1=$(./mc ls minio1/test-bucket/encrypted --insecure | wc -l)
if [ "${count1}" -ne 1 ]; then
echo "BUG: object minio1/test-bucket/encrypted not found"
exit_1
fi
count2=$(./mc ls minio1/test-bucket/mpartobj --insecure | wc -l)
if [ "${count2}" -ne 1 ]; then
echo "BUG: object minio1/test-bucket/mpartobj not found"
exit_1
fi
count3=$(./mc ls minio1/test-bucket/defpartsize --insecure | wc -l)
if [ "${count3}" -ne 1 ]; then
echo "BUG: object minio1/test-bucket/defpartsize not found"
exit_1
fi
count4=$(./mc ls minio1/test-bucket/custpartsize --insecure | wc -l)
if [ "${count4}" -ne 1 ]; then
echo "BUG: object minio1/test-bucket/custpartsize not found"
exit_1
fi
# List the objects from replicated site
echo "Objects from replicated instance"
./mc ls minio2/test-bucket --insecure
repcount1=$(./mc ls minio2/test-bucket/encrypted --insecure | wc -l)
if [ "${repcount1}" -ne 1 ]; then
echo "BUG: object test-bucket/encrypted not replicated"
exit_1
fi
repcount2=$(./mc ls minio2/test-bucket/mpartobj --insecure | wc -l)
if [ "${repcount2}" -ne 1 ]; then
echo "BUG: object test-bucket/mpartobj not replicated"
exit_1
fi
repcount3=$(./mc ls minio2/test-bucket/defpartsize --insecure | wc -l)
if [ "${repcount3}" -ne 1 ]; then
echo "BUG: object test-bucket/defpartsize not replicated"
exit_1
fi
repcount4=$(./mc ls minio2/test-bucket/custpartsize --insecure | wc -l)
if [ "${repcount4}" -ne 1 ]; then
echo "BUG: object test-bucket/custpartsize not replicated"
exit_1
fi
# Stat the objects from source site
echo "Stat minio1/test-bucket/encrypted"
./mc stat minio1/test-bucket/encrypted --insecure --json
stat_out1=$(./mc stat minio1/test-bucket/encrypted --insecure --json)
src_obj1_algo=$(echo "${stat_out1}" | jq '.metadata."X-Amz-Server-Side-Encryption"')
src_obj1_keyid=$(echo "${stat_out1}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"')
echo "Stat minio1/test-bucket/defpartsize"
./mc stat minio1/test-bucket/defpartsize --insecure --json
stat_out2=$(./mc stat minio1/test-bucket/defpartsize --insecure --json)
src_obj2_algo=$(echo "${stat_out2}" | jq '.metadata."X-Amz-Server-Side-Encryption"')
src_obj2_keyid=$(echo "${stat_out2}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"')
echo "Stat minio1/test-bucket/custpartsize"
./mc stat minio1/test-bucket/custpartsize --insecure --json
stat_out3=$(./mc stat minio1/test-bucket/custpartsize --insecure --json)
src_obj3_algo=$(echo "${stat_out3}" | jq '.metadata."X-Amz-Server-Side-Encryption"')
src_obj3_keyid=$(echo "${stat_out3}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"')
echo "Stat minio1/test-bucket/mpartobj"
./mc stat minio1/test-bucket/mpartobj --encrypt-key "minio1/test-bucket/mpartobj=iliketobecrazybutnotsomuchreally" --insecure --json
stat_out4=$(./mc stat minio1/test-bucket/mpartobj --encrypt-key "minio1/test-bucket/mpartobj=iliketobecrazybutnotsomuchreally" --insecure --json)
src_obj4_etag=$(echo "${stat_out4}" | jq '.etag')
src_obj4_size=$(echo "${stat_out4}" | jq '.size')
src_obj4_md5=$(echo "${stat_out4}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
# Stat the objects from replicated site
echo "Stat minio2/test-bucket/encrypted"
./mc stat minio2/test-bucket/encrypted --insecure --json
stat_out1_rep=$(./mc stat minio2/test-bucket/encrypted --insecure --json)
rep_obj1_algo=$(echo "${stat_out1_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption"')
rep_obj1_keyid=$(echo "${stat_out1_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"')
echo "Stat minio2/test-bucket/defpartsize"
./mc stat minio2/test-bucket/defpartsize --insecure --json
stat_out2_rep=$(./mc stat minio2/test-bucket/defpartsize --insecure --json)
rep_obj2_algo=$(echo "${stat_out2_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption"')
rep_obj2_keyid=$(echo "${stat_out2_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"')
echo "Stat minio2/test-bucket/custpartsize"
./mc stat minio2/test-bucket/custpartsize --insecure --json
stat_out3_rep=$(./mc stat minio2/test-bucket/custpartsize --insecure --json)
rep_obj3_algo=$(echo "${stat_out3_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption"')
rep_obj3_keyid=$(echo "${stat_out3_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"')
echo "Stat minio2/test-bucket/mpartobj"
./mc stat minio2/test-bucket/mpartobj --encrypt-key "minio2/test-bucket/mpartobj=iliketobecrazybutnotsomuchreally" --insecure --json
stat_out4_rep=$(./mc stat minio2/test-bucket/mpartobj --encrypt-key "minio2/test-bucket/mpartobj=iliketobecrazybutnotsomuchreally" --insecure --json)
rep_obj4_etag=$(echo "${stat_out4}" | jq '.etag')
rep_obj4_size=$(echo "${stat_out4}" | jq '.size')
rep_obj4_md5=$(echo "${stat_out4}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
# Check the algo and keyId of replicated objects
if [ "${rep_obj1_algo}" != "${src_obj1_algo}" ]; then
echo "BUG: Algorithm: '${rep_obj1_algo}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_algo}'"
exit_1
fi
if [ "${rep_obj1_keyid}" != "${src_obj1_keyid}" ]; then
echo "BUG: KeyId: '${rep_obj1_keyid}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_keyid}'"
exit_1
fi
if [ "${rep_obj2_algo}" != "${src_obj2_algo}" ]; then
echo "BUG: Algorithm: '${rep_obj2_algo}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_algo}'"
exit_1
fi
if [ "${rep_obj2_keyid}" != "${src_obj2_keyid}" ]; then
echo "BUG: KeyId: '${rep_obj2_keyid}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_keyid}'"
exit_1
fi
if [ "${rep_obj3_algo}" != "${src_obj3_algo}" ]; then
echo "BUG: Algorithm: '${rep_obj3_algo}' of replicated object: 'minio2/test-bucket/custpartsize' doesn't match with source value: '${src_obj3_algo}'"
exit_1
fi
if [ "${rep_obj3_keyid}" != "${src_obj3_keyid}" ]; then
echo "BUG: KeyId: '${rep_obj3_keyid}' of replicated object: 'minio2/test-bucket/custpartsize' doesn't match with source value: '${src_obj3_keyid}'"
exit_1
fi
# Check the etag, size and md5 of replicated SSEC object
if [ "${rep_obj4_etag}" != "${src_obj4_etag}" ]; then
echo "BUG: Etag: '${rep_obj4_etag}' of replicated object: 'minio2/test-bucket/mpartobj' doesn't match with source value: '${src_obj4_etag}'"
exit_1
fi
if [ "${rep_obj4_size}" != "${src_obj4_size}" ]; then
echo "BUG: Size: '${rep_obj4_size}' of replicated object: 'minio2/test-bucket/mpartobj' doesn't match with source value: '${src_obj4_size}'"
exit_1
fi
if [ "${src_obj4_md5}" != "${rep_obj4_md5}" ]; then
echo "BUG: MD5 checksum of object 'minio2/test-bucket/mpartobj' doesn't match with source. Expected: '${src_obj4_md5}', Found: '${rep_obj4_md5}'"
exit_1
fi
# Check content of replicated objects
./mc cat minio2/test-bucket/encrypted --insecure
./mc cat minio2/test-bucket/mpartobj --encrypt-key "minio2/test-bucket/mpartobj=iliketobecrazybutnotsomuchreally" --insecure >/dev/null || exit_1
./mc cat minio2/test-bucket/defpartsize --insecure >/dev/null || exit_1
./mc cat minio2/test-bucket/custpartsize --insecure >/dev/null || exit_1
cleanup

193
docs/site-replication/run-ssec-object-replication-with-compression.sh Executable file
View File

@ -0,0 +1,193 @@
#!/usr/bin/env bash
# shellcheck disable=SC2120
exit_1() {
cleanup
echo "minio1 ============"
cat /tmp/minio1_1.log
echo "minio2 ============"
cat /tmp/minio2_1.log
exit 1
}
cleanup() {
echo -n "Cleaning up instances of MinIO ..."
pkill minio || sudo pkill minio
pkill -9 minio || sudo pkill -9 minio
rm -rf /tmp/minio{1,2}
echo "done"
}
cleanup
export MINIO_CI_CD=1
export MINIO_BROWSER=off
export MINIO_ROOT_USER="minio"
export MINIO_ROOT_PASSWORD="minio123"
# Create certificates for TLS enabled MinIO
echo -n "Setup certs for MinIO instances ..."
wget -O certgen https://github.com/minio/certgen/releases/latest/download/certgen-linux-amd64 && chmod +x certgen
./certgen --host localhost
mkdir -p ~/.minio/certs
mv public.crt ~/.minio/certs || sudo mv public.crt ~/.minio/certs
mv private.key ~/.minio/certs || sudo mv private.key ~/.minio/certs
echo "done"
# Start MinIO instances
echo -n "Starting MinIO instances ..."
minio server --address ":9001" --console-address ":10000" /tmp/minio1/{1...4}/disk{1...4} /tmp/minio1/{5...8}/disk{1...4} >/tmp/minio1_1.log 2>&1 &
minio server --address ":9002" --console-address ":11000" /tmp/minio2/{1...4}/disk{1...4} /tmp/minio2/{5...8}/disk{1...4} >/tmp/minio2_1.log 2>&1 &
echo "done"
if [ ! -f ./mc ]; then
echo -n "Downloading MinIO client ..."
wget -O mc https://dl.min.io/client/mc/release/linux-amd64/mc &&
chmod +x mc
echo "done"
fi
sleep 10
export MC_HOST_minio1=https://minio:minio123@localhost:9001
export MC_HOST_minio2=https://minio:minio123@localhost:9002
# Prepare data for tests
echo -n "Preparing test data ..."
mkdir -p /tmp/data
echo "Hello world" >/tmp/data/plainfile
echo "Hello from encrypted world" >/tmp/data/encrypted
touch /tmp/data/defpartsize
shred -s 500M /tmp/data/defpartsize
touch /tmp/data/mpartobj.txt
shred -s 500M /tmp/data/mpartobj.txt
echo "done"
# Enable compression for site minio1
./mc admin config set minio1 compression enable=on extensions=".txt" --insecure
./mc admin config set minio1 compression allow_encryption=on --insecure
# Create bucket in source cluster
echo "Create bucket in source MinIO instance"
./mc mb minio1/test-bucket --insecure
# Load objects to source site
echo "Loading objects to source MinIO instance"
./mc cp /tmp/data/plainfile minio1/test-bucket --insecure
./mc cp /tmp/data/encrypted minio1/test-bucket --encrypt-key "minio1/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure
./mc cp /tmp/data/defpartsize minio1/test-bucket --encrypt-key "minio1/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure
# Below should fail as compression and SSEC used at the same time
RESULT=$({ ./mc put /tmp/data/mpartobj.txt minio1/test-bucket --encrypt-key "minio1/test-bucket/mpartobj.txt=iliketobecrazybutnotsomuchreally" --insecure; } 2>&1)
if [[ ${RESULT} != *"Server side encryption specified with SSE-C with compression not allowed"* ]]; then
echo "BUG: Loading an SSE-C object to site with compression should fail. Succeeded though."
exit_1
fi
# Add replication site
./mc admin replicate add minio1 minio2 --insecure
# sleep for replication to complete
sleep 30
# List the objects from source site
echo "Objects from source instance"
./mc ls minio1/test-bucket --insecure
count1=$(./mc ls minio1/test-bucket/plainfile --insecure | wc -l)
if [ "${count1}" -ne 1 ]; then
echo "BUG: object minio1/test-bucket/plainfile not found"
exit_1
fi
count2=$(./mc ls minio1/test-bucket/encrypted --insecure | wc -l)
if [ "${count2}" -ne 1 ]; then
echo "BUG: object minio1/test-bucket/encrypted not found"
exit_1
fi
count3=$(./mc ls minio1/test-bucket/defpartsize --insecure | wc -l)
if [ "${count3}" -ne 1 ]; then
echo "BUG: object minio1/test-bucket/defpartsize not found"
exit_1
fi
sleep 120
# List the objects from replicated site
echo "Objects from replicated instance"
./mc ls minio2/test-bucket --insecure
repcount1=$(./mc ls minio2/test-bucket/plainfile --insecure | wc -l)
if [ "${repcount1}" -ne 1 ]; then
echo "BUG: object test-bucket/plainfile not replicated"
exit_1
fi
repcount2=$(./mc ls minio2/test-bucket/encrypted --insecure | wc -l)
if [ "${repcount2}" -ne 1 ]; then
echo "BUG: object test-bucket/encrypted not replicated"
exit_1
fi
repcount3=$(./mc ls minio2/test-bucket/defpartsize --insecure | wc -l)
if [ "${repcount3}" -ne 1 ]; then
echo "BUG: object test-bucket/defpartsize not replicated"
exit_1
fi
# Stat the SSEC objects from source site
echo "Stat minio1/test-bucket/encrypted"
./mc stat minio1/test-bucket/encrypted --encrypt-key "minio1/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json
stat_out1=$(./mc stat minio1/test-bucket/encrypted --encrypt-key "minio1/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json)
src_obj1_etag=$(echo "${stat_out1}" | jq '.etag')
src_obj1_size=$(echo "${stat_out1}" | jq '.size')
src_obj1_md5=$(echo "${stat_out1}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
echo "Stat minio1/test-bucket/defpartsize"
./mc stat minio1/test-bucket/defpartsize --encrypt-key "minio1/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json
stat_out2=$(./mc stat minio1/test-bucket/defpartsize --encrypt-key "minio1/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json)
src_obj2_etag=$(echo "${stat_out2}" | jq '.etag')
src_obj2_size=$(echo "${stat_out2}" | jq '.size')
src_obj2_md5=$(echo "${stat_out2}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
# Stat the SSEC objects from replicated site
echo "Stat minio2/test-bucket/encrypted"
./mc stat minio2/test-bucket/encrypted --encrypt-key "minio2/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json
stat_out1_rep=$(./mc stat minio2/test-bucket/encrypted --encrypt-key "minio2/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json)
rep_obj1_etag=$(echo "${stat_out1_rep}" | jq '.etag')
rep_obj1_size=$(echo "${stat_out1_rep}" | jq '.size')
rep_obj1_md5=$(echo "${stat_out1_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
echo "Stat minio2/test-bucket/defpartsize"
./mc stat minio2/test-bucket/defpartsize --encrypt-key "minio2/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json
stat_out2_rep=$(./mc stat minio2/test-bucket/defpartsize --encrypt-key "minio2/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json)
rep_obj2_etag=$(echo "${stat_out2_rep}" | jq '.etag')
rep_obj2_size=$(echo "${stat_out2_rep}" | jq '.size')
rep_obj2_md5=$(echo "${stat_out2_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
# Check the etag and size of replicated SSEC objects
if [ "${rep_obj1_etag}" != "${src_obj1_etag}" ]; then
echo "BUG: Etag: '${rep_obj1_etag}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_etag}'"
exit_1
fi
if [ "${rep_obj1_size}" != "${src_obj1_size}" ]; then
echo "BUG: Size: '${rep_obj1_size}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_size}'"
exit_1
fi
if [ "${rep_obj2_etag}" != "${src_obj2_etag}" ]; then
echo "BUG: Etag: '${rep_obj2_etag}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_etag}'"
exit_1
fi
if [ "${rep_obj2_size}" != "${src_obj2_size}" ]; then
echo "BUG: Size: '${rep_obj2_size}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_size}'"
exit_1
fi
# Check content of replicated SSEC objects
./mc cat minio2/test-bucket/encrypted --encrypt-key "minio2/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure
./mc cat minio2/test-bucket/defpartsize --encrypt-key "minio2/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure >/dev/null || exit_1
# Check the MD5 checksums of encrypted objects from source and target
if [ "${src_obj1_md5}" != "${rep_obj1_md5}" ]; then
echo "BUG: MD5 checksum of object 'minio2/test-bucket/encrypted' doesn't match with source. Expected: '${src_obj1_md5}', Found: '${rep_obj1_md5}'"
exit_1
fi
if [ "${src_obj2_md5}" != "${rep_obj2_md5}" ]; then
echo "BUG: MD5 checksum of object 'minio2/test-bucket/defpartsize' doesn't match with source. Expected: '${src_obj2_md5}', Found: '${rep_obj2_md5}'"
exit_1
fi
cleanup

219
docs/site-replication/run-ssec-object-replication.sh Executable file
View File

@ -0,0 +1,219 @@
#!/usr/bin/env bash
# shellcheck disable=SC2120
exit_1() {
cleanup
echo "minio1 ============"
cat /tmp/minio1_1.log
echo "minio2 ============"
cat /tmp/minio2_1.log
exit 1
}
cleanup() {
echo -n "Cleaning up instances of MinIO ..."
pkill minio || sudo pkill minio
pkill -9 minio || sudo pkill -9 minio
rm -rf /tmp/minio{1,2}
echo "done"
}
cleanup
export MINIO_CI_CD=1
export MINIO_BROWSER=off
export MINIO_ROOT_USER="minio"
export MINIO_ROOT_PASSWORD="minio123"
# Create certificates for TLS enabled MinIO
echo -n "Setup certs for MinIO instances ..."
wget -O certgen https://github.com/minio/certgen/releases/latest/download/certgen-linux-amd64 && chmod +x certgen
./certgen --host localhost
mkdir -p ~/.minio/certs
mv public.crt ~/.minio/certs || sudo mv public.crt ~/.minio/certs
mv private.key ~/.minio/certs || sudo mv private.key ~/.minio/certs
echo "done"
# Start MinIO instances
echo -n "Starting MinIO instances ..."
minio server --address ":9001" --console-address ":10000" /tmp/minio1/{1...4}/disk{1...4} /tmp/minio1/{5...8}/disk{1...4} >/tmp/minio1_1.log 2>&1 &
minio server --address ":9002" --console-address ":11000" /tmp/minio2/{1...4}/disk{1...4} /tmp/minio2/{5...8}/disk{1...4} >/tmp/minio2_1.log 2>&1 &
echo "done"
if [ ! -f ./mc ]; then
echo -n "Downloading MinIO client ..."
wget -O mc https://dl.min.io/client/mc/release/linux-amd64/mc &&
chmod +x mc
echo "done"
fi
sleep 10
export MC_HOST_minio1=https://minio:minio123@localhost:9001
export MC_HOST_minio2=https://minio:minio123@localhost:9002
# Prepare data for tests
echo -n "Preparing test data ..."
mkdir -p /tmp/data
echo "Hello world" >/tmp/data/plainfile
echo "Hello from encrypted world" >/tmp/data/encrypted
touch /tmp/data/defpartsize
shred -s 500M /tmp/data/defpartsize
touch /tmp/data/custpartsize
shred -s 500M /tmp/data/custpartsize
echo "done"
# Add replication site
./mc admin replicate add minio1 minio2 --insecure
# sleep for replication to complete
sleep 30
# Create bucket in source cluster
echo "Create bucket in source MinIO instance"
./mc mb minio1/test-bucket --insecure
# Load objects to source site
echo "Loading objects to source MinIO instance"
./mc cp /tmp/data/plainfile minio1/test-bucket --insecure
./mc cp /tmp/data/encrypted minio1/test-bucket --encrypt-key "minio1/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure
./mc cp /tmp/data/defpartsize minio1/test-bucket --encrypt-key "minio1/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure
./mc put /tmp/data/custpartsize minio1/test-bucket --encrypt-key "minio1/test-bucket/custpartsize=iliketobecrazybutnotsomuchreally" --insecure --part-size 50MiB
sleep 120
# List the objects from source site
echo "Objects from source instance"
./mc ls minio1/test-bucket --insecure
count1=$(./mc ls minio1/test-bucket/plainfile --insecure | wc -l)
if [ "${count1}" -ne 1 ]; then
echo "BUG: object minio1/test-bucket/plainfile not found"
exit_1
fi
count2=$(./mc ls minio1/test-bucket/encrypted --insecure | wc -l)
if [ "${count2}" -ne 1 ]; then
echo "BUG: object minio1/test-bucket/encrypted not found"
exit_1
fi
count3=$(./mc ls minio1/test-bucket/defpartsize --insecure | wc -l)
if [ "${count3}" -ne 1 ]; then
echo "BUG: object minio1/test-bucket/defpartsize not found"
exit_1
fi
count4=$(./mc ls minio1/test-bucket/custpartsize --insecure | wc -l)
if [ "${count4}" -ne 1 ]; then
echo "BUG: object minio1/test-bucket/custpartsize not found"
exit_1
fi
# List the objects from replicated site
echo "Objects from replicated instance"
./mc ls minio2/test-bucket --insecure
repcount1=$(./mc ls minio2/test-bucket/plainfile --insecure | wc -l)
if [ "${repcount1}" -ne 1 ]; then
echo "BUG: object test-bucket/plainfile not replicated"
exit_1
fi
repcount2=$(./mc ls minio2/test-bucket/encrypted --insecure | wc -l)
if [ "${repcount2}" -ne 1 ]; then
echo "BUG: object test-bucket/encrypted not replicated"
exit_1
fi
repcount3=$(./mc ls minio2/test-bucket/defpartsize --insecure | wc -l)
if [ "${repcount3}" -ne 1 ]; then
echo "BUG: object test-bucket/defpartsize not replicated"
exit_1
fi
repcount4=$(./mc ls minio2/test-bucket/custpartsize --insecure | wc -l)
if [ "${repcount4}" -ne 1 ]; then
echo "BUG: object test-bucket/custpartsize not replicated"
exit_1
fi
# Stat the SSEC objects from source site
echo "Stat minio1/test-bucket/encrypted"
./mc stat minio1/test-bucket/encrypted --encrypt-key "minio1/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json
stat_out1=$(./mc stat minio1/test-bucket/encrypted --encrypt-key "minio1/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json)
src_obj1_etag=$(echo "${stat_out1}" | jq '.etag')
src_obj1_size=$(echo "${stat_out1}" | jq '.size')
src_obj1_md5=$(echo "${stat_out1}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
echo "Stat minio1/test-bucket/defpartsize"
./mc stat minio1/test-bucket/defpartsize --encrypt-key "minio1/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json
stat_out2=$(./mc stat minio1/test-bucket/defpartsize --encrypt-key "minio1/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json)
src_obj2_etag=$(echo "${stat_out2}" | jq '.etag')
src_obj2_size=$(echo "${stat_out2}" | jq '.size')
src_obj2_md5=$(echo "${stat_out2}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
echo "Stat minio1/test-bucket/custpartsize"
./mc stat minio1/test-bucket/custpartsize --encrypt-key "minio1/test-bucket/custpartsize=iliketobecrazybutnotsomuchreally" --insecure --json
stat_out3=$(./mc stat minio1/test-bucket/custpartsize --encrypt-key "minio1/test-bucket/custpartsize=iliketobecrazybutnotsomuchreally" --insecure --json)
src_obj3_etag=$(echo "${stat_out3}" | jq '.etag')
src_obj3_size=$(echo "${stat_out3}" | jq '.size')
src_obj3_md5=$(echo "${stat_out3}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
# Stat the SSEC objects from replicated site
echo "Stat minio2/test-bucket/encrypted"
./mc stat minio2/test-bucket/encrypted --encrypt-key "minio2/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json
stat_out1_rep=$(./mc stat minio2/test-bucket/encrypted --encrypt-key "minio2/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json)
rep_obj1_etag=$(echo "${stat_out1_rep}" | jq '.etag')
rep_obj1_size=$(echo "${stat_out1_rep}" | jq '.size')
rep_obj1_md5=$(echo "${stat_out1_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
echo "Stat minio2/test-bucket/defpartsize"
./mc stat minio2/test-bucket/defpartsize --encrypt-key "minio2/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json
stat_out2_rep=$(./mc stat minio2/test-bucket/defpartsize --encrypt-key "minio2/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json)
rep_obj2_etag=$(echo "${stat_out2_rep}" | jq '.etag')
rep_obj2_size=$(echo "${stat_out2_rep}" | jq '.size')
rep_obj2_md5=$(echo "${stat_out2_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
echo "Stat minio2/test-bucket/custpartsize"
./mc stat minio2/test-bucket/custpartsize --encrypt-key "minio2/test-bucket/custpartsize=iliketobecrazybutnotsomuchreally" --insecure --json
stat_out3_rep=$(./mc stat minio2/test-bucket/custpartsize --encrypt-key "minio2/test-bucket/custpartsize=iliketobecrazybutnotsomuchreally" --insecure --json)
rep_obj3_etag=$(echo "${stat_out3_rep}" | jq '.etag')
rep_obj3_size=$(echo "${stat_out3_rep}" | jq '.size')
rep_obj3_md5=$(echo "${stat_out3_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
# Check the etag and size of replicated SSEC objects
if [ "${rep_obj1_etag}" != "${src_obj1_etag}" ]; then
echo "BUG: Etag: '${rep_obj1_etag}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_etag}'"
exit_1
fi
if [ "${rep_obj1_size}" != "${src_obj1_size}" ]; then
echo "BUG: Size: '${rep_obj1_size}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_size}'"
exit_1
fi
if [ "${rep_obj2_etag}" != "${src_obj2_etag}" ]; then
echo "BUG: Etag: '${rep_obj2_etag}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_etag}'"
exit_1
fi
if [ "${rep_obj2_size}" != "${src_obj2_size}" ]; then
echo "BUG: Size: '${rep_obj2_size}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_size}'"
exit_1
fi
if [ "${rep_obj3_etag}" != "${src_obj3_etag}" ]; then
echo "BUG: Etag: '${rep_obj3_etag}' of replicated object: 'minio2/test-bucket/custpartsize' doesn't match with source value: '${src_obj3_etag}'"
exit_1
fi
if [ "${rep_obj3_size}" != "${src_obj3_size}" ]; then
echo "BUG: Size: '${rep_obj3_size}' of replicated object: 'minio2/test-bucket/custpartsize' doesn't match with source value: '${src_obj3_size}'"
exit_1
fi
# Check content of replicated SSEC objects
./mc cat minio2/test-bucket/encrypted --encrypt-key "minio2/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure
./mc cat minio2/test-bucket/defpartsize --encrypt-key "minio2/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure >/dev/null || exit_1
./mc cat minio2/test-bucket/custpartsize --encrypt-key "minio2/test-bucket/custpartsize=iliketobecrazybutnotsomuchreally" --insecure >/dev/null || exit_1
# Check the MD5 checksums of encrypted objects from source and target
if [ "${src_obj1_md5}" != "${rep_obj1_md5}" ]; then
echo "BUG: MD5 checksum of object 'minio2/test-bucket/encrypted' doesn't match with source. Expected: '${src_obj1_md5}', Found: '${rep_obj1_md5}'"
exit_1
fi
if [ "${src_obj2_md5}" != "${rep_obj2_md5}" ]; then
echo "BUG: MD5 checksum of object 'minio2/test-bucket/defpartsize' doesn't match with source. Expected: '${src_obj2_md5}', Found: '${rep_obj2_md5}'"
exit_1
fi
if [ "${src_obj3_md5}" != "${rep_obj3_md5}" ]; then
echo "BUG: MD5 checksum of object 'minio2/test-bucket/custpartsize' doesn't match with source. Expected: '${src_obj3_md5}', Found: '${rep_obj3_md5}'"
exit_1
fi
cleanup

3
internal/bucket/replication/replication.go
View File

@ -220,9 +220,6 @@ func (c Config) GetDestination() Destination {
// Replicate returns true if the object should be replicated.
func (c Config) Replicate(obj ObjectOpts) bool {
if obj.SSEC {
return false
}
for _, rule := range c.FilterActionableRules(obj) {
if rule.Status == Disabled {
continue

14
internal/bucket/replication/replication_test.go
View File

@ -250,12 +250,12 @@ func TestReplicate(t *testing.T) {
{ObjectOpts{Name: "c1test"}, cfgs[0], true}, // 2. valid ObjectOpts passing empty Filter
{ObjectOpts{Name: "c1test", VersionID: "vid"}, cfgs[0], true}, // 3. valid ObjectOpts passing empty Filter
{ObjectOpts{Name: "c1test", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[0], true}, // 4. DeleteMarker version replication valid case - matches DeleteMarkerReplication status
{ObjectOpts{Name: "c1test", VersionID: "vid", OpType: DeleteReplicationType}, cfgs[0], true}, // 5. permanent delete of version, matches DeleteReplication status - valid case
{ObjectOpts{Name: "c1test", VersionID: "vid", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[0], true}, // 6. permanent delete of version, matches DeleteReplication status
{ObjectOpts{Name: "c1test", VersionID: "vid", DeleteMarker: true, SSEC: true, OpType: DeleteReplicationType}, cfgs[0], false}, // 7. permanent delete of version, disqualified by SSE-C
{ObjectOpts{Name: "c1test", DeleteMarker: true, SSEC: true, OpType: DeleteReplicationType}, cfgs[0], false}, // 8. setting DeleteMarker on SSE-C encrypted object, disqualified by SSE-C
{ObjectOpts{Name: "c1test", SSEC: true}, cfgs[0], false}, // 9. replication of SSE-C encrypted object, disqualified
{ObjectOpts{Name: "c1test", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[0], true}, // 4. DeleteMarker version replication valid case - matches DeleteMarkerReplication status
{ObjectOpts{Name: "c1test", VersionID: "vid", OpType: DeleteReplicationType}, cfgs[0], true}, // 5. permanent delete of version, matches DeleteReplication status - valid case
{ObjectOpts{Name: "c1test", VersionID: "vid", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[0], true}, // 6. permanent delete of version, matches DeleteReplication status
{ObjectOpts{Name: "c1test", VersionID: "vid", DeleteMarker: true, SSEC: true, OpType: DeleteReplicationType}, cfgs[0], true}, // 7. permanent delete of version
{ObjectOpts{Name: "c1test", DeleteMarker: true, SSEC: true, OpType: DeleteReplicationType}, cfgs[0], true}, // 8. setting DeleteMarker on SSE-C encrypted object
{ObjectOpts{Name: "c1test", SSEC: true}, cfgs[0], true}, // 9. replication of SSE-C encrypted object
// using config 2 - no filters, only replication of object, metadata enabled
{ObjectOpts{Name: "c2test"}, cfgs[1], true}, // 10. valid ObjectOpts passing empty Filter
@ -264,7 +264,7 @@ func TestReplicate(t *testing.T) {
{ObjectOpts{Name: "c2test", VersionID: "vid", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[1], false}, // 13. permanent delete of DeleteMarker version, disallowed by DeleteReplication status
{ObjectOpts{Name: "c2test", VersionID: "vid", DeleteMarker: true, SSEC: true, OpType: DeleteReplicationType}, cfgs[1], false}, // 14. permanent delete of version, disqualified by SSE-C & DeleteReplication status
{ObjectOpts{Name: "c2test", DeleteMarker: true, SSEC: true, OpType: DeleteReplicationType}, cfgs[1], false}, // 15. setting DeleteMarker on SSE-C encrypted object, disqualified by SSE-C & DeleteMarkerReplication status
{ObjectOpts{Name: "c2test", SSEC: true}, cfgs[1], false}, // 16. replication of SSE-C encrypted object, disqualified by default
{ObjectOpts{Name: "c2test", SSEC: true}, cfgs[1], true}, // 16. replication of SSE-C encrypted object
// using config 2 - has more than one rule with overlapping prefixes
{ObjectOpts{Name: "xy/c3test", UserTags: "k1=v1"}, cfgs[2], true}, // 17. matches rule 1 for replication of content/metadata
{ObjectOpts{Name: "xyz/c3test", UserTags: "k1=v1"}, cfgs[2], true}, // 18. matches rule 1 for replication of content/metadata

2
internal/crypto/error.go
View File

@ -82,6 +82,8 @@ var (
// ErrIncompatibleEncryptionMethod indicates that both SSE-C headers and SSE-S3 headers were specified, and are incompatible
// The client needs to remove the SSE-S3 header or the SSE-C headers
ErrIncompatibleEncryptionMethod = Errorf("Server side encryption specified with both SSE-C and SSE-S3 headers")
// ErrIncompatibleEncryptionWithCompression indicates that both data compression and SSE-C not allowed at the same time
ErrIncompatibleEncryptionWithCompression = Errorf("Server side encryption specified with SSE-C with compression not allowed")
// ErrInvalidEncryptionKeyID returns error when KMS key id contains invalid characters
ErrInvalidEncryptionKeyID = Errorf("KMS KeyID contains unsupported characters")

3
internal/http/headers.go
View File

@ -233,6 +233,9 @@ const (
// Header indicates a Tag operation was performed on one/more peers successfully, though the
// current cluster does not have the object yet. This is in a site/bucket replication scenario.
MinIOTaggingProxied = "X-Minio-Tagging-Proxied"
// Header indicates the actual replicated object size
// In case of SSEC objects getting replicated (multipart) actual size would be needed at target
MinIOReplicationActualObjectSize = "X-Minio-Replication-Actual-Object-Size"
// predicted date/time of transition
MinIOTransition = "X-Minio-Transition"