MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 12:52:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

replication: pass checksum headers to replica (#19834)

Browse Source
This commit is contained in:
Poorna
2024-06-06 02:36:42 -07:00
committed by GitHub
parent 7edc352d23
commit 5aaef9790f
4 changed files with 44 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
cmd/bucket-replication.go
View File

@@ -763,12 +763,32 @@ func (m caseInsensitiveMap) Lookup(key string) (string, bool) {
return "", false
}
func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (putOpts minio.PutObjectOptions, err error) {
func getCRCMeta(oi ObjectInfo, partNum int) map[string]string {
meta := make(map[string]string)
cs := oi.decryptChecksums(partNum)
for k, v := range cs {
cksum := hash.NewChecksumString(k, v)
if cksum == nil {
continue
}
if cksum.Valid() {
meta[cksum.Type.Key()] = v
}
}
return meta
}
func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, partNum int) (putOpts minio.PutObjectOptions, err error) {
meta := make(map[string]string)
for k, v := range objInfo.UserDefined {
// In case of SSE-C objects copy the allowed internal headers as well
if !crypto.SSEC.IsEncrypted(objInfo.UserDefined) || !slices.Contains(maps.Keys(validSSEReplicationHeaders), k) {
if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
if strings.EqualFold(k, ReservedMetadataPrefixLower+"crc") {
for k, v := range getCRCMeta(objInfo, partNum) {
meta[k] = v
}
}
continue
}
if isStandardHeader(k) {
@@ -782,6 +802,12 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (put
}
}
if len(objInfo.Checksum) > 0 {
for k, v := range getCRCMeta(objInfo, 0) {
meta[k] = v
}
}
if sc == "" && (objInfo.StorageClass == storageclass.STANDARD || objInfo.StorageClass == storageclass.RRS) {
sc = objInfo.StorageClass
}
@@ -1239,7 +1265,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
// use core client to avoid doing multipart on PUT
c := &minio.Core{Client: tgt.Client}
putOpts, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo)
putOpts, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo, 0)
if err != nil {
replLogIf(ctx, fmt.Errorf("failure setting options for replication bucket:%s err:%w", bucket, err))
sendEvent(eventArgs{
@@ -1506,7 +1532,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
}
} else {
var putOpts minio.PutObjectOptions
putOpts, err = putReplicationOpts(ctx, tgt.StorageClass, objInfo)
putOpts, err = putReplicationOpts(ctx, tgt.StorageClass, objInfo, 0)
if err != nil {
replLogIf(ctx, fmt.Errorf("failed to set replicate options for object %s/%s(%s) (target %s) err:%w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), err))
sendEvent(eventArgs{
@@ -1614,6 +1640,10 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
cHeader := http.Header{}
cHeader.Add(xhttp.MinIOSourceReplicationRequest, "true")
crc := getCRCMeta(objInfo, partInfo.Number)
for k, v := range crc {
cHeader.Add(k, v)
}
popts := minio.PutObjectPartOptions{
SSE: opts.ServerSideEncryption,
CustomHeader: cHeader,
@@ -1633,8 +1663,12 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
return fmt.Errorf("Part size mismatch: got %d, want %d", pInfo.Size, partInfo.ActualSize)
}
uploadedParts = append(uploadedParts, minio.CompletePart{
PartNumber: pInfo.PartNumber,
ETag: pInfo.ETag,
PartNumber: pInfo.PartNumber,
ETag: pInfo.ETag,
ChecksumCRC32: pInfo.ChecksumCRC32,
ChecksumCRC32C: pInfo.ChecksumCRC32C,
ChecksumSHA1: pInfo.ChecksumSHA1,
ChecksumSHA256: pInfo.ChecksumSHA256,
})
}