s3/zip: extract metadata properly for Zipped objects (#15123)

s3/zip: extra metadata properly for Zipped objects

fixes #15121
This commit is contained in:
Harshavardhana 2022-06-21 14:11:12 -07:00 committed by GitHub
parent 10522438b7
commit f293df647c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 51 additions and 35 deletions

View File

@ -18,6 +18,7 @@
package cmd package cmd
import ( import (
"encoding/base64"
"io" "io"
"math" "math"
"time" "time"
@ -178,6 +179,26 @@ type ObjectInfo struct {
SuccessorModTime time.Time SuccessorModTime time.Time
} }
// ArchiveInfo returns any saved zip archive meta information
func (o ObjectInfo) ArchiveInfo() []byte {
if len(o.UserDefined) == 0 {
return nil
}
z, ok := o.UserDefined[archiveInfoMetadataKey]
if !ok {
return nil
}
if len(z) > 0 && z[0] >= 32 {
// FS/gateway mode does base64 encoding on roundtrip.
// zipindex has version as first byte, which is below any base64 value.
zipInfo, _ := base64.StdEncoding.DecodeString(z)
if len(zipInfo) != 0 {
return zipInfo
}
}
return []byte(z)
}
// Clone - Returns a cloned copy of current objectInfo // Clone - Returns a cloned copy of current objectInfo
func (o ObjectInfo) Clone() (cinfo ObjectInfo) { func (o ObjectInfo) Clone() (cinfo ObjectInfo) {
cinfo = ObjectInfo{ cinfo = ObjectInfo{

View File

@ -140,21 +140,10 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
return return
} }
var zipInfo []byte zipInfo := zipObjInfo.ArchiveInfo()
if z, ok := zipObjInfo.UserDefined[archiveInfoMetadataKey]; ok {
if globalIsErasure {
zipInfo = []byte(z)
} else {
zipInfo, err = base64.StdEncoding.DecodeString(z)
logger.LogIf(ctx, err)
// Will attempt to re-read...
}
}
if len(zipInfo) == 0 { if len(zipInfo) == 0 {
zipInfo, err = updateObjectMetadataWithZipInfo(ctx, objectAPI, bucket, zipPath, opts) zipInfo, err = updateObjectMetadataWithZipInfo(ctx, objectAPI, bucket, zipPath, opts)
} }
if err != nil { if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return return
@ -247,15 +236,11 @@ func listObjectsV2InArchive(ctx context.Context, objectAPI ObjectLayer, bucket,
return ListObjectsV2Info{}, nil return ListObjectsV2Info{}, nil
} }
var zipInfo []byte zipInfo := zipObjInfo.ArchiveInfo()
if len(zipInfo) == 0 {
if z, ok := zipObjInfo.UserDefined[archiveInfoMetadataKey]; ok {
zipInfo = []byte(z)
} else {
// Always update the latest version // Always update the latest version
zipInfo, err = updateObjectMetadataWithZipInfo(ctx, objectAPI, bucket, zipPath, ObjectOptions{}) zipInfo, err = updateObjectMetadataWithZipInfo(ctx, objectAPI, bucket, zipPath, ObjectOptions{})
} }
if err != nil { if err != nil {
return ListObjectsV2Info{}, err return ListObjectsV2Info{}, err
} }
@ -332,11 +317,10 @@ func getFilesListFromZIPObject(ctx context.Context, objectAPI ObjectLayer, bucke
return nil, ObjectInfo{}, err return nil, ObjectInfo{}, err
} }
b, err := ioutil.ReadAll(gr) b, err := ioutil.ReadAll(gr)
gr.Close()
if err != nil { if err != nil {
gr.Close()
return nil, ObjectInfo{}, err return nil, ObjectInfo{}, err
} }
gr.Close()
if size > len(b) { if size > len(b) {
size = len(b) size = len(b)
} }
@ -439,11 +423,8 @@ func (api objectAPIHandlers) headObjectInArchiveFileHandler(ctx context.Context,
return return
} }
var zipInfo []byte zipInfo := zipObjInfo.ArchiveInfo()
if len(zipInfo) == 0 {
if z, ok := zipObjInfo.UserDefined[archiveInfoMetadataKey]; ok {
zipInfo = []byte(z)
} else {
zipInfo, err = updateObjectMetadataWithZipInfo(ctx, objectAPI, bucket, zipPath, opts) zipInfo, err = updateObjectMetadataWithZipInfo(ctx, objectAPI, bucket, zipPath, opts)
} }
if err != nil { if err != nil {
@ -498,20 +479,34 @@ func updateObjectMetadataWithZipInfo(ctx context.Context, objectAPI ObjectLayer,
} }
srcInfo.UserDefined[archiveTypeMetadataKey] = archiveType srcInfo.UserDefined[archiveTypeMetadataKey] = archiveType
if globalIsErasure { var zipInfoStr string
srcInfo.UserDefined[archiveInfoMetadataKey] = string(zipInfo) if globalIsGateway {
zipInfoStr = base64.StdEncoding.EncodeToString(zipInfo)
} else { } else {
srcInfo.UserDefined[archiveInfoMetadataKey] = base64.StdEncoding.EncodeToString(zipInfo) zipInfoStr = string(zipInfo)
} }
srcInfo.metadataOnly = true
// Always update the same version id & modtime if globalIsGateway {
srcInfo.UserDefined[archiveInfoMetadataKey] = zipInfoStr
// Passing opts twice as source & destination options will update the metadata // Use CopyObject API only for Gateway mode.
// of the same object version to avoid creating a new version. if _, err = objectAPI.CopyObject(ctx, bucket, object, bucket, object, srcInfo, opts, opts); err != nil {
_, err = objectAPI.CopyObject(ctx, bucket, object, bucket, object, srcInfo, opts, opts) return nil, err
if err != nil { }
return nil, err } else {
popts := ObjectOptions{
MTime: srcInfo.ModTime,
VersionID: srcInfo.VersionID,
EvalMetadataFn: func(oi ObjectInfo) error {
oi.UserDefined[archiveTypeMetadataKey] = archiveType
oi.UserDefined[archiveInfoMetadataKey] = zipInfoStr
return nil
},
}
// For all other modes use in-place update to update metadata on a specific version.
if _, err = objectAPI.PutObjectMetadata(ctx, bucket, object, popts); err != nil {
return nil, err
}
} }
return zipInfo, nil return zipInfo, nil