feat: add support for GetObjectAttributes API (#18732)

cmd/api-errors.go

@@ -430,6 +430,9 @@ const (
 	ErrLambdaARNInvalid
 	ErrLambdaARNNotFound
 
+	// New Codes for GetObjectAttributes and GetObjectVersionAttributes
+	ErrInvalidAttributeName
+
 	apiErrCodeEnd // This is used only for the testing code
 )
 
@@ -2063,6 +2066,11 @@ var errorCodes = errorCodeMap{
 		Description:    "The specified policy is not found.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrInvalidAttributeName: {
+		Code:           "InvalidArgument",
+		Description:    "Invalid attribute name specified.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	// Add your error structure here.
 }
 

cmd/api-router.go

@@ -227,6 +227,9 @@ func registerAPIRouter(router *mux.Router) {
 		// HeadObject
 		router.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
 			collectAPIStats("headobject", maxClients(gz(httpTraceAll(api.HeadObjectHandler)))))
+		// GetObjectAttribytes
+		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
+			collectAPIStats("getobjectattributes", maxClients(gz(httpTraceHdrs(api.GetObjectAttributesHandler))))).Queries("attributes", "")
 		// CopyObjectPart
 		router.Methods(http.MethodPut).Path("/{object:.+}").
 			HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").

cmd/encryption-v1.go

@@ -1082,6 +1082,30 @@ func (o *ObjectInfo) metadataDecrypter() objectMetaDecryptFn {
 	}
 }
 
+// decryptChecksums will attempt to decode checksums and return it/them if set.
+// if part > 0, and we have the checksum for the part that will be returned.
+func (o *ObjectInfo) decryptPartsChecksums() {
+	data := o.Checksum
+	if len(data) == 0 {
+		return
+	}
+	if _, encrypted := crypto.IsEncrypted(o.UserDefined); encrypted {
+		decrypted, err := o.metadataDecrypter()("object-checksum", data)
+		if err != nil {
+			logger.LogIf(GlobalContext, err)
+			return
+		}
+		data = decrypted
+	}
+	cs := hash.ReadPartCheckSums(data)
+	if len(cs) == len(o.Parts) {
+		for i := range o.Parts {
+			o.Parts[i].Checksums = cs[i]
+		}
+	}
+	return
+}
+
 // metadataEncryptFn provides an encryption function for metadata.
 // Will return nil, nil if unencrypted.
 func (o *ObjectInfo) metadataEncryptFn(headers http.Header) (objectMetaEncryptFn, error) {

cmd/object-api-datatypes.go

@@ -612,3 +612,42 @@ type NewMultipartUploadResult struct {
 	UploadID     string
 	ChecksumAlgo string
 }
+
+type getObjectAttributesResponse struct {
+	ETag         string                    `xml:",omitempty"`
+	Checksum     *objectAttributesChecksum `xml:",omitempty"`
+	ObjectParts  *objectAttributesParts    `xml:",omitempty"`
+	StorageClass string                    `xml:",omitempty"`
+	ObjectSize   int64                     `xml:",omitempty"`
+}
+
+type objectAttributesChecksum struct {
+	ChecksumCRC32  string `xml:",omitempty"`
+	ChecksumCRC32C string `xml:",omitempty"`
+	ChecksumSHA1   string `xml:",omitempty"`
+	ChecksumSHA256 string `xml:",omitempty"`
+}
+
+type objectAttributesParts struct {
+	IsTruncated          bool
+	MaxParts             int
+	NextPartNumberMarker int
+	PartNumberMarker     int
+	PartsCount           int
+	Parts                []*objectAttributesPart `xml:"Part"`
+}
+
+type objectAttributesPart struct {
+	PartNumber     int
+	Size           int64
+	ChecksumCRC32  string `xml:",omitempty"`
+	ChecksumCRC32C string `xml:",omitempty"`
+	ChecksumSHA1   string `xml:",omitempty"`
+	ChecksumSHA256 string `xml:",omitempty"`
+}
+
+type objectAttributesErrorResponse struct {
+	ArgumentValue *string `xml:"ArgumentValue,omitempty"`
+	ArgumentName  *string `xml:"ArgumentName"`
+	APIErrorResponse
+}

cmd/object-api-interface.go

@@ -69,6 +69,9 @@ type ObjectOptions struct {
 	Tagging                 bool // Is only in GET/HEAD operations to return tagging metadata along with regular metadata and body.
 
 	UserDefined         map[string]string   // only set in case of POST/PUT operations
+	ObjectAttributes    map[string]struct{} // Attribute tags defined by the users for the GetObjectAttributes request
+	MaxParts            int                 // used in GetObjectAttributes. Signals how many parts we should return
+	PartNumberMarker    int                 // used in GetObjectAttributes. Signals the part number after which results should be returned
 	PartNumber          int                 // only useful in case of GetObject/HeadObject
 	CheckPrecondFn      CheckPreconditionFn // only set during GetObject/HeadObject/CopyObjectPart preconditional valuation
 	EvalMetadataFn      EvalMetadataFn      // only set for retention settings, meant to be used only when updating metadata in-place.

cmd/object-api-options.go

@@ -133,6 +133,121 @@ func getOpts(ctx context.Context, r *http.Request, bucket, object string) (Objec
 	return opts, nil
 }
 
+func getAndValidateAttributesOpts(ctx context.Context, w http.ResponseWriter, r *http.Request, bucket, object string) (opts ObjectOptions, valid bool) {
+	var argumentName string
+	var argumentValue string
+	var apiErr APIError
+	var err error
+	valid = true
+
+	defer func() {
+		if valid {
+			return
+		}
+
+		errResp := objectAttributesErrorResponse{
+			ArgumentName:  &argumentName,
+			ArgumentValue: &argumentValue,
+			APIErrorResponse: getAPIErrorResponse(
+				ctx,
+				apiErr,
+				r.URL.Path,
+				w.Header().Get(xhttp.AmzRequestID),
+				w.Header().Get(xhttp.AmzRequestHostID),
+			),
+		}
+
+		writeResponse(w, apiErr.HTTPStatusCode, encodeResponse(errResp), mimeXML)
+	}()
+
+	opts, err = getOpts(ctx, r, bucket, object)
+	if err != nil {
+		switch vErr := err.(type) {
+		case InvalidVersionID:
+			apiErr = toAPIError(ctx, vErr)
+			argumentName = strings.ToLower("versionId")
+			argumentValue = vErr.VersionID
+		default:
+			apiErr = toAPIError(ctx, vErr)
+		}
+		valid = false
+		return
+	}
+
+	opts.MaxParts, err = parseIntHeader(bucket, object, r.Header, xhttp.AmzMaxParts)
+	if err != nil {
+		apiErr = toAPIError(ctx, err)
+		argumentName = strings.ToLower(xhttp.AmzMaxParts)
+		valid = false
+		return
+	}
+
+	if opts.MaxParts == 0 {
+		opts.MaxParts = maxPartsList
+	}
+
+	opts.PartNumberMarker, err = parseIntHeader(bucket, object, r.Header, xhttp.AmzPartNumberMarker)
+	if err != nil {
+		apiErr = toAPIError(ctx, err)
+		argumentName = strings.ToLower(xhttp.AmzPartNumberMarker)
+		valid = false
+		return
+	}
+
+	opts.ObjectAttributes = parseObjectAttributes(r.Header)
+	if len(opts.ObjectAttributes) < 1 {
+		apiErr = errorCodes.ToAPIErr(ErrInvalidAttributeName)
+		argumentName = strings.ToLower(xhttp.AmzObjectAttributes)
+		valid = false
+		return
+	}
+
+	for tag := range opts.ObjectAttributes {
+		switch tag {
+		case xhttp.ETag:
+		case xhttp.Checksum:
+		case xhttp.StorageClass:
+		case xhttp.ObjectSize:
+		case xhttp.ObjectParts:
+		default:
+			apiErr = errorCodes.ToAPIErr(ErrInvalidAttributeName)
+			argumentName = strings.ToLower(xhttp.AmzObjectAttributes)
+			argumentValue = tag
+			valid = false
+			return
+		}
+	}
+
+	return
+}
+
+func parseObjectAttributes(h http.Header) (attributes map[string]struct{}) {
+	attributes = make(map[string]struct{})
+	for _, v := range strings.Split(strings.TrimSpace(h.Get(xhttp.AmzObjectAttributes)), ",") {
+		if v != "" {
+			attributes[v] = struct{}{}
+		}
+	}
+
+	return
+}
+
+func parseIntHeader(bucket, object string, h http.Header, headerName string) (value int, err error) {
+	stringInt := strings.TrimSpace(h.Get(headerName))
+	if stringInt == "" {
+		return
+	}
+	value, err = strconv.Atoi(stringInt)
+	if err != nil {
+		return 0, InvalidArgument{
+			Bucket: bucket,
+			Object: object,
+			Err:    fmt.Errorf("Unable to parse %s, value should be an integer", headerName),
+		}
+	}
+	return
+}
+
 func parseBoolHeader(bucket, object string, h http.Header, headerName string) (bool, error) {
 	value := strings.TrimSpace(h.Get(headerName))
 	if value != "" {

cmd/object-handlers.go

@@ -689,6 +689,155 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 	})
 }
 
+// GetObjectAttributes ...
+func (api objectAPIHandlers) getObjectAttributesHandler(ctx context.Context, objectAPI ObjectLayer, bucket, object string, w http.ResponseWriter, r *http.Request) {
+	opts, valid := getAndValidateAttributesOpts(ctx, w, r, bucket, object)
+	if !valid {
+		return
+	}
+
+	var s3Error APIErrorCode
+	if opts.VersionID != "" {
+		s3Error = checkRequestAuthType(ctx, r, policy.GetObjectVersionAttributesAction, bucket, object)
+		if s3Error == ErrNone {
+			s3Error = checkRequestAuthType(ctx, r, policy.GetObjectVersionAction, bucket, object)
+		}
+	} else {
+		s3Error = checkRequestAuthType(ctx, r, policy.GetObjectAttributesAction, bucket, object)
+		if s3Error == ErrNone {
+			s3Error = checkRequestAuthType(ctx, r, policy.GetObjectAction, bucket, object)
+		}
+	}
+
+	if s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
+		return
+	}
+
+	objInfo, err := objectAPI.GetObjectInfo(ctx, bucket, object, opts)
+	if err != nil {
+		s3Error = checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, object)
+		if s3Error == ErrNone {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
+
+	if _, err = DecryptObjectInfo(&objInfo, r); err != nil {
+		writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
+		return
+	}
+
+	if checkPreconditions(ctx, w, r, objInfo, opts) {
+		return
+	}
+
+	OA := new(getObjectAttributesResponse)
+
+	if opts.Versioned {
+		w.Header().Set(xhttp.AmzVersionID, objInfo.VersionID)
+	}
+
+	lastModified := objInfo.ModTime.UTC().Format(http.TimeFormat)
+	w.Header().Set(xhttp.LastModified, lastModified)
+	w.Header().Del(xhttp.ContentType)
+
+	if _, ok := opts.ObjectAttributes[xhttp.Checksum]; ok {
+		chkSums := objInfo.decryptChecksums(0)
+		// AWS does not appear to append part number on this API call.
+		switch {
+		case chkSums["CRC32"] != "":
+			OA.Checksum = new(objectAttributesChecksum)
+			OA.Checksum.ChecksumCRC32 = strings.Split(chkSums["CRC32"], "-")[0]
+		case chkSums["CRC32C"] != "":
+			OA.Checksum = new(objectAttributesChecksum)
+			OA.Checksum.ChecksumCRC32C = strings.Split(chkSums["CRC32C"], "-")[0]
+		case chkSums["SHA256"] != "":
+			OA.Checksum = new(objectAttributesChecksum)
+			OA.Checksum.ChecksumSHA1 = strings.Split(chkSums["SHA1"], "-")[0]
+		case chkSums["SHA1"] != "":
+			OA.Checksum = new(objectAttributesChecksum)
+			OA.Checksum.ChecksumSHA256 = strings.Split(chkSums["SHA256"], "-")[0]
+		}
+
+	}
+
+	if _, ok := opts.ObjectAttributes[xhttp.ETag]; ok {
+		OA.ETag = objInfo.ETag
+	}
+
+	if _, ok := opts.ObjectAttributes[xhttp.ObjectSize]; ok {
+		OA.ObjectSize, _ = objInfo.GetActualSize()
+	}
+
+	if _, ok := opts.ObjectAttributes[xhttp.StorageClass]; ok {
+		OA.StorageClass = objInfo.StorageClass
+	}
+
+	objInfo.decryptPartsChecksums()
+
+	if _, ok := opts.ObjectAttributes[xhttp.ObjectParts]; ok {
+		OA.ObjectParts = new(objectAttributesParts)
+		OA.ObjectParts.PartNumberMarker = opts.PartNumberMarker
+
+		OA.ObjectParts.MaxParts = opts.MaxParts
+		partsLength := len(objInfo.Parts)
+		OA.ObjectParts.PartsCount = partsLength
+
+		if opts.MaxParts > -1 {
+			for i, v := range objInfo.Parts {
+				if v.Number <= opts.PartNumberMarker {
+					continue
+				}
+
+				if len(OA.ObjectParts.Parts) == opts.MaxParts {
+					break
+				}
+
+				OA.ObjectParts.NextPartNumberMarker = v.Number
+				OA.ObjectParts.Parts = append(OA.ObjectParts.Parts, &objectAttributesPart{
+					ChecksumSHA1:   objInfo.Parts[i].Checksums["SHA1"],
+					ChecksumSHA256: objInfo.Parts[i].Checksums["SHA256"],
+					ChecksumCRC32:  objInfo.Parts[i].Checksums["CRC32"],
+					ChecksumCRC32C: objInfo.Parts[i].Checksums["CRC32C"],
+					PartNumber:     objInfo.Parts[i].Number,
+					Size:           objInfo.Parts[i].Size,
+				})
+			}
+		}
+
+		if OA.ObjectParts.NextPartNumberMarker != partsLength {
+			OA.ObjectParts.IsTruncated = true
+		}
+	}
+
+	outBytes, err := xml.Marshal(OA)
+	if err != nil {
+		writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
+	}
+
+	writeResponse(
+		w,
+		200,
+		append([]byte(`<?xml version="1.0" encoding="UTF-8"?>`), outBytes...),
+		mimeXML,
+	)
+
+	sendEvent(eventArgs{
+		EventName:    event.ObjectAccessedAttributes,
+		BucketName:   bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         handlers.GetSourceIP(r),
+	})
+
+	return
+}
+
 // GetObjectHandler - GET Object
 // ----------
 // This implementation of the GET operation retrieves object. To use GET,
@@ -1043,6 +1192,30 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 	})
 }
 
+// GetObjectAttributesHandles - GET Object
+// -----------
+// This operation retrieves metadata and part metadata from an object without returning the object itself.
+func (api objectAPIHandlers) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetObjectAttributes")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrServerNotInitialized))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object, err := unescapePath(vars["object"])
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	api.getObjectAttributesHandler(ctx, objectAPI, bucket, object, w, r)
+}
+
 // HeadObjectHandler - HEAD Object
 // -----------
 // The HEAD operation retrieves metadata from an object without returning the object itself.

internal/event/name.go

@@ -36,6 +36,7 @@ const (
 	ObjectAccessedGetRetention
 	ObjectAccessedGetLegalHold
 	ObjectAccessedHead
+	ObjectAccessedAttributes
 	ObjectCreatedCompleteMultipartUpload
 	ObjectCreatedCopy
 	ObjectCreatedPost
@@ -84,7 +85,7 @@ func (name Name) Expand() []Name {
 	case ObjectAccessedAll:
 		return []Name{
 			ObjectAccessedGet, ObjectAccessedHead,
-			ObjectAccessedGetRetention, ObjectAccessedGetLegalHold,
+			ObjectAccessedGetRetention, ObjectAccessedGetLegalHold, ObjectAccessedAttributes,
 		}
 	case ObjectCreatedAll:
 		return []Name{
@@ -162,6 +163,8 @@ func (name Name) String() string {
 		return "s3:ObjectAccessed:GetLegalHold"
 	case ObjectAccessedHead:
 		return "s3:ObjectAccessed:Head"
+	case ObjectAccessedAttributes:
+		return "s3:ObjectAccessed:Attributes"
 	case ObjectCreatedAll:
 		return "s3:ObjectCreated:*"
 	case ObjectCreatedCompleteMultipartUpload:
@@ -278,6 +281,8 @@ func ParseName(s string) (Name, error) {
 		return ObjectAccessedGetLegalHold, nil
 	case "s3:ObjectAccessed:Head":
 		return ObjectAccessedHead, nil
+	case "s3:ObjectAccessed:Attributes":
+		return ObjectAccessedAttributes, nil
 	case "s3:ObjectCreated:*":
 		return ObjectCreatedAll, nil
 	case "s3:ObjectCreated:CompleteMultipartUpload":

internal/event/name_test.go

@@ -31,7 +31,7 @@ func TestNameExpand(t *testing.T) {
 	}{
 		{BucketCreated, []Name{BucketCreated}},
 		{BucketRemoved, []Name{BucketRemoved}},
-		{ObjectAccessedAll, []Name{ObjectAccessedGet, ObjectAccessedHead, ObjectAccessedGetRetention, ObjectAccessedGetLegalHold}},
+		{ObjectAccessedAll, []Name{ObjectAccessedGet, ObjectAccessedHead, ObjectAccessedGetRetention, ObjectAccessedGetLegalHold, ObjectAccessedAttributes}},
 		{ObjectCreatedAll, []Name{
 			ObjectCreatedCompleteMultipartUpload, ObjectCreatedCopy, ObjectCreatedPost, ObjectCreatedPut,
 			ObjectCreatedPutRetention, ObjectCreatedPutLegalHold, ObjectCreatedPutTagging, ObjectCreatedDeleteTagging,

internal/event/rulesmap_test.go

@@ -154,12 +154,12 @@ func TestRulesMapMatch(t *testing.T) {
 
 func TestNewRulesMap(t *testing.T) {
 	rulesMapCase1 := make(RulesMap)
-	rulesMapCase1.add([]Name{ObjectAccessedGet, ObjectAccessedHead, ObjectAccessedGetRetention, ObjectAccessedGetLegalHold},
+	rulesMapCase1.add([]Name{ObjectAccessedGet, ObjectAccessedHead, ObjectAccessedGetRetention, ObjectAccessedGetLegalHold, ObjectAccessedAttributes},
 		"*", TargetID{"1", "webhook"})
 
 	rulesMapCase2 := make(RulesMap)
 	rulesMapCase2.add([]Name{
-		ObjectAccessedGet, ObjectAccessedHead,
+		ObjectAccessedGet, ObjectAccessedHead, ObjectAccessedAttributes,
 		ObjectCreatedPut, ObjectAccessedGetRetention, ObjectAccessedGetLegalHold,
 	}, "*", TargetID{"1", "webhook"})
 

internal/hash/checksum.go

@@ -240,6 +240,49 @@ func ReadCheckSums(b []byte, part int) map[string]string {
 	return res
 }
 
+// ReadPartCheckSums will read all part checksums from b and return them.
+func ReadPartCheckSums(b []byte) (res []map[string]string) {
+	for len(b) > 0 {
+		t, n := binary.Uvarint(b)
+		if n <= 0 {
+			break
+		}
+		b = b[n:]
+
+		typ := ChecksumType(t)
+		length := typ.RawByteLen()
+		if length == 0 || len(b) < length {
+			break
+		}
+		// Skip main checksum
+		b = b[length:]
+		if !typ.Is(ChecksumIncludesMultipart) {
+			continue
+		}
+		parts, n := binary.Uvarint(b)
+		if n <= 0 {
+			break
+		}
+		if len(res) == 0 {
+			res = make([]map[string]string, parts)
+		}
+		b = b[n:]
+		for part := 0; part < int(parts); part++ {
+			if len(b) < length {
+				break
+			}
+			// Read part checksum
+			cs := base64.StdEncoding.EncodeToString(b[:length])
+			b = b[length:]
+			if res[part] == nil {
+				res[part] = make(map[string]string, 1)
+			}
+			res[part][typ.String()] = cs
+		}
+	}
+	return res
+}
+
 // NewChecksumWithType is similar to NewChecksumString but expects input algo of ChecksumType.
 func NewChecksumWithType(alg ChecksumType, value string) *Checksum {
 	if !alg.IsSet() {

internal/http/headers.go

@@ -55,6 +55,12 @@ const (
 	IfMatch           = "If-Match"
 	IfNoneMatch       = "If-None-Match"
 
+	// Request tags used in GetObjectAttributes
+	Checksum     = "Checksum"
+	StorageClass = "StorageClass"
+	ObjectSize   = "ObjectSize"
+	ObjectParts  = "ObjectParts"
+
 	// S3 storage class
 	AmzStorageClass = "x-amz-storage-class"
 
@@ -123,6 +129,11 @@ const (
 	AmzSecurityToken        = "X-Amz-Security-Token"
 	AmzDecodedContentLength = "X-Amz-Decoded-Content-Length"
 	AmzTrailer              = "X-Amz-Trailer"
+	AmzMaxParts             = "X-Amz-Max-Parts"
+	AmzPartNumberMarker     = "X-Amz-Part-Number-Marker"
+
+	// Constants used for GetObjectAttributes and GetObjectVersionAttributes
+	AmzObjectAttributes = "X-Amz-Object-Attributes"
 
 	AmzMetaUnencryptedContentLength = "X-Amz-Meta-X-Amz-Unencrypted-Content-Length"
 	AmzMetaUnencryptedContentMD5    = "X-Amz-Meta-X-Amz-Unencrypted-Content-Md5"