From c601cb2f1eb24622f00ab0246584aa96d290009d Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 19 Aug 2019 11:02:54 -1000 Subject: [PATCH] Add listBucketObjectsVersions implementation (#8093) This API implementation simply behaves like listObjects() but returns back single version for each object, this implementation should be considered dummy it is only meant for some applications which rely on this. --- cmd/api-resources.go | 21 +++++++ cmd/api-response.go | 93 ++++++++++++++++++++++++++++++ cmd/api-router.go | 4 +- cmd/bucket-handlers-listobjects.go | 77 +++++++++++++++++++++++++ cmd/generic-handlers.go | 1 - 5 files changed, 194 insertions(+), 2 deletions(-) diff --git a/cmd/api-resources.go b/cmd/api-resources.go index 623378351..13f79fb34 100644 --- a/cmd/api-resources.go +++ b/cmd/api-resources.go @@ -42,6 +42,27 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, return } +func getListBucketObjectVersionsArgs(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType, versionIDMarker string, errCode APIErrorCode) { + errCode = ErrNone + + if values.Get("max-keys") != "" { + var err error + if maxkeys, err = strconv.Atoi(values.Get("max-keys")); err != nil { + errCode = ErrInvalidMaxKeys + return + } + } else { + maxkeys = maxObjectList + } + + prefix = values.Get("prefix") + marker = values.Get("key-marker") + delimiter = values.Get("delimiter") + encodingType = values.Get("encoding-type") + versionIDMarker = values.Get("version-id-marker") + return +} + // Parse bucket url queries for ListObjects V2. func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int, encodingType string, errCode APIErrorCode) { errCode = ErrNone diff --git a/cmd/api-response.go b/cmd/api-response.go index 72fc01719..1c72d7c24 100644 --- a/cmd/api-response.go +++ b/cmd/api-response.go @@ -44,6 +44,45 @@ type LocationResponse struct { Location string `xml:",chardata"` } +// ListVersionsResponse - format for list bucket versions response. +type ListVersionsResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListVersionsResult" json:"-"` + + Name string + Prefix string + KeyMarker string + + // When response is truncated (the IsTruncated element value in the response + // is true), you can use the key name in this field as marker in the subsequent + // request to get next set of objects. Server lists objects in alphabetical + // order Note: This element is returned only if you have delimiter request parameter + // specified. If response does not include the NextMaker and it is truncated, + // you can use the value of the last Key in the response as the marker in the + // subsequent request to get the next set of object keys. + NextKeyMarker string `xml:"NextKeyMarker,omitempty"` + + // When the number of responses exceeds the value of MaxKeys, + // NextVersionIdMarker specifies the first object version not + // returned that satisfies the search criteria. Use this value + // for the version-id-marker request parameter in a subsequent request. + NextVersionIDMarker string `xml:"NextVersionIdMarker"` + + // Marks the last version of the Key returned in a truncated response. + VersionIDMarker string `xml:"VersionIdMarker"` + + MaxKeys int + Delimiter string + // A flag that indicates whether or not ListObjects returned all of the results + // that satisfied the search criteria. + IsTruncated bool + + CommonPrefixes []CommonPrefix + Versions []ObjectVersion + + // Encoding type used to encode object keys in the response. + EncodingType string `xml:"EncodingType,omitempty"` +} + // ListObjectsResponse - format for list objects response. type ListObjectsResponse struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"` @@ -191,6 +230,14 @@ type Bucket struct { CreationDate string // time string of format "2006-01-02T15:04:05.000Z" } +// ObjectVersion container for object version metadata +type ObjectVersion struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Version" json:"-"` + Object + VersionID string `xml:"VersionId"` + IsLatest bool +} + // Object container for object metadata type Object struct { Key string @@ -328,6 +375,52 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse { return data } +// generates an ListBucketVersions response for the said bucket with other enumerated options. +func generateListVersionsResponse(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListVersionsResponse { + var versions []ObjectVersion + var prefixes []CommonPrefix + var owner = Owner{} + var data = ListVersionsResponse{} + + owner.ID = globalMinioDefaultOwnerID + for _, object := range resp.Objects { + var content = ObjectVersion{} + if object.Name == "" { + continue + } + content.Key = s3EncodeName(object.Name, encodingType) + content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong) + if object.ETag != "" { + content.ETag = "\"" + object.ETag + "\"" + } + content.Size = object.Size + content.StorageClass = object.StorageClass + content.Owner = owner + content.VersionID = "null" + content.IsLatest = true + versions = append(versions, content) + } + data.Name = bucket + data.Versions = versions + + data.EncodingType = encodingType + data.Prefix = s3EncodeName(prefix, encodingType) + data.KeyMarker = s3EncodeName(marker, encodingType) + data.Delimiter = s3EncodeName(delimiter, encodingType) + data.MaxKeys = maxKeys + + data.NextKeyMarker = s3EncodeName(resp.NextMarker, encodingType) + data.IsTruncated = resp.IsTruncated + + for _, prefix := range resp.Prefixes { + var prefixItem = CommonPrefix{} + prefixItem.Prefix = s3EncodeName(prefix, encodingType) + prefixes = append(prefixes, prefixItem) + } + data.CommonPrefixes = prefixes + return data +} + // generates an ListObjectsV1 response for the said bucket with other enumerated options. func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse { var contents []Object diff --git a/cmd/api-router.go b/cmd/api-router.go index 18ead4f37..c25aa4828 100644 --- a/cmd/api-router.go +++ b/cmd/api-router.go @@ -64,7 +64,7 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool) bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") // PutObjectPart bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") - // ListObjectPxarts + // ListObjectParts bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}") // CompleteMultipartUpload bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}") @@ -129,6 +129,8 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool) bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "") // ListObjectsV2 bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2") + // ListBucketVersions + bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListBucketObjectVersionsHandler)).Queries("versions", "") // ListObjectsV1 (Legacy) bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler)) // PutBucketLifecycle diff --git a/cmd/bucket-handlers-listobjects.go b/cmd/bucket-handlers-listobjects.go index 3874b2383..f8e87cf7f 100644 --- a/cmd/bucket-handlers-listobjects.go +++ b/cmd/bucket-handlers-listobjects.go @@ -49,6 +49,83 @@ func validateListObjectsArgs(prefix, marker, delimiter, encodingType string, max return ErrNone } +// ListBucketObjectVersions - GET Bucket Object versions +// You can use the versions subresource to list metadata about all +// of the versions of objects in a bucket. +func (api objectAPIHandlers) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "ListBucketObjectVersions") + + defer logger.AuditLog(w, r, "ListBucketObjectVersions", mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r)) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) + return + } + + urlValues := r.URL.Query() + + // Extract all the listBucketVersions query params to their native values. + // versionIDMarker is ignored here. + prefix, marker, delimiter, maxkeys, encodingType, _, errCode := getListBucketObjectVersionsArgs(urlValues) + if errCode != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r)) + return + } + + // Validate the query params before beginning to serve the request. + if s3Error := validateListObjectsArgs(prefix, marker, delimiter, encodingType, maxkeys); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) + return + } + + listObjects := objectAPI.ListObjects + + // Inititate a list objects operation based on the input params. + // On success would return back ListObjectsInfo object to be + // marshaled into S3 compatible XML header. + listObjectsInfo, err := listObjects(ctx, bucket, prefix, marker, delimiter, maxkeys) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) + return + } + + for i := range listObjectsInfo.Objects { + var actualSize int64 + if listObjectsInfo.Objects[i].IsCompressed() { + // Read the decompressed size from the meta.json. + actualSize = listObjectsInfo.Objects[i].GetActualSize() + if actualSize < 0 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDecompressedSize), + r.URL, guessIsBrowserReq(r)) + return + } + // Set the info.Size to the actualSize. + listObjectsInfo.Objects[i].Size = actualSize + } else if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) { + listObjectsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsInfo.Objects[i], false) + listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].DecryptedSize() + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) + return + } + } + } + + response := generateListVersionsResponse(bucket, prefix, marker, delimiter, encodingType, maxkeys, listObjectsInfo) + + // Write success response. + writeSuccessResponseXML(w, encodeResponse(response)) +} + // ListObjectsV2Handler - GET Bucket (List Objects) Version 2. // -------------------------- // This implementation of the GET operation returns some or all (up to 1000) diff --git a/cmd/generic-handlers.go b/cmd/generic-handlers.go index c66f2fee7..677df01d3 100644 --- a/cmd/generic-handlers.go +++ b/cmd/generic-handlers.go @@ -481,7 +481,6 @@ var notimplementedBucketResourceNames = map[string]bool{ "requestPayment": true, "tagging": true, "versioning": true, - "versions": true, "website": true, }