MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-23 21:55:53 -05:00
Code Issues Packages Projects Releases Wiki Activity

s3: Add support of encodingType parameter (#7265)

Browse Source 
This commit honors encoding-type parameter in object listing,
parts listing and multipart uploads listing.
This commit is contained in:
Anis Elleuch 2019-02-24 07:14:24 +01:00 committed by Nitish Tiwari
parent dab314900d
commit 5efbe8a1b3
7 changed files with 124 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/api-errors.go
View File

@ -73,6 +73,7 @@ const (
ErrInvalidCopyPartRange
ErrInvalidCopyPartRangeSource
ErrInvalidMaxKeys
ErrInvalidEncodingMethod
ErrInvalidMaxUploads
ErrInvalidMaxParts
ErrInvalidPartNumberMarker
@ -357,6 +358,11 @@ var errorCodes = errorCodeMap{
Description: "Argument maxKeys must be an integer between 0 and 2147483647",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidEncodingMethod: {
Code: "InvalidArgument",
Description: "Invalid Encoding Method specified in Request",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidMaxParts: {
Code: "InvalidArgument",
Description: "Argument max-parts must be an integer between 0 and 2147483647",

64
cmd/api-response.go
View File

@ -305,6 +305,21 @@ func getObjectLocation(r *http.Request, domains []string, bucket, object string)
return u.String()
}
// s3EncodeName encodes string in response when encodingType
// is specified in AWS S3 requests.
func s3EncodeName(name string, encodingType string) (result string) {
// Quick path to exit
if encodingType == "" {
return name
}
encodingType = strings.ToLower(encodingType)
switch encodingType {
case "url":
return url.QueryEscape(name)
}
return name
}
// generates ListBucketsResponse from array of BucketInfo which can be
// serialized to match XML and JSON API spec output.
func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
@ -327,7 +342,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
}
// generates an ListObjectsV1 response for the said bucket with other enumerated options.
func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
var contents []Object
var prefixes []CommonPrefix
var owner = Owner{}
@ -339,7 +354,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
if object.Name == "" {
continue
}
content.Key = object.Name
content.Key = s3EncodeName(object.Name, encodingType)
content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
if object.ETag != "" {
content.ETag = "\"" + object.ETag + "\""
@ -349,20 +364,20 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
content.Owner = owner
contents = append(contents, content)
}
// TODO - support EncodingType in xml decoding
data.Name = bucket
data.Contents = contents
data.Prefix = prefix
data.Marker = marker
data.Delimiter = delimiter
data.EncodingType = encodingType
data.Prefix = s3EncodeName(prefix, encodingType)
data.Marker = s3EncodeName(marker, encodingType)
data.Delimiter = s3EncodeName(delimiter, encodingType)
data.MaxKeys = maxKeys
data.NextMarker = resp.NextMarker
data.NextMarker = s3EncodeName(resp.NextMarker, encodingType)
data.IsTruncated = resp.IsTruncated
for _, prefix := range resp.Prefixes {
var prefixItem = CommonPrefix{}
prefixItem.Prefix = prefix
prefixItem.Prefix = s3EncodeName(prefix, encodingType)
prefixes = append(prefixes, prefixItem)
}
data.CommonPrefixes = prefixes
@ -370,7 +385,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
}
// generates an ListObjectsV2 response for the said bucket with other enumerated options.
func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string) ListObjectsV2Response {
func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string) ListObjectsV2Response {
var contents []Object
var commonPrefixes []CommonPrefix
var owner = Owner{}
@ -385,7 +400,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
if object.Name == "" {
continue
}
content.Key = object.Name
content.Key = s3EncodeName(object.Name, encodingType)
content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
if object.ETag != "" {
content.ETag = "\"" + object.ETag + "\""
@ -395,20 +410,20 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
content.Owner = owner
contents = append(contents, content)
}
// TODO - support EncodingType in xml decoding
data.Name = bucket
data.Contents = contents
data.EncodingType = encodingType
data.StartAfter = startAfter
data.Delimiter = delimiter
data.Prefix = prefix
data.Delimiter = s3EncodeName(delimiter, encodingType)
data.Prefix = s3EncodeName(prefix, encodingType)
data.MaxKeys = maxKeys
data.ContinuationToken = token
data.NextContinuationToken = nextToken
data.IsTruncated = isTruncated
for _, prefix := range prefixes {
var prefixItem = CommonPrefix{}
prefixItem.Prefix = prefix
prefixItem.Prefix = s3EncodeName(prefix, encodingType)
commonPrefixes = append(commonPrefixes, prefixItem)
}
data.CommonPrefixes = commonPrefixes
@ -452,11 +467,10 @@ func generateCompleteMultpartUploadResponse(bucket, key, location, etag string)
}
// generates ListPartsResponse from ListPartsInfo.
func generateListPartsResponse(partsInfo ListPartsInfo) ListPartsResponse {
// TODO - support EncodingType in xml decoding
func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) ListPartsResponse {
listPartsResponse := ListPartsResponse{}
listPartsResponse.Bucket = partsInfo.Bucket
listPartsResponse.Key = partsInfo.Object
listPartsResponse.Key = s3EncodeName(partsInfo.Object, encodingType)
listPartsResponse.UploadID = partsInfo.UploadID
listPartsResponse.StorageClass = globalMinioDefaultStorageClass
listPartsResponse.Initiator.ID = globalMinioDefaultOwnerID
@ -480,29 +494,29 @@ func generateListPartsResponse(partsInfo ListPartsInfo) ListPartsResponse {
}
// generates ListMultipartUploadsResponse for given bucket and ListMultipartsInfo.
func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMultipartsInfo) ListMultipartUploadsResponse {
func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMultipartsInfo, encodingType string) ListMultipartUploadsResponse {
listMultipartUploadsResponse := ListMultipartUploadsResponse{}
listMultipartUploadsResponse.Bucket = bucket
listMultipartUploadsResponse.Delimiter = multipartsInfo.Delimiter
listMultipartUploadsResponse.Delimiter = s3EncodeName(multipartsInfo.Delimiter, encodingType)
listMultipartUploadsResponse.IsTruncated = multipartsInfo.IsTruncated
listMultipartUploadsResponse.EncodingType = multipartsInfo.EncodingType
listMultipartUploadsResponse.Prefix = multipartsInfo.Prefix
listMultipartUploadsResponse.KeyMarker = multipartsInfo.KeyMarker
listMultipartUploadsResponse.NextKeyMarker = multipartsInfo.NextKeyMarker
listMultipartUploadsResponse.EncodingType = encodingType
listMultipartUploadsResponse.Prefix = s3EncodeName(multipartsInfo.Prefix, encodingType)
listMultipartUploadsResponse.KeyMarker = s3EncodeName(multipartsInfo.KeyMarker, encodingType)
listMultipartUploadsResponse.NextKeyMarker = s3EncodeName(multipartsInfo.NextKeyMarker, encodingType)
listMultipartUploadsResponse.MaxUploads = multipartsInfo.MaxUploads
listMultipartUploadsResponse.NextUploadIDMarker = multipartsInfo.NextUploadIDMarker
listMultipartUploadsResponse.UploadIDMarker = multipartsInfo.UploadIDMarker
listMultipartUploadsResponse.CommonPrefixes = make([]CommonPrefix, len(multipartsInfo.CommonPrefixes))
for index, commonPrefix := range multipartsInfo.CommonPrefixes {
listMultipartUploadsResponse.CommonPrefixes[index] = CommonPrefix{
Prefix: commonPrefix,
Prefix: s3EncodeName(commonPrefix, encodingType),
}
}
listMultipartUploadsResponse.Uploads = make([]Upload, len(multipartsInfo.Uploads))
for index, upload := range multipartsInfo.Uploads {
newUpload := Upload{}
newUpload.UploadID = upload.UploadID
newUpload.Key = upload.Object
newUpload.Key = s3EncodeName(upload.Object, encodingType)
newUpload.Initiated = upload.Initiated.UTC().Format(timeFormatAMZLong)
listMultipartUploadsResponse.Uploads[index] = newUpload
}

22
cmd/bucket-handlers-listobjects.go
View File

@ -18,6 +18,7 @@ package cmd
import (
"net/http"
"strings"
"github.com/gorilla/mux"
"github.com/minio/minio/cmd/crypto"
@ -32,12 +33,19 @@ import (
// - delimiter if set should be equal to '/', otherwise the request is rejected.
// - marker if set should have a common prefix with 'prefix' param, otherwise
// the request is rejected.
func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIErrorCode {
func validateListObjectsArgs(prefix, marker, delimiter, encodingType string, maxKeys int) APIErrorCode {
// Max keys cannot be negative.
if maxKeys < 0 {
return ErrInvalidMaxKeys
}
if encodingType != "" {
// Only url encoding type is supported
if strings.ToLower(encodingType) != "url" {
return ErrInvalidEncodingMethod
}
}
/// Minio special conditions for ListObjects.
// Verify if delimiter is anything other than '/', which we do not support.
@ -78,7 +86,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
urlValues := r.URL.Query()
// Extract all the listObjectsV2 query params to their native values.
prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _, errCode := getListObjectsV2Args(urlValues)
prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(urlValues)
if errCode != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
return
@ -86,7 +94,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
// Validate the query params before beginning to serve the request.
// fetch-owner is not validated since it is a boolean
if s3Error := validateListObjectsArgs(prefix, token, delimiter, maxKeys); s3Error != ErrNone {
if s3Error := validateListObjectsArgs(prefix, token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
return
}
@ -126,7 +134,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
}
response := generateListObjectsV2Response(bucket, prefix, token, listObjectsV2Info.NextContinuationToken, startAfter,
delimiter, fetchOwner, listObjectsV2Info.IsTruncated, maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes)
delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated, maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes)
// Write success response.
writeSuccessResponseXML(w, encodeResponse(response))
@ -158,14 +166,14 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
}
// Extract all the litsObjectsV1 query params to their native values.
prefix, marker, delimiter, maxKeys, _, s3Error := getListObjectsV1Args(r.URL.Query())
prefix, marker, delimiter, maxKeys, encodingType, s3Error := getListObjectsV1Args(r.URL.Query())
if s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
return
}
// Validate all the query params before beginning to serve the request.
if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
if s3Error := validateListObjectsArgs(prefix, marker, delimiter, encodingType, maxKeys); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
return
}
@ -204,7 +212,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
}
}
}
response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, maxKeys, listObjectsInfo)
response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType, maxKeys, listObjectsInfo)
// Write success response.
writeSuccessResponseXML(w, encodeResponse(response))

4
cmd/bucket-handlers.go
View File

@ -155,7 +155,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
return
}
prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _, errCode := getBucketMultipartResources(r.URL.Query())
prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, encodingType, errCode := getBucketMultipartResources(r.URL.Query())
if errCode != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
return
@ -180,7 +180,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
return
}
// generate response
response := generateListMultipartUploadsResponse(bucket, listMultipartsInfo)
response := generateListMultipartUploadsResponse(bucket, listMultipartsInfo, encodingType)
encodedSuccessResponse := encodeResponse(response)
// write success response.

4
cmd/object-handlers.go
View File

@ -2094,7 +2094,7 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
return
}
uploadID, partNumberMarker, maxParts, _, s3Error := getObjectResources(r.URL.Query())
uploadID, partNumberMarker, maxParts, encodingType, s3Error := getObjectResources(r.URL.Query())
if s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
return
@ -2145,7 +2145,7 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
}
}
response := generateListPartsResponse(listPartsInfo)
response := generateListPartsResponse(listPartsInfo, encodingType)
encodedSuccessResponse := encodeResponse(response)
// Write success response.

103
cmd/server_test.go
View File

@ -1658,62 +1658,59 @@ func (s *TestSuiteCommon) TestListObjectsHandler(c *check) {
c.Assert(err, nil)
c.Assert(response.StatusCode, http.StatusOK)
buffer1 := bytes.NewReader([]byte("Hello World"))
request, err = newTestSignedRequest("PUT", getPutObjectURL(s.endPoint, bucketName, "bar"),
int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey, s.signer)
c.Assert(err, nil)
for _, objectName := range []string{"foo bar 1", "foo bar 2"} {
buffer := bytes.NewReader([]byte("Hello World"))
request, err = newTestSignedRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName),
int64(buffer.Len()), buffer, s.accessKey, s.secretKey, s.signer)
c.Assert(err, nil)
client = http.Client{Transport: s.transport}
response, err = client.Do(request)
c.Assert(err, nil)
c.Assert(response.StatusCode, http.StatusOK)
client = http.Client{Transport: s.transport}
response, err = client.Do(request)
c.Assert(err, nil)
c.Assert(response.StatusCode, http.StatusOK)
}
// create listObjectsV1 request with valid parameters
request, err = newTestSignedRequest("GET", getListObjectsV1URL(s.endPoint, bucketName, "1000"),
0, nil, s.accessKey, s.secretKey, s.signer)
c.Assert(err, nil)
client = http.Client{Transport: s.transport}
// execute the HTTP request.
response, err = client.Do(request)
c.Assert(err, nil)
c.Assert(response.StatusCode, http.StatusOK)
var testCases = []struct {
getURL string
expectedStrings []string
}{
{getListObjectsV1URL(s.endPoint, bucketName, "", "1000", ""), []string{"<Key>foo bar 1</Key>", "<Key>foo bar 2</Key>"}},
{getListObjectsV1URL(s.endPoint, bucketName, "", "1000", "url"), []string{"<Key>foo+bar+1</Key>", "<Key>foo+bar+2</Key>"}},
{getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "", ""),
[]string{
"<Key>foo bar 1</Key>",
"<Key>foo bar 2</Key>",
"<Owner><ID></ID><DisplayName></DisplayName></Owner>",
},
},
{getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "true", ""),
[]string{
"<Key>foo bar 1</Key>",
"<Key>foo bar 2</Key>",
fmt.Sprintf("<Owner><ID>%s</ID><DisplayName></DisplayName></Owner>", globalMinioDefaultOwnerID),
},
},
{getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "", "url"), []string{"<Key>foo+bar+1</Key>", "<Key>foo+bar+2</Key>"}},
}
getContent, err := ioutil.ReadAll(response.Body)
c.Assert(err, nil)
c.Assert(strings.Contains(string(getContent), "<Key>bar</Key>"), true)
for i, testCase := range testCases {
// create listObjectsV1 request with valid parameters
request, err = newTestSignedRequest("GET", testCase.getURL, 0, nil, s.accessKey, s.secretKey, s.signer)
c.Assert(err, nil)
client = http.Client{Transport: s.transport}
// execute the HTTP request.
response, err = client.Do(request)
c.Assert(err, nil)
c.Assert(response.StatusCode, http.StatusOK)
// create listObjectsV2 request with valid parameters
request, err = newTestSignedRequest("GET", getListObjectsV2URL(s.endPoint, bucketName, "1000", ""),
0, nil, s.accessKey, s.secretKey, s.signer)
c.Assert(err, nil)
client = http.Client{Transport: s.transport}
// execute the HTTP request.
response, err = client.Do(request)
c.Assert(err, nil)
c.Assert(response.StatusCode, http.StatusOK)
getContent, err = ioutil.ReadAll(response.Body)
c.Assert(err, nil)
c.Assert(strings.Contains(string(getContent), "<Key>bar</Key>"), true)
c.Assert(strings.Contains(string(getContent), "<Owner><ID></ID><DisplayName></DisplayName></Owner>"), true)
// create listObjectsV2 request with valid parameters and fetch-owner activated
request, err = newTestSignedRequest("GET", getListObjectsV2URL(s.endPoint, bucketName, "1000", "true"),
0, nil, s.accessKey, s.secretKey, s.signer)
c.Assert(err, nil)
client = http.Client{Transport: s.transport}
// execute the HTTP request.
response, err = client.Do(request)
c.Assert(err, nil)
c.Assert(response.StatusCode, http.StatusOK)
getContent, err = ioutil.ReadAll(response.Body)
c.Assert(err, nil)
c.Assert(strings.Contains(string(getContent), "<Key>bar</Key>"), true)
c.Assert(strings.Contains(string(getContent), fmt.Sprintf("<Owner><ID>%s</ID><DisplayName></DisplayName></Owner>",
globalMinioDefaultOwnerID)), true)
getContent, err := ioutil.ReadAll(response.Body)
c.Assert(err, nil)
fmt.Printf("Test %d: %+v vs %+v\n", i+1, string(getContent), testCase.expectedStrings)
for _, expectedStr := range testCase.expectedStrings {
c.Assert(strings.Contains(string(getContent), expectedStr), true)
}
}
}
// TestListObjectsHandlerErrors - Setting invalid parameters to List Objects
@ -1733,7 +1730,7 @@ func (s *TestSuiteCommon) TestListObjectsHandlerErrors(c *check) {
c.Assert(response.StatusCode, http.StatusOK)
// create listObjectsV1 request with invalid value of max-keys parameter. max-keys is set to -2.
request, err = newTestSignedRequest("GET", getListObjectsV1URL(s.endPoint, bucketName, "-2"),
request, err = newTestSignedRequest("GET", getListObjectsV1URL(s.endPoint, bucketName, "", "-2", ""),
0, nil, s.accessKey, s.secretKey, s.signer)
c.Assert(err, nil)
client = http.Client{Transport: s.transport}
@ -1744,7 +1741,7 @@ func (s *TestSuiteCommon) TestListObjectsHandlerErrors(c *check) {
verifyError(c, response, "InvalidArgument", "Argument maxKeys must be an integer between 0 and 2147483647", http.StatusBadRequest)
// create listObjectsV2 request with invalid value of max-keys parameter. max-keys is set to -2.
request, err = newTestSignedRequest("GET", getListObjectsV2URL(s.endPoint, bucketName, "-2", ""),
request, err = newTestSignedRequest("GET", getListObjectsV2URL(s.endPoint, bucketName, "", "-2", "", ""),
0, nil, s.accessKey, s.secretKey, s.signer)
c.Assert(err, nil)
client = http.Client{Transport: s.transport}

14
cmd/test-utils_test.go
View File

@ -1458,16 +1458,19 @@ func getBucketLocationURL(endPoint, bucketName string) string {
}
// return URL for listing objects in the bucket with V1 legacy API.
func getListObjectsV1URL(endPoint, bucketName string, maxKeys string) string {
func getListObjectsV1URL(endPoint, bucketName, prefix, maxKeys, encodingType string) string {
queryValue := url.Values{}
if maxKeys != "" {
queryValue.Set("max-keys", maxKeys)
}
return makeTestTargetURL(endPoint, bucketName, "", queryValue)
if encodingType != "" {
queryValue.Set("encoding-type", encodingType)
}
return makeTestTargetURL(endPoint, bucketName, prefix, queryValue)
}
// return URL for listing objects in the bucket with V2 API.
func getListObjectsV2URL(endPoint, bucketName string, maxKeys string, fetchOwner string) string {
func getListObjectsV2URL(endPoint, bucketName, prefix, maxKeys, fetchOwner, encodingType string) string {
queryValue := url.Values{}
queryValue.Set("list-type", "2") // Enables list objects V2 URL.
if maxKeys != "" {
@ -1476,7 +1479,10 @@ func getListObjectsV2URL(endPoint, bucketName string, maxKeys string, fetchOwner
if fetchOwner != "" {
queryValue.Set("fetch-owner", fetchOwner)
}
return makeTestTargetURL(endPoint, bucketName, "", queryValue)
if encodingType != "" {
queryValue.Set("encoding-type", encodingType)
}
return makeTestTargetURL(endPoint, bucketName, prefix, queryValue)
}
// return URL for a new multipart upload.