diff --git a/cmd/api-errors.go b/cmd/api-errors.go index 97a11e98f..67c201cd0 100644 --- a/cmd/api-errors.go +++ b/cmd/api-errors.go @@ -73,6 +73,7 @@ const ( ErrInvalidCopyPartRange ErrInvalidCopyPartRangeSource ErrInvalidMaxKeys + ErrInvalidEncodingMethod ErrInvalidMaxUploads ErrInvalidMaxParts ErrInvalidPartNumberMarker @@ -357,6 +358,11 @@ var errorCodes = errorCodeMap{ Description: "Argument maxKeys must be an integer between 0 and 2147483647", HTTPStatusCode: http.StatusBadRequest, }, + ErrInvalidEncodingMethod: { + Code: "InvalidArgument", + Description: "Invalid Encoding Method specified in Request", + HTTPStatusCode: http.StatusBadRequest, + }, ErrInvalidMaxParts: { Code: "InvalidArgument", Description: "Argument max-parts must be an integer between 0 and 2147483647", diff --git a/cmd/api-response.go b/cmd/api-response.go index 49522150a..55b979de4 100644 --- a/cmd/api-response.go +++ b/cmd/api-response.go @@ -305,6 +305,21 @@ func getObjectLocation(r *http.Request, domains []string, bucket, object string) return u.String() } +// s3EncodeName encodes string in response when encodingType +// is specified in AWS S3 requests. +func s3EncodeName(name string, encodingType string) (result string) { + // Quick path to exit + if encodingType == "" { + return name + } + encodingType = strings.ToLower(encodingType) + switch encodingType { + case "url": + return url.QueryEscape(name) + } + return name +} + // generates ListBucketsResponse from array of BucketInfo which can be // serialized to match XML and JSON API spec output. func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse { @@ -327,7 +342,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse { } // generates an ListObjectsV1 response for the said bucket with other enumerated options. -func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse { +func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse { var contents []Object var prefixes []CommonPrefix var owner = Owner{} @@ -339,7 +354,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max if object.Name == "" { continue } - content.Key = object.Name + content.Key = s3EncodeName(object.Name, encodingType) content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong) if object.ETag != "" { content.ETag = "\"" + object.ETag + "\"" @@ -349,20 +364,20 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max content.Owner = owner contents = append(contents, content) } - // TODO - support EncodingType in xml decoding data.Name = bucket data.Contents = contents - data.Prefix = prefix - data.Marker = marker - data.Delimiter = delimiter + data.EncodingType = encodingType + data.Prefix = s3EncodeName(prefix, encodingType) + data.Marker = s3EncodeName(marker, encodingType) + data.Delimiter = s3EncodeName(delimiter, encodingType) data.MaxKeys = maxKeys - data.NextMarker = resp.NextMarker + data.NextMarker = s3EncodeName(resp.NextMarker, encodingType) data.IsTruncated = resp.IsTruncated for _, prefix := range resp.Prefixes { var prefixItem = CommonPrefix{} - prefixItem.Prefix = prefix + prefixItem.Prefix = s3EncodeName(prefix, encodingType) prefixes = append(prefixes, prefixItem) } data.CommonPrefixes = prefixes @@ -370,7 +385,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max } // generates an ListObjectsV2 response for the said bucket with other enumerated options. -func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string) ListObjectsV2Response { +func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string) ListObjectsV2Response { var contents []Object var commonPrefixes []CommonPrefix var owner = Owner{} @@ -385,7 +400,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, if object.Name == "" { continue } - content.Key = object.Name + content.Key = s3EncodeName(object.Name, encodingType) content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong) if object.ETag != "" { content.ETag = "\"" + object.ETag + "\"" @@ -395,20 +410,20 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, content.Owner = owner contents = append(contents, content) } - // TODO - support EncodingType in xml decoding data.Name = bucket data.Contents = contents + data.EncodingType = encodingType data.StartAfter = startAfter - data.Delimiter = delimiter - data.Prefix = prefix + data.Delimiter = s3EncodeName(delimiter, encodingType) + data.Prefix = s3EncodeName(prefix, encodingType) data.MaxKeys = maxKeys data.ContinuationToken = token data.NextContinuationToken = nextToken data.IsTruncated = isTruncated for _, prefix := range prefixes { var prefixItem = CommonPrefix{} - prefixItem.Prefix = prefix + prefixItem.Prefix = s3EncodeName(prefix, encodingType) commonPrefixes = append(commonPrefixes, prefixItem) } data.CommonPrefixes = commonPrefixes @@ -452,11 +467,10 @@ func generateCompleteMultpartUploadResponse(bucket, key, location, etag string) } // generates ListPartsResponse from ListPartsInfo. -func generateListPartsResponse(partsInfo ListPartsInfo) ListPartsResponse { - // TODO - support EncodingType in xml decoding +func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) ListPartsResponse { listPartsResponse := ListPartsResponse{} listPartsResponse.Bucket = partsInfo.Bucket - listPartsResponse.Key = partsInfo.Object + listPartsResponse.Key = s3EncodeName(partsInfo.Object, encodingType) listPartsResponse.UploadID = partsInfo.UploadID listPartsResponse.StorageClass = globalMinioDefaultStorageClass listPartsResponse.Initiator.ID = globalMinioDefaultOwnerID @@ -480,29 +494,29 @@ func generateListPartsResponse(partsInfo ListPartsInfo) ListPartsResponse { } // generates ListMultipartUploadsResponse for given bucket and ListMultipartsInfo. -func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMultipartsInfo) ListMultipartUploadsResponse { +func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMultipartsInfo, encodingType string) ListMultipartUploadsResponse { listMultipartUploadsResponse := ListMultipartUploadsResponse{} listMultipartUploadsResponse.Bucket = bucket - listMultipartUploadsResponse.Delimiter = multipartsInfo.Delimiter + listMultipartUploadsResponse.Delimiter = s3EncodeName(multipartsInfo.Delimiter, encodingType) listMultipartUploadsResponse.IsTruncated = multipartsInfo.IsTruncated - listMultipartUploadsResponse.EncodingType = multipartsInfo.EncodingType - listMultipartUploadsResponse.Prefix = multipartsInfo.Prefix - listMultipartUploadsResponse.KeyMarker = multipartsInfo.KeyMarker - listMultipartUploadsResponse.NextKeyMarker = multipartsInfo.NextKeyMarker + listMultipartUploadsResponse.EncodingType = encodingType + listMultipartUploadsResponse.Prefix = s3EncodeName(multipartsInfo.Prefix, encodingType) + listMultipartUploadsResponse.KeyMarker = s3EncodeName(multipartsInfo.KeyMarker, encodingType) + listMultipartUploadsResponse.NextKeyMarker = s3EncodeName(multipartsInfo.NextKeyMarker, encodingType) listMultipartUploadsResponse.MaxUploads = multipartsInfo.MaxUploads listMultipartUploadsResponse.NextUploadIDMarker = multipartsInfo.NextUploadIDMarker listMultipartUploadsResponse.UploadIDMarker = multipartsInfo.UploadIDMarker listMultipartUploadsResponse.CommonPrefixes = make([]CommonPrefix, len(multipartsInfo.CommonPrefixes)) for index, commonPrefix := range multipartsInfo.CommonPrefixes { listMultipartUploadsResponse.CommonPrefixes[index] = CommonPrefix{ - Prefix: commonPrefix, + Prefix: s3EncodeName(commonPrefix, encodingType), } } listMultipartUploadsResponse.Uploads = make([]Upload, len(multipartsInfo.Uploads)) for index, upload := range multipartsInfo.Uploads { newUpload := Upload{} newUpload.UploadID = upload.UploadID - newUpload.Key = upload.Object + newUpload.Key = s3EncodeName(upload.Object, encodingType) newUpload.Initiated = upload.Initiated.UTC().Format(timeFormatAMZLong) listMultipartUploadsResponse.Uploads[index] = newUpload } diff --git a/cmd/bucket-handlers-listobjects.go b/cmd/bucket-handlers-listobjects.go index f94c49846..588ca934e 100644 --- a/cmd/bucket-handlers-listobjects.go +++ b/cmd/bucket-handlers-listobjects.go @@ -18,6 +18,7 @@ package cmd import ( "net/http" + "strings" "github.com/gorilla/mux" "github.com/minio/minio/cmd/crypto" @@ -32,12 +33,19 @@ import ( // - delimiter if set should be equal to '/', otherwise the request is rejected. // - marker if set should have a common prefix with 'prefix' param, otherwise // the request is rejected. -func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIErrorCode { +func validateListObjectsArgs(prefix, marker, delimiter, encodingType string, maxKeys int) APIErrorCode { // Max keys cannot be negative. if maxKeys < 0 { return ErrInvalidMaxKeys } + if encodingType != "" { + // Only url encoding type is supported + if strings.ToLower(encodingType) != "url" { + return ErrInvalidEncodingMethod + } + } + /// Minio special conditions for ListObjects. // Verify if delimiter is anything other than '/', which we do not support. @@ -78,7 +86,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http urlValues := r.URL.Query() // Extract all the listObjectsV2 query params to their native values. - prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _, errCode := getListObjectsV2Args(urlValues) + prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(urlValues) if errCode != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r)) return @@ -86,7 +94,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http // Validate the query params before beginning to serve the request. // fetch-owner is not validated since it is a boolean - if s3Error := validateListObjectsArgs(prefix, token, delimiter, maxKeys); s3Error != ErrNone { + if s3Error := validateListObjectsArgs(prefix, token, delimiter, encodingType, maxKeys); s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) return } @@ -126,7 +134,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http } response := generateListObjectsV2Response(bucket, prefix, token, listObjectsV2Info.NextContinuationToken, startAfter, - delimiter, fetchOwner, listObjectsV2Info.IsTruncated, maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes) + delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated, maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes) // Write success response. writeSuccessResponseXML(w, encodeResponse(response)) @@ -158,14 +166,14 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http } // Extract all the litsObjectsV1 query params to their native values. - prefix, marker, delimiter, maxKeys, _, s3Error := getListObjectsV1Args(r.URL.Query()) + prefix, marker, delimiter, maxKeys, encodingType, s3Error := getListObjectsV1Args(r.URL.Query()) if s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) return } // Validate all the query params before beginning to serve the request. - if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone { + if s3Error := validateListObjectsArgs(prefix, marker, delimiter, encodingType, maxKeys); s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) return } @@ -204,7 +212,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http } } } - response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, maxKeys, listObjectsInfo) + response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType, maxKeys, listObjectsInfo) // Write success response. writeSuccessResponseXML(w, encodeResponse(response)) diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index a66047d4e..a4b15491b 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -155,7 +155,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, return } - prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _, errCode := getBucketMultipartResources(r.URL.Query()) + prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, encodingType, errCode := getBucketMultipartResources(r.URL.Query()) if errCode != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r)) return @@ -180,7 +180,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, return } // generate response - response := generateListMultipartUploadsResponse(bucket, listMultipartsInfo) + response := generateListMultipartUploadsResponse(bucket, listMultipartsInfo, encodingType) encodedSuccessResponse := encodeResponse(response) // write success response. diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 49b5a1741..3305d3bc6 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -2094,7 +2094,7 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht return } - uploadID, partNumberMarker, maxParts, _, s3Error := getObjectResources(r.URL.Query()) + uploadID, partNumberMarker, maxParts, encodingType, s3Error := getObjectResources(r.URL.Query()) if s3Error != ErrNone { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) return @@ -2145,7 +2145,7 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht } } - response := generateListPartsResponse(listPartsInfo) + response := generateListPartsResponse(listPartsInfo, encodingType) encodedSuccessResponse := encodeResponse(response) // Write success response. diff --git a/cmd/server_test.go b/cmd/server_test.go index 8806cc984..bd97217e3 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -1658,62 +1658,59 @@ func (s *TestSuiteCommon) TestListObjectsHandler(c *check) { c.Assert(err, nil) c.Assert(response.StatusCode, http.StatusOK) - buffer1 := bytes.NewReader([]byte("Hello World")) - request, err = newTestSignedRequest("PUT", getPutObjectURL(s.endPoint, bucketName, "bar"), - int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey, s.signer) - c.Assert(err, nil) + for _, objectName := range []string{"foo bar 1", "foo bar 2"} { + buffer := bytes.NewReader([]byte("Hello World")) + request, err = newTestSignedRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buffer.Len()), buffer, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) - client = http.Client{Transport: s.transport} - response, err = client.Do(request) - c.Assert(err, nil) - c.Assert(response.StatusCode, http.StatusOK) + client = http.Client{Transport: s.transport} + response, err = client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + } - // create listObjectsV1 request with valid parameters - request, err = newTestSignedRequest("GET", getListObjectsV1URL(s.endPoint, bucketName, "1000"), - 0, nil, s.accessKey, s.secretKey, s.signer) - c.Assert(err, nil) - client = http.Client{Transport: s.transport} - // execute the HTTP request. - response, err = client.Do(request) - c.Assert(err, nil) - c.Assert(response.StatusCode, http.StatusOK) + var testCases = []struct { + getURL string + expectedStrings []string + }{ + {getListObjectsV1URL(s.endPoint, bucketName, "", "1000", ""), []string{"foo bar 1", "foo bar 2"}}, + {getListObjectsV1URL(s.endPoint, bucketName, "", "1000", "url"), []string{"foo+bar+1", "foo+bar+2"}}, + {getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "", ""), + []string{ + "foo bar 1", + "foo bar 2", + "", + }, + }, + {getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "true", ""), + []string{ + "foo bar 1", + "foo bar 2", + fmt.Sprintf("%s", globalMinioDefaultOwnerID), + }, + }, + {getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "", "url"), []string{"foo+bar+1", "foo+bar+2"}}, + } - getContent, err := ioutil.ReadAll(response.Body) - c.Assert(err, nil) - c.Assert(strings.Contains(string(getContent), "bar"), true) + for i, testCase := range testCases { + // create listObjectsV1 request with valid parameters + request, err = newTestSignedRequest("GET", testCase.getURL, 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + client = http.Client{Transport: s.transport} + // execute the HTTP request. + response, err = client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) - // create listObjectsV2 request with valid parameters - request, err = newTestSignedRequest("GET", getListObjectsV2URL(s.endPoint, bucketName, "1000", ""), - 0, nil, s.accessKey, s.secretKey, s.signer) - c.Assert(err, nil) - client = http.Client{Transport: s.transport} - // execute the HTTP request. - response, err = client.Do(request) - c.Assert(err, nil) - c.Assert(response.StatusCode, http.StatusOK) - - getContent, err = ioutil.ReadAll(response.Body) - c.Assert(err, nil) - c.Assert(strings.Contains(string(getContent), "bar"), true) - c.Assert(strings.Contains(string(getContent), ""), true) - - // create listObjectsV2 request with valid parameters and fetch-owner activated - request, err = newTestSignedRequest("GET", getListObjectsV2URL(s.endPoint, bucketName, "1000", "true"), - 0, nil, s.accessKey, s.secretKey, s.signer) - c.Assert(err, nil) - client = http.Client{Transport: s.transport} - // execute the HTTP request. - response, err = client.Do(request) - c.Assert(err, nil) - c.Assert(response.StatusCode, http.StatusOK) - - getContent, err = ioutil.ReadAll(response.Body) - c.Assert(err, nil) - - c.Assert(strings.Contains(string(getContent), "bar"), true) - c.Assert(strings.Contains(string(getContent), fmt.Sprintf("%s", - globalMinioDefaultOwnerID)), true) + getContent, err := ioutil.ReadAll(response.Body) + c.Assert(err, nil) + fmt.Printf("Test %d: %+v vs %+v\n", i+1, string(getContent), testCase.expectedStrings) + for _, expectedStr := range testCase.expectedStrings { + c.Assert(strings.Contains(string(getContent), expectedStr), true) + } + } } // TestListObjectsHandlerErrors - Setting invalid parameters to List Objects @@ -1733,7 +1730,7 @@ func (s *TestSuiteCommon) TestListObjectsHandlerErrors(c *check) { c.Assert(response.StatusCode, http.StatusOK) // create listObjectsV1 request with invalid value of max-keys parameter. max-keys is set to -2. - request, err = newTestSignedRequest("GET", getListObjectsV1URL(s.endPoint, bucketName, "-2"), + request, err = newTestSignedRequest("GET", getListObjectsV1URL(s.endPoint, bucketName, "", "-2", ""), 0, nil, s.accessKey, s.secretKey, s.signer) c.Assert(err, nil) client = http.Client{Transport: s.transport} @@ -1744,7 +1741,7 @@ func (s *TestSuiteCommon) TestListObjectsHandlerErrors(c *check) { verifyError(c, response, "InvalidArgument", "Argument maxKeys must be an integer between 0 and 2147483647", http.StatusBadRequest) // create listObjectsV2 request with invalid value of max-keys parameter. max-keys is set to -2. - request, err = newTestSignedRequest("GET", getListObjectsV2URL(s.endPoint, bucketName, "-2", ""), + request, err = newTestSignedRequest("GET", getListObjectsV2URL(s.endPoint, bucketName, "", "-2", "", ""), 0, nil, s.accessKey, s.secretKey, s.signer) c.Assert(err, nil) client = http.Client{Transport: s.transport} diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go index 09624573a..b2fdfc039 100644 --- a/cmd/test-utils_test.go +++ b/cmd/test-utils_test.go @@ -1458,16 +1458,19 @@ func getBucketLocationURL(endPoint, bucketName string) string { } // return URL for listing objects in the bucket with V1 legacy API. -func getListObjectsV1URL(endPoint, bucketName string, maxKeys string) string { +func getListObjectsV1URL(endPoint, bucketName, prefix, maxKeys, encodingType string) string { queryValue := url.Values{} if maxKeys != "" { queryValue.Set("max-keys", maxKeys) } - return makeTestTargetURL(endPoint, bucketName, "", queryValue) + if encodingType != "" { + queryValue.Set("encoding-type", encodingType) + } + return makeTestTargetURL(endPoint, bucketName, prefix, queryValue) } // return URL for listing objects in the bucket with V2 API. -func getListObjectsV2URL(endPoint, bucketName string, maxKeys string, fetchOwner string) string { +func getListObjectsV2URL(endPoint, bucketName, prefix, maxKeys, fetchOwner, encodingType string) string { queryValue := url.Values{} queryValue.Set("list-type", "2") // Enables list objects V2 URL. if maxKeys != "" { @@ -1476,7 +1479,10 @@ func getListObjectsV2URL(endPoint, bucketName string, maxKeys string, fetchOwner if fetchOwner != "" { queryValue.Set("fetch-owner", fetchOwner) } - return makeTestTargetURL(endPoint, bucketName, "", queryValue) + if encodingType != "" { + queryValue.Set("encoding-type", encodingType) + } + return makeTestTargetURL(endPoint, bucketName, prefix, queryValue) } // return URL for a new multipart upload.