MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-10-29 15:55:00 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix: normalize object layer inputs (#11534)

Browse Source 
Cases where we have applications making request
for `//` in object names make sure that all
are normalized to `/` and all such requests that
are prefixed '/' are removed. To ensure a
consistent view from all operations.
This commit is contained in:
Harshavardhana 2021-03-09 12:58:22 -08:00 committed by GitHub
parent eac66e67ec
commit 691035832a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 152 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cmd/acl-handlers.go
View File

@ -20,7 +20,6 @@ import (
"encoding/xml"
"io"
"net/http"
"net/url"
"github.com/gorilla/mux"
xhttp "github.com/minio/minio/cmd/http"
@ -180,7 +179,7 @@ func (api objectAPIHandlers) PutObjectACLHandler(w http.ResponseWriter, r *http.
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
object, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -244,7 +243,7 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
object, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return

60
cmd/api-resources.go
View File

@ -36,8 +36,19 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string,
maxkeys = maxObjectList
}
prefix = values.Get("prefix")
marker = values.Get("marker")
var err error
prefix, err = unescapePath(values.Get("prefix"))
if err != nil {
errCode = ErrInvalidRequest
return
}
marker, err = unescapePath(values.Get("marker"))
if err != nil {
errCode = ErrInvalidRequest
return
}
delimiter = values.Get("delimiter")
encodingType = values.Get("encoding-type")
return
@ -56,8 +67,19 @@ func getListBucketObjectVersionsArgs(values url.Values) (prefix, marker, delimit
maxkeys = maxObjectList
}
prefix = values.Get("prefix")
marker = values.Get("key-marker")
var err error
prefix, err = unescapePath(values.Get("prefix"))
if err != nil {
errCode = ErrInvalidRequest
return
}
marker, err = unescapePath(values.Get("key-marker"))
if err != nil {
errCode = ErrInvalidRequest
return
}
delimiter = values.Get("delimiter")
encodingType = values.Get("encoding-type")
versionIDMarker = values.Get("version-id-marker")
@ -86,8 +108,19 @@ func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimit
maxkeys = maxObjectList
}
prefix = values.Get("prefix")
startAfter = values.Get("start-after")
var err error
prefix, err = unescapePath(values.Get("prefix"))
if err != nil {
errCode = ErrInvalidRequest
return
}
startAfter, err = unescapePath(values.Get("start-after"))
if err != nil {
errCode = ErrInvalidRequest
return
}
delimiter = values.Get("delimiter")
fetchOwner = values.Get("fetch-owner") == "true"
encodingType = values.Get("encoding-type")
@ -117,8 +150,19 @@ func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadID
maxUploads = maxUploadsList
}
prefix = values.Get("prefix")
keyMarker = values.Get("key-marker")
var err error
prefix, err = unescapePath(values.Get("prefix"))
if err != nil {
errCode = ErrInvalidRequest
return
}
keyMarker, err = unescapePath(values.Get("key-marker"))
if err != nil {
errCode = ErrInvalidRequest
return
}
uploadIDMarker = values.Get("upload-id-marker")
delimiter = values.Get("delimiter")
encodingType = values.Get("encoding-type")

27
cmd/api-response.go
View File

@ -416,9 +416,11 @@ func getObjectLocation(r *http.Request, domains []string, bucket, object string)
func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
listbuckets := make([]Bucket, 0, len(buckets))
var data = ListBucketsResponse{}
var owner = Owner{}
var owner = Owner{
ID: globalMinioDefaultOwnerID,
DisplayName: "minio",
}
owner.ID = globalMinioDefaultOwnerID
for _, bucket := range buckets {
var listbucket = Bucket{}
listbucket.Name = bucket.Name
@ -435,10 +437,12 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
// generates an ListBucketVersions response for the said bucket with other enumerated options.
func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo) ListVersionsResponse {
versions := make([]ObjectVersion, 0, len(resp.Objects))
var owner = Owner{}
var owner = Owner{
ID: globalMinioDefaultOwnerID,
DisplayName: "minio",
}
var data = ListVersionsResponse{}
owner.ID = globalMinioDefaultOwnerID
for _, object := range resp.Objects {
var content = ObjectVersion{}
if object.Name == "" {
@ -491,10 +495,12 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
// generates an ListObjectsV1 response for the said bucket with other enumerated options.
func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
contents := make([]Object, 0, len(resp.Objects))
var owner = Owner{}
var owner = Owner{
ID: globalMinioDefaultOwnerID,
DisplayName: "minio",
}
var data = ListObjectsResponse{}
owner.ID = globalMinioDefaultOwnerID
for _, object := range resp.Objects {
var content = Object{}
if object.Name == "" {
@ -538,12 +544,11 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
// generates an ListObjectsV2 response for the said bucket with other enumerated options.
func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string, metadata bool) ListObjectsV2Response {
contents := make([]Object, 0, len(objects))
var owner = Owner{}
var data = ListObjectsV2Response{}
if fetchOwner {
owner.ID = globalMinioDefaultOwnerID
var owner = Owner{
ID: globalMinioDefaultOwnerID,
DisplayName: "minio",
}
var data = ListObjectsV2Response{}
for _, object := range objects {
var content = Object{}

6
cmd/bucket-handlers.go
View File

@ -855,6 +855,12 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
}
object := formValues.Get("Key")
object, err = unescapePath(object)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
successRedirect := formValues.Get("success_action_redirect")
successStatus := formValues.Get("success_action_status")
var redirectURL *url.URL

16
cmd/config/cache/help.go vendored
View File

@ -40,19 +40,13 @@ var (
},
config.HelpKV{
Key: Exclude,
Description: `comma separated wildcard exclusion patterns e.g. "bucket/*.tmp,*.exe"`,
Description: `exclude cache for following patterns e.g. "bucket/*.tmp,*.exe"`,
Optional: true,
Type: "csv",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
config.HelpKV{
Key: After,
Description: `minimum accesses before caching an object`,
Description: `minimum number of access before caching an object`,
Optional: true,
Type: "number",
},
@ -80,5 +74,11 @@ var (
Optional: true,
Type: "string",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
)

40
cmd/object-handlers.go
View File

@ -115,7 +115,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
}
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
object, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -322,7 +322,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
}
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
object, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -561,7 +561,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
}
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
object, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -876,7 +876,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
}
vars := mux.Vars(r)
dstBucket := vars["bucket"]
dstObject, err := url.PathUnescape(vars["object"])
dstObject, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -1395,7 +1395,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
object, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -1703,7 +1703,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
object, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -1831,7 +1831,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
vars := mux.Vars(r)
dstBucket := vars["bucket"]
dstObject, err := url.PathUnescape(vars["object"])
dstObject, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -2153,7 +2153,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
}
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
object, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -2391,7 +2391,7 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter,
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
object, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -2431,7 +2431,7 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
object, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -2570,7 +2570,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
object, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -2765,7 +2765,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
object, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -2941,7 +2941,7 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
object, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -3040,7 +3040,7 @@ func (api objectAPIHandlers) GetObjectLegalHoldHandler(w http.ResponseWriter, r
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
object, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -3105,7 +3105,7 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
object, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -3211,7 +3211,7 @@ func (api objectAPIHandlers) GetObjectRetentionHandler(w http.ResponseWriter, r
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
object, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -3271,7 +3271,7 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
object, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -3321,7 +3321,7 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
object, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -3409,7 +3409,7 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
object, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
@ -3471,7 +3471,7 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
object, err := unescapePath(vars["object"])
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return

2
cmd/object-handlers_test.go
View File

@ -2983,7 +2983,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
}
if actualError.Key != objectName {
t.Errorf("MinIO %s: error response object name differs from expected value", instanceType)
t.Errorf("MinIO %s: error response object name (%s) differs from expected value (%s)", instanceType, actualError.Key, objectName)
}
}

19
cmd/server_test.go
View File

@ -1267,20 +1267,7 @@ func (s *TestSuiteCommon) TestPutObjectLongName(c *check) {
c.Assert(response.StatusCode, http.StatusBadRequest)
verifyError(c, response, "KeyTooLongError", "Your key is too long", http.StatusBadRequest)
// make object name with prefix as slash
longObjName = fmt.Sprintf("/%0255d/%0255d", 1, 1)
buffer = bytes.NewReader([]byte("hello world"))
// create new HTTP request to insert the object.
request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, longObjName),
int64(buffer.Len()), buffer, s.accessKey, s.secretKey, s.signer)
c.Assert(err, nil)
// execute the HTTP request.
response, err = s.client.Do(request)
c.Assert(err, nil)
c.Assert(response.StatusCode, http.StatusBadRequest)
verifyError(c, response, "XMinioInvalidObjectName", "Object name contains a leading slash.", http.StatusBadRequest)
//make object name as unsupported
// make object name as unsupported
longObjName = fmt.Sprintf("%0256d", 1)
buffer = bytes.NewReader([]byte("hello world"))
request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, longObjName),
@ -1595,14 +1582,14 @@ func (s *TestSuiteCommon) TestListObjectsHandler(c *check) {
[]string{
"<Key>foo bar 1</Key>",
"<Key>foo bar 2</Key>",
"<Owner><ID></ID><DisplayName></DisplayName></Owner>",
fmt.Sprintf("<Owner><ID>%s</ID><DisplayName>minio</DisplayName></Owner>", globalMinioDefaultOwnerID),
},
},
{getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "true", ""),
[]string{
"<Key>foo bar 1</Key>",
"<Key>foo bar 2</Key>",
fmt.Sprintf("<Owner><ID>%s</ID><DisplayName></DisplayName></Owner>", globalMinioDefaultOwnerID),
fmt.Sprintf("<Owner><ID>%s</ID><DisplayName>minio</DisplayName></Owner>", globalMinioDefaultOwnerID),
},
},
{getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "", "url"), []string{"<Key>foo+bar+1</Key>", "<Key>foo+bar+2</Key>"}},

50
cmd/utils.go
View File

@ -29,6 +29,7 @@ import (
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"reflect"
"runtime"
@ -686,18 +687,51 @@ func ceilFrac(numerator, denominator int64) (ceil int64) {
return
}
// unescapeGeneric is similar to url.PathUnescape or url.QueryUnescape
// depending on input, additionally also handles situations such as
// `//` are normalized as `/`, also removes any `/` prefix before
// returning.
func unescapeGeneric(p string, escapeFn func(string) (string, error)) (string, error) {
ep, err := escapeFn(p)
if err != nil {
return "", err
}
if len(ep) > 0 && ep[0] == '/' {
// Path ends with '/' preserve it
if ep[len(ep)-1] == '/' {
ep = path.Clean(ep)
ep += slashSeparator
} else {
ep = path.Clean(ep)
}
ep = ep[1:]
}
return ep, nil
}
// unescapePath is similar to unescapeGeneric but for specifically
// path unescaping.
func unescapePath(p string) (string, error) {
return unescapeGeneric(p, url.PathUnescape)
}
// similar to unescapeGeneric but never returns any error if the unescaping
// fails, returns the input as is in such occasion, not meant to be
// used where strict validation is expected.
func likelyUnescapeGeneric(p string, escapeFn func(string) (string, error)) string {
ep, err := unescapeGeneric(p, escapeFn)
if err != nil {
return p
}
return ep
}
// Returns context with ReqInfo details set in the context.
func newContext(r *http.Request, w http.ResponseWriter, api string) context.Context {
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
if err != nil {
object = vars["object"]
}
prefix, err := url.QueryUnescape(vars["prefix"])
if err != nil {
prefix = vars["prefix"]
}
object := likelyUnescapeGeneric(vars["object"], url.PathUnescape)
prefix := likelyUnescapeGeneric(vars["prefix"], url.QueryUnescape)
if prefix != "" {
object = prefix
}

4
cmd/web-handlers.go
View File

@ -1107,7 +1107,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
object, err := unescapePath(vars["object"])
if err != nil {
writeWebErrorResponse(w, err)
return
@ -1353,7 +1353,7 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]
object, err := url.PathUnescape(vars["object"])
object, err := unescapePath(vars["object"])
if err != nil {
writeWebErrorResponse(w, err)
return