From ee7e70c992e87224bfe67bc2fcbd454541c23ae0 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 13 Sep 2016 19:00:01 -0700 Subject: [PATCH] tests: Add tests for ListMultipartUploads, DeleteMultipleObjects. (#2649) Additionally adds PostPolicyHandler tests. --- cmd/bucket-handlers_test.go | 109 ++++++++++++++++++- cmd/fs-v1_test.go | 2 +- cmd/post-policy_test.go | 201 ++++++++++++++++++++++++++++++++++++ cmd/server_test.go | 32 ++++-- cmd/test-utils_test.go | 25 ++++- 5 files changed, 353 insertions(+), 16 deletions(-) create mode 100644 cmd/post-policy_test.go diff --git a/cmd/bucket-handlers_test.go b/cmd/bucket-handlers_test.go index 247774c83..82f1e2586 100644 --- a/cmd/bucket-handlers_test.go +++ b/cmd/bucket-handlers_test.go @@ -93,10 +93,10 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType string, t TestEr for i, testCase := range testCases { // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. rec := httptest.NewRecorder() - // construct HTTP request for PUT bucket policy endpoint. + // construct HTTP request for Get bucket location. req, err := newTestSignedRequest("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey) if err != nil { - t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: %v", i+1, instanceType, err) + t.Fatalf("Test %d: %s: Failed to create HTTP request for GetBucketLocationHandler: %v", i+1, instanceType, err) } // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler. // Call the ServeHTTP to execute the handler. @@ -186,10 +186,10 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType string, t TestErrHandle for i, testCase := range testCases { // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. rec := httptest.NewRecorder() - // construct HTTP request for PUT bucket policy endpoint. + // construct HTTP request for HEAD bucket. req, err := newTestSignedRequest("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey) if err != nil { - t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: %v", i+1, instanceType, err) + t.Fatalf("Test %d: %s: Failed to create HTTP request for HeadBucketHandler: %v", i+1, instanceType, err) } // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler. // Call the ServeHTTP to execute the handler. @@ -199,3 +199,104 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType string, t TestErrHandle } } } + +// Wrapper for calling TestListMultipartUploadsHandler tests for both XL multiple disks and single node setup. +func TestListMultipartUploadsHandler(t *testing.T) { + ExecObjectLayerTest(t, testListMultipartUploads) +} + +// testListMultipartUploadsHandler - Tests validate listing of multipart uploads. +func testListMultipartUploadsHandler(obj ObjectLayer, instanceType string, t TestErrHandler) { + initBucketPolicies(obj) + + // get random bucket name. + bucketName := getRandomBucketName() + + // Register the API end points with XL/FS object layer. + apiRouter := initTestAPIEndPoints(obj, []string{"ListMultipartUploads"}) + // initialize the server and obtain the credentials and root. + // credentials are necessary to sign the HTTP request. + rootPath, err := newTestConfig("us-east-1") + if err != nil { + t.Fatalf("Init Test config failed") + } + // remove the root folder after the test ends. + defer removeAll(rootPath) + + credentials := serverConfig.GetCredential() + + // bucketnames[0]. + // objectNames[0]. + // uploadIds [0]. + // Create bucket before initiating NewMultipartUpload. + err = obj.MakeBucket(bucketName) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + // Collection of non-exhaustive ListMultipartUploads test cases, valid errors + // and success responses. + testCases := []struct { + // Inputs to ListMultipartUploads. + bucket string + prefix string + keyMarker string + uploadIDMarker string + delimiter string + maxUploads string + expectedRespStatus int + shouldPass bool + }{ + // 1 - invalid bucket name. + {".test", "", "", "", "", "0", http.StatusBadRequest, false}, + // 2 - bucket not found. + {"volatile-bucket-1", "", "", "", "", "0", http.StatusNotFound, false}, + // 3 - invalid delimiter. + {bucketName, "", "", "", "-", "0", http.StatusBadRequest, false}, + // 4 - invalid prefix and marker combination. + {bucketName, "asia", "europe-object", "", "", "0", http.StatusNotImplemented, false}, + // 5 - invalid upload id and marker combination. + {bucketName, "asia", "asia/europe/", "abc", "", "0", http.StatusBadRequest, false}, + // 6 - invalid max upload id. + {bucketName, "", "", "", "", "-1", http.StatusBadRequest, false}, + // 7 - good case delimiter. + {bucketName, "", "", "", "/", "100", http.StatusOK, true}, + // 8 - good case without delimiter. + {bucketName, "", "", "", "", "100", http.StatusOK, true}, + } + + for i, testCase := range testCases { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + + // construct HTTP request for List multipart uploads endpoint. + u := getListMultipartUploadsURLWithParams("", testCase.bucket, testCase.prefix, testCase.keyMarker, testCase.uploadIDMarker, testCase.delimiter, testCase.maxUploads) + req, gerr := newTestSignedRequest("GET", u, 0, nil, credentials.AccessKeyID, credentials.SecretAccessKey) + if gerr != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for ListMultipartUploadsHandler: %v", i+1, instanceType, gerr) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + if rec.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) + } + } + + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + + // construct HTTP request for List multipart uploads endpoint. + u := getListMultipartUploadsURLWithParams("", bucketName, "", "", "", "", "") + req, err := newTestSignedRequest("GET", u, 0, nil, "", "") // Generate an anonymous request. + if err != nil { + t.Fatalf("Test %s: Failed to create HTTP request for ListMultipartUploadsHandler: %v", instanceType, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusForbidden { + t.Errorf("Test %s: Expected the response status to be `http.StatusForbidden`, but instead found `%d`", instanceType, rec.Code) + } +} diff --git a/cmd/fs-v1_test.go b/cmd/fs-v1_test.go index 0e7890a4b..37ebd0e6b 100644 --- a/cmd/fs-v1_test.go +++ b/cmd/fs-v1_test.go @@ -93,7 +93,7 @@ func TestFSShutdown(t *testing.T) { for i := 1; i <= 5; i++ { naughty := newNaughtyDisk(fsStorage, map[int]error{i: errFaultyDisk}, nil) fs.storage = naughty - if err := fs.Shutdown(); err != errFaultyDisk { + if err := fs.Shutdown(); errorCause(err) != errFaultyDisk { t.Fatal(i, ", Got unexpected fs shutdown error: ", err) } } diff --git a/cmd/post-policy_test.go b/cmd/post-policy_test.go new file mode 100644 index 000000000..434f62863 --- /dev/null +++ b/cmd/post-policy_test.go @@ -0,0 +1,201 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "bytes" + "encoding/base64" + "fmt" + "mime/multipart" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +const ( + expirationDateFormat = "2006-01-02T15:04:05.999Z" + iso8601DateFormat = "20060102T150405Z" +) + +// newPostPolicyBytes - creates a bare bones postpolicy string with key and bucket matches. +func newPostPolicyBytes(credential, bucketName, objectKey string, expiration time.Time) []byte { + t := time.Now().UTC() + // Add the expiration date. + expirationStr := fmt.Sprintf(`"expiration": "%s"`, expiration.Format(expirationDateFormat)) + // Add the bucket condition, only accept buckets equal to the one passed. + bucketConditionStr := fmt.Sprintf(`["eq", "$bucket", "%s"]`, bucketName) + // Add the key condition, only accept keys equal to the one passed. + keyConditionStr := fmt.Sprintf(`["eq", "$key", "%s"]`, objectKey) + // Add the algorithm condition, only accept AWS SignV4 Sha256. + algorithmConditionStr := `["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"]` + // Add the date condition, only accept the current date. + dateConditionStr := fmt.Sprintf(`["eq", "$x-amz-date", "%s"]`, t.Format(iso8601DateFormat)) + // Add the credential string, only accept the credential passed. + credentialConditionStr := fmt.Sprintf(`["eq", "$x-amz-credential", "%s"]`, credential) + + // Combine all conditions into one string. + conditionStr := fmt.Sprintf(`"conditions":[%s, %s, %s, %s, %s]`, bucketConditionStr, keyConditionStr, algorithmConditionStr, dateConditionStr, credentialConditionStr) + retStr := "{" + retStr = retStr + expirationStr + "," + retStr = retStr + conditionStr + retStr = retStr + "}" + + return []byte(retStr) +} + +// Wrapper for calling TestPostPolicyHandlerHandler tests for both XL multiple disks and single node setup. +func TestPostPolicyHandler(t *testing.T) { + ExecObjectLayerTest(t, testPostPolicyHandler) +} + +// testPostPolicyHandler - Tests validate post policy handler uploading objects. +func testPostPolicyHandler(obj ObjectLayer, instanceType string, t TestErrHandler) { + // get random bucket name. + bucketName := getRandomBucketName() + + // Register the API end points with XL/FS object layer. + apiRouter := initTestAPIEndPoints(obj, []string{"PostPolicy"}) + + // initialize the server and obtain the credentials and root. + // credentials are necessary to sign the HTTP request. + rootPath, err := newTestConfig("us-east-1") + if err != nil { + t.Fatalf("Init Test config failed") + } + // remove the root folder after the test ends. + defer removeAll(rootPath) + + // bucketnames[0]. + // objectNames[0]. + // uploadIds [0]. + // Create bucket before initiating NewMultipartUpload. + err = obj.MakeBucket(bucketName) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + // Collection of non-exhaustive ListMultipartUploads test cases, valid errors + // and success responses. + testCases := []struct { + objectName string + data []byte + expectedRespStatus int + shouldPass bool + }{ + // Success case. + { + objectName: "test", + data: []byte("Hello, World"), + expectedRespStatus: http.StatusNoContent, + shouldPass: true, + }, + // Bad case. + { + objectName: "test", + data: []byte("Hello, World"), + expectedRespStatus: http.StatusBadRequest, + shouldPass: false, + }, + } + + for i, testCase := range testCases { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + req, perr := newPostRequest("", bucketName, testCase.objectName, testCase.data, testCase.shouldPass) + if perr != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for PostPolicyHandler: %v", i+1, instanceType, perr) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + if rec.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) + } + } + +} + +// postPresignSignatureV4 - presigned signature for PostPolicy requests. +func postPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string { + // Get signining key. + signingkey := getSigningKey(secretAccessKey, t, location) + // Calculate signature. + signature := getSignature(signingkey, policyBase64) + return signature +} + +func newPostRequest(endPoint, bucketName, objectName string, objData []byte, shouldPass bool) (*http.Request, error) { + // Keep time. + t := time.Now().UTC() + // Expire the request five minutes from now. + expirationTime := t.Add(time.Minute * 5) + // Get the user credential. + credentials := serverConfig.GetCredential() + credStr := getCredential(credentials.AccessKeyID, serverConfig.GetRegion(), t) + // Create a new post policy. + policy := newPostPolicyBytes(credStr, bucketName, objectName, expirationTime) + // Only need the encoding. + encodedPolicy := base64.StdEncoding.EncodeToString(policy) + + formData := make(map[string]string) + if shouldPass { + // Presign with V4 signature based on the policy. + signature := postPresignSignatureV4(encodedPolicy, t, credentials.SecretAccessKey, serverConfig.GetRegion()) + + formData = map[string]string{ + "bucket": bucketName, + "key": objectName, + "x-amz-credential": credStr, + "policy": encodedPolicy, + "x-amz-signature": signature, + "x-amz-date": t.Format(iso8601DateFormat), + "x-amz-algorithm": "AWS4-HMAC-SHA256", + } + } + + // Create the multipart form. + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + + // Set the normal formData + for k, v := range formData { + w.WriteField(k, v) + } + // Set the File formData + writer, err := w.CreateFormFile("file", "s3verify/post/object") + if err != nil { + // return nil, err + return nil, err + } + writer.Write(objData) + // Close before creating the new request. + w.Close() + + // Set the body equal to the created policy. + reader := bytes.NewReader(buf.Bytes()) + + req, err := http.NewRequest("POST", makeTestTargetURL(endPoint, bucketName, objectName, nil), reader) + if err != nil { + return nil, err + } + + // Set form content-type. + req.Header.Set("Content-Type", w.FormDataContentType()) + return req, nil +} diff --git a/cmd/server_test.go b/cmd/server_test.go index 8dd7b29c6..158c28d8b 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -309,6 +309,7 @@ func (s *TestSuiteCommon) TestDeleteBucketNotEmpty(c *C) { } +// Test deletes multple objects and verifies server resonse. func (s *TestSuiteCommon) TestDeleteMultipleObjects(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() @@ -347,18 +348,11 @@ func (s *TestSuiteCommon) TestDeleteMultipleObjects(c *C) { ObjectName: objName, }) } - // Append a non-existent object for which the response should be marked - // as deleted. - delObjReq.Objects = append(delObjReq.Objects, ObjectIdentifier{ - ObjectName: fmt.Sprintf("%d/%s", 10, objectName), - }) - // Marshal delete request. deleteReqBytes, err := xml.Marshal(delObjReq) c.Assert(err, IsNil) - // object name was "prefix/myobject", an attempt to delelte "prefix" - // Should not delete "prefix/myobject" + // Delete list of objects. request, err = newTestSignedRequest("POST", getMultiDeleteObjectURL(s.endPoint, bucketName), int64(len(deleteReqBytes)), bytes.NewReader(deleteReqBytes), s.accessKey, s.secretKey) c.Assert(err, IsNil) @@ -372,11 +366,31 @@ func (s *TestSuiteCommon) TestDeleteMultipleObjects(c *C) { c.Assert(err, IsNil) err = xml.Unmarshal(delRespBytes, &deleteResp) c.Assert(err, IsNil) - for i := 0; i <= 10; i++ { + for i := 0; i < 10; i++ { // All the objects should be under deleted list (including non-existent object) c.Assert(deleteResp.DeletedObjects[i], DeepEquals, delObjReq.Objects[i]) } c.Assert(len(deleteResp.Errors), Equals, 0) + + // Attempt second time results should be same, NoSuchKey for objects not found + // shouldn't be set. + request, err = newTestSignedRequest("POST", getMultiDeleteObjectURL(s.endPoint, bucketName), + int64(len(deleteReqBytes)), bytes.NewReader(deleteReqBytes), s.accessKey, s.secretKey) + c.Assert(err, IsNil) + client = http.Client{} + response, err = client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusOK) + + deleteResp = DeleteObjectsResponse{} + delRespBytes, err = ioutil.ReadAll(response.Body) + c.Assert(err, IsNil) + err = xml.Unmarshal(delRespBytes, &deleteResp) + c.Assert(err, IsNil) + for i := 0; i < 10; i++ { + c.Assert(deleteResp.DeletedObjects[i], DeepEquals, delObjReq.Objects[i]) + } + c.Assert(len(deleteResp.Errors), Equals, 0) } // Tests delete object responses and success. diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go index 36c2f05d4..b39a887dc 100644 --- a/cmd/test-utils_test.go +++ b/cmd/test-utils_test.go @@ -577,6 +577,11 @@ func signRequest(req *http.Request, accessKey, secretKey string) error { return nil } +// getCredential generate a credential string. +func getCredential(accessKeyID, location string, t time.Time) string { + return accessKeyID + "/" + getScope(t, location) +} + // Returns new HTTP request object. func newTestRequest(method, urlStr string, contentLength int64, body io.ReadSeeker) (*http.Request, error) { if method == "" { @@ -1046,14 +1051,26 @@ func getAbortMultipartUploadURL(endPoint, bucketName, objectName, uploadID strin return makeTestTargetURL(endPoint, bucketName, objectName, queryValue) } -// return URL for a new multipart upload. +// return URL for a listing pending multipart uploads. func getListMultipartURL(endPoint, bucketName string) string { queryValue := url.Values{} queryValue.Set("uploads", "") return makeTestTargetURL(endPoint, bucketName, "", queryValue) } -// return URL for a new multipart upload. +// return URL for listing pending multipart uploads with parameters. +func getListMultipartUploadsURLWithParams(endPoint, bucketName, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads string) string { + queryValue := url.Values{} + queryValue.Set("uploads", "") + queryValue.Set("prefix", prefix) + queryValue.Set("delimiter", delimiter) + queryValue.Set("key-marker", keyMarker) + queryValue.Set("upload-id-marker", uploadIDMarker) + queryValue.Set("max-uploads", maxUploads) + return makeTestTargetURL(endPoint, bucketName, "", queryValue) +} + +// return URL for a listing parts on a given upload id. func getListMultipartURLWithParams(endPoint, bucketName, objectName, uploadID, maxParts string) string { queryValues := url.Values{} queryValues.Set("uploadId", uploadID) @@ -1289,8 +1306,12 @@ func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Hand // Register GetBucketLocation handler. case "GetBucketLocation": bucket.Methods("GET").HandlerFunc(api.GetBucketLocationHandler).Queries("location", "") + // Register HeadBucket handler. case "HeadBucket": bucket.Methods("HEAD").HandlerFunc(api.HeadBucketHandler) + // Register ListMultipartUploads handler. + case "ListMultipartUploads": + bucket.Methods("GET").HandlerFunc(api.ListMultipartUploadsHandler).Queries("uploads", "") // Register all api endpoints by default. default: registerAPIRouter(muxRouter, api)