tests: Add tests for ListMultipartUploads, DeleteMultipleObjects. (#2649)

Additionally adds PostPolicyHandler tests.
This commit is contained in:
Harshavardhana 2016-09-13 19:00:01 -07:00
parent 54a9f59a13
commit ee7e70c992
5 changed files with 353 additions and 16 deletions

View File

@ -93,10 +93,10 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType string, t TestEr
for i, testCase := range testCases {
// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
rec := httptest.NewRecorder()
// construct HTTP request for PUT bucket policy endpoint.
// construct HTTP request for Get bucket location.
req, err := newTestSignedRequest("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
if err != nil {
t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
t.Fatalf("Test %d: %s: Failed to create HTTP request for GetBucketLocationHandler: <ERROR> %v", i+1, instanceType, err)
}
// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
// Call the ServeHTTP to execute the handler.
@ -186,10 +186,10 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType string, t TestErrHandle
for i, testCase := range testCases {
// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
rec := httptest.NewRecorder()
// construct HTTP request for PUT bucket policy endpoint.
// construct HTTP request for HEAD bucket.
req, err := newTestSignedRequest("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
if err != nil {
t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
t.Fatalf("Test %d: %s: Failed to create HTTP request for HeadBucketHandler: <ERROR> %v", i+1, instanceType, err)
}
// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
// Call the ServeHTTP to execute the handler.
@ -199,3 +199,104 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType string, t TestErrHandle
}
}
}
// Wrapper for calling TestListMultipartUploadsHandler tests for both XL multiple disks and single node setup.
func TestListMultipartUploadsHandler(t *testing.T) {
ExecObjectLayerTest(t, testListMultipartUploads)
}
// testListMultipartUploadsHandler - Tests validate listing of multipart uploads.
func testListMultipartUploadsHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
initBucketPolicies(obj)
// get random bucket name.
bucketName := getRandomBucketName()
// Register the API end points with XL/FS object layer.
apiRouter := initTestAPIEndPoints(obj, []string{"ListMultipartUploads"})
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
rootPath, err := newTestConfig("us-east-1")
if err != nil {
t.Fatalf("Init Test config failed")
}
// remove the root folder after the test ends.
defer removeAll(rootPath)
credentials := serverConfig.GetCredential()
// bucketnames[0].
// objectNames[0].
// uploadIds [0].
// Create bucket before initiating NewMultipartUpload.
err = obj.MakeBucket(bucketName)
if err != nil {
// Failed to create newbucket, abort.
t.Fatalf("%s : %s", instanceType, err.Error())
}
// Collection of non-exhaustive ListMultipartUploads test cases, valid errors
// and success responses.
testCases := []struct {
// Inputs to ListMultipartUploads.
bucket string
prefix string
keyMarker string
uploadIDMarker string
delimiter string
maxUploads string
expectedRespStatus int
shouldPass bool
}{
// 1 - invalid bucket name.
{".test", "", "", "", "", "0", http.StatusBadRequest, false},
// 2 - bucket not found.
{"volatile-bucket-1", "", "", "", "", "0", http.StatusNotFound, false},
// 3 - invalid delimiter.
{bucketName, "", "", "", "-", "0", http.StatusBadRequest, false},
// 4 - invalid prefix and marker combination.
{bucketName, "asia", "europe-object", "", "", "0", http.StatusNotImplemented, false},
// 5 - invalid upload id and marker combination.
{bucketName, "asia", "asia/europe/", "abc", "", "0", http.StatusBadRequest, false},
// 6 - invalid max upload id.
{bucketName, "", "", "", "", "-1", http.StatusBadRequest, false},
// 7 - good case delimiter.
{bucketName, "", "", "", "/", "100", http.StatusOK, true},
// 8 - good case without delimiter.
{bucketName, "", "", "", "", "100", http.StatusOK, true},
}
for i, testCase := range testCases {
// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
rec := httptest.NewRecorder()
// construct HTTP request for List multipart uploads endpoint.
u := getListMultipartUploadsURLWithParams("", testCase.bucket, testCase.prefix, testCase.keyMarker, testCase.uploadIDMarker, testCase.delimiter, testCase.maxUploads)
req, gerr := newTestSignedRequest("GET", u, 0, nil, credentials.AccessKeyID, credentials.SecretAccessKey)
if gerr != nil {
t.Fatalf("Test %d: %s: Failed to create HTTP request for ListMultipartUploadsHandler: <ERROR> %v", i+1, instanceType, gerr)
}
// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
// Call the ServeHTTP to execute the handler.
apiRouter.ServeHTTP(rec, req)
if rec.Code != testCase.expectedRespStatus {
t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
}
}
// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
rec := httptest.NewRecorder()
// construct HTTP request for List multipart uploads endpoint.
u := getListMultipartUploadsURLWithParams("", bucketName, "", "", "", "", "")
req, err := newTestSignedRequest("GET", u, 0, nil, "", "") // Generate an anonymous request.
if err != nil {
t.Fatalf("Test %s: Failed to create HTTP request for ListMultipartUploadsHandler: <ERROR> %v", instanceType, err)
}
// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
// Call the ServeHTTP to execute the handler.
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusForbidden {
t.Errorf("Test %s: Expected the response status to be `http.StatusForbidden`, but instead found `%d`", instanceType, rec.Code)
}
}

View File

@ -93,7 +93,7 @@ func TestFSShutdown(t *testing.T) {
for i := 1; i <= 5; i++ {
naughty := newNaughtyDisk(fsStorage, map[int]error{i: errFaultyDisk}, nil)
fs.storage = naughty
if err := fs.Shutdown(); err != errFaultyDisk {
if err := fs.Shutdown(); errorCause(err) != errFaultyDisk {
t.Fatal(i, ", Got unexpected fs shutdown error: ", err)
}
}

201
cmd/post-policy_test.go Normal file
View File

@ -0,0 +1,201 @@
/*
* Minio Cloud Storage, (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import (
"bytes"
"encoding/base64"
"fmt"
"mime/multipart"
"net/http"
"net/http/httptest"
"testing"
"time"
)
const (
expirationDateFormat = "2006-01-02T15:04:05.999Z"
iso8601DateFormat = "20060102T150405Z"
)
// newPostPolicyBytes - creates a bare bones postpolicy string with key and bucket matches.
func newPostPolicyBytes(credential, bucketName, objectKey string, expiration time.Time) []byte {
t := time.Now().UTC()
// Add the expiration date.
expirationStr := fmt.Sprintf(`"expiration": "%s"`, expiration.Format(expirationDateFormat))
// Add the bucket condition, only accept buckets equal to the one passed.
bucketConditionStr := fmt.Sprintf(`["eq", "$bucket", "%s"]`, bucketName)
// Add the key condition, only accept keys equal to the one passed.
keyConditionStr := fmt.Sprintf(`["eq", "$key", "%s"]`, objectKey)
// Add the algorithm condition, only accept AWS SignV4 Sha256.
algorithmConditionStr := `["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"]`
// Add the date condition, only accept the current date.
dateConditionStr := fmt.Sprintf(`["eq", "$x-amz-date", "%s"]`, t.Format(iso8601DateFormat))
// Add the credential string, only accept the credential passed.
credentialConditionStr := fmt.Sprintf(`["eq", "$x-amz-credential", "%s"]`, credential)
// Combine all conditions into one string.
conditionStr := fmt.Sprintf(`"conditions":[%s, %s, %s, %s, %s]`, bucketConditionStr, keyConditionStr, algorithmConditionStr, dateConditionStr, credentialConditionStr)
retStr := "{"
retStr = retStr + expirationStr + ","
retStr = retStr + conditionStr
retStr = retStr + "}"
return []byte(retStr)
}
// Wrapper for calling TestPostPolicyHandlerHandler tests for both XL multiple disks and single node setup.
func TestPostPolicyHandler(t *testing.T) {
ExecObjectLayerTest(t, testPostPolicyHandler)
}
// testPostPolicyHandler - Tests validate post policy handler uploading objects.
func testPostPolicyHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
// get random bucket name.
bucketName := getRandomBucketName()
// Register the API end points with XL/FS object layer.
apiRouter := initTestAPIEndPoints(obj, []string{"PostPolicy"})
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
rootPath, err := newTestConfig("us-east-1")
if err != nil {
t.Fatalf("Init Test config failed")
}
// remove the root folder after the test ends.
defer removeAll(rootPath)
// bucketnames[0].
// objectNames[0].
// uploadIds [0].
// Create bucket before initiating NewMultipartUpload.
err = obj.MakeBucket(bucketName)
if err != nil {
// Failed to create newbucket, abort.
t.Fatalf("%s : %s", instanceType, err.Error())
}
// Collection of non-exhaustive ListMultipartUploads test cases, valid errors
// and success responses.
testCases := []struct {
objectName string
data []byte
expectedRespStatus int
shouldPass bool
}{
// Success case.
{
objectName: "test",
data: []byte("Hello, World"),
expectedRespStatus: http.StatusNoContent,
shouldPass: true,
},
// Bad case.
{
objectName: "test",
data: []byte("Hello, World"),
expectedRespStatus: http.StatusBadRequest,
shouldPass: false,
},
}
for i, testCase := range testCases {
// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
rec := httptest.NewRecorder()
req, perr := newPostRequest("", bucketName, testCase.objectName, testCase.data, testCase.shouldPass)
if perr != nil {
t.Fatalf("Test %d: %s: Failed to create HTTP request for PostPolicyHandler: <ERROR> %v", i+1, instanceType, perr)
}
// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
// Call the ServeHTTP to execute the handler.
apiRouter.ServeHTTP(rec, req)
if rec.Code != testCase.expectedRespStatus {
t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
}
}
}
// postPresignSignatureV4 - presigned signature for PostPolicy requests.
func postPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string {
// Get signining key.
signingkey := getSigningKey(secretAccessKey, t, location)
// Calculate signature.
signature := getSignature(signingkey, policyBase64)
return signature
}
func newPostRequest(endPoint, bucketName, objectName string, objData []byte, shouldPass bool) (*http.Request, error) {
// Keep time.
t := time.Now().UTC()
// Expire the request five minutes from now.
expirationTime := t.Add(time.Minute * 5)
// Get the user credential.
credentials := serverConfig.GetCredential()
credStr := getCredential(credentials.AccessKeyID, serverConfig.GetRegion(), t)
// Create a new post policy.
policy := newPostPolicyBytes(credStr, bucketName, objectName, expirationTime)
// Only need the encoding.
encodedPolicy := base64.StdEncoding.EncodeToString(policy)
formData := make(map[string]string)
if shouldPass {
// Presign with V4 signature based on the policy.
signature := postPresignSignatureV4(encodedPolicy, t, credentials.SecretAccessKey, serverConfig.GetRegion())
formData = map[string]string{
"bucket": bucketName,
"key": objectName,
"x-amz-credential": credStr,
"policy": encodedPolicy,
"x-amz-signature": signature,
"x-amz-date": t.Format(iso8601DateFormat),
"x-amz-algorithm": "AWS4-HMAC-SHA256",
}
}
// Create the multipart form.
var buf bytes.Buffer
w := multipart.NewWriter(&buf)
// Set the normal formData
for k, v := range formData {
w.WriteField(k, v)
}
// Set the File formData
writer, err := w.CreateFormFile("file", "s3verify/post/object")
if err != nil {
// return nil, err
return nil, err
}
writer.Write(objData)
// Close before creating the new request.
w.Close()
// Set the body equal to the created policy.
reader := bytes.NewReader(buf.Bytes())
req, err := http.NewRequest("POST", makeTestTargetURL(endPoint, bucketName, objectName, nil), reader)
if err != nil {
return nil, err
}
// Set form content-type.
req.Header.Set("Content-Type", w.FormDataContentType())
return req, nil
}

View File

@ -309,6 +309,7 @@ func (s *TestSuiteCommon) TestDeleteBucketNotEmpty(c *C) {
}
// Test deletes multple objects and verifies server resonse.
func (s *TestSuiteCommon) TestDeleteMultipleObjects(c *C) {
// generate a random bucket name.
bucketName := getRandomBucketName()
@ -347,18 +348,11 @@ func (s *TestSuiteCommon) TestDeleteMultipleObjects(c *C) {
ObjectName: objName,
})
}
// Append a non-existent object for which the response should be marked
// as deleted.
delObjReq.Objects = append(delObjReq.Objects, ObjectIdentifier{
ObjectName: fmt.Sprintf("%d/%s", 10, objectName),
})
// Marshal delete request.
deleteReqBytes, err := xml.Marshal(delObjReq)
c.Assert(err, IsNil)
// object name was "prefix/myobject", an attempt to delelte "prefix"
// Should not delete "prefix/myobject"
// Delete list of objects.
request, err = newTestSignedRequest("POST", getMultiDeleteObjectURL(s.endPoint, bucketName),
int64(len(deleteReqBytes)), bytes.NewReader(deleteReqBytes), s.accessKey, s.secretKey)
c.Assert(err, IsNil)
@ -372,11 +366,31 @@ func (s *TestSuiteCommon) TestDeleteMultipleObjects(c *C) {
c.Assert(err, IsNil)
err = xml.Unmarshal(delRespBytes, &deleteResp)
c.Assert(err, IsNil)
for i := 0; i <= 10; i++ {
for i := 0; i < 10; i++ {
// All the objects should be under deleted list (including non-existent object)
c.Assert(deleteResp.DeletedObjects[i], DeepEquals, delObjReq.Objects[i])
}
c.Assert(len(deleteResp.Errors), Equals, 0)
// Attempt second time results should be same, NoSuchKey for objects not found
// shouldn't be set.
request, err = newTestSignedRequest("POST", getMultiDeleteObjectURL(s.endPoint, bucketName),
int64(len(deleteReqBytes)), bytes.NewReader(deleteReqBytes), s.accessKey, s.secretKey)
c.Assert(err, IsNil)
client = http.Client{}
response, err = client.Do(request)
c.Assert(err, IsNil)
c.Assert(response.StatusCode, Equals, http.StatusOK)
deleteResp = DeleteObjectsResponse{}
delRespBytes, err = ioutil.ReadAll(response.Body)
c.Assert(err, IsNil)
err = xml.Unmarshal(delRespBytes, &deleteResp)
c.Assert(err, IsNil)
for i := 0; i < 10; i++ {
c.Assert(deleteResp.DeletedObjects[i], DeepEquals, delObjReq.Objects[i])
}
c.Assert(len(deleteResp.Errors), Equals, 0)
}
// Tests delete object responses and success.

View File

@ -577,6 +577,11 @@ func signRequest(req *http.Request, accessKey, secretKey string) error {
return nil
}
// getCredential generate a credential string.
func getCredential(accessKeyID, location string, t time.Time) string {
return accessKeyID + "/" + getScope(t, location)
}
// Returns new HTTP request object.
func newTestRequest(method, urlStr string, contentLength int64, body io.ReadSeeker) (*http.Request, error) {
if method == "" {
@ -1046,14 +1051,26 @@ func getAbortMultipartUploadURL(endPoint, bucketName, objectName, uploadID strin
return makeTestTargetURL(endPoint, bucketName, objectName, queryValue)
}
// return URL for a new multipart upload.
// return URL for a listing pending multipart uploads.
func getListMultipartURL(endPoint, bucketName string) string {
queryValue := url.Values{}
queryValue.Set("uploads", "")
return makeTestTargetURL(endPoint, bucketName, "", queryValue)
}
// return URL for a new multipart upload.
// return URL for listing pending multipart uploads with parameters.
func getListMultipartUploadsURLWithParams(endPoint, bucketName, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads string) string {
queryValue := url.Values{}
queryValue.Set("uploads", "")
queryValue.Set("prefix", prefix)
queryValue.Set("delimiter", delimiter)
queryValue.Set("key-marker", keyMarker)
queryValue.Set("upload-id-marker", uploadIDMarker)
queryValue.Set("max-uploads", maxUploads)
return makeTestTargetURL(endPoint, bucketName, "", queryValue)
}
// return URL for a listing parts on a given upload id.
func getListMultipartURLWithParams(endPoint, bucketName, objectName, uploadID, maxParts string) string {
queryValues := url.Values{}
queryValues.Set("uploadId", uploadID)
@ -1289,8 +1306,12 @@ func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Hand
// Register GetBucketLocation handler.
case "GetBucketLocation":
bucket.Methods("GET").HandlerFunc(api.GetBucketLocationHandler).Queries("location", "")
// Register HeadBucket handler.
case "HeadBucket":
bucket.Methods("HEAD").HandlerFunc(api.HeadBucketHandler)
// Register ListMultipartUploads handler.
case "ListMultipartUploads":
bucket.Methods("GET").HandlerFunc(api.ListMultipartUploadsHandler).Queries("uploads", "")
// Register all api endpoints by default.
default:
registerAPIRouter(muxRouter, api)