From 1ec86dac2cd35f8eba63fa70254234bdb5d3920c Mon Sep 17 00:00:00 2001 From: Krishna Srinivas Date: Tue, 5 Jul 2016 01:48:41 +0530 Subject: [PATCH] server-tests: unify XL and FS tests into common code. server_test.go contains common test code. server_xl_test.go contains XL tests specific to XL. (#2089) --- server_test.go | 92 +- server_xl_test.go | 2141 +-------------------------------------------- 2 files changed, 78 insertions(+), 2155 deletions(-) diff --git a/server_test.go b/server_test.go index 7656f9825..49c32efc7 100644 --- a/server_test.go +++ b/server_test.go @@ -33,32 +33,36 @@ import ( . "gopkg.in/check.v1" ) -// API suite container. -type TestSuiteFS struct { +// API suite container common to both FS and XL. +type TestSuiteCommon struct { + serverType string testServer TestServer endPoint string accessKey string secretKey string } -// Initializing the test suite. -var _ = Suite(&TestSuiteFS{}) +// Init and run test on FS backend. +var _ = Suite(&TestSuiteCommon{serverType: "FS"}) + +// Init and run test on XL backend. +var _ = Suite(&TestSuiteCommon{serverType: "XL"}) // Setting up the test suite. // Starting the Test server with temporary FS backend. -func (s *TestSuiteFS) SetUpSuite(c *C) { - s.testServer = StartTestServer(c, "FS") +func (s *TestSuiteCommon) SetUpSuite(c *C) { + s.testServer = StartTestServer(c, s.serverType) s.endPoint = s.testServer.Server.URL s.accessKey = s.testServer.AccessKey s.secretKey = s.testServer.SecretKey } // Called implicitly by "gopkg.in/check.v1" after all tests are run. -func (s *TestSuiteFS) TearDownSuite(c *C) { +func (s *TestSuiteCommon) TearDownSuite(c *C) { s.testServer.Stop() } -func (s *TestSuiteFS) TestAuth(c *C) { +func (s *TestSuiteCommon) TestAuth(c *C) { secretID, err := genSecretAccessKey() c.Assert(err, IsNil) @@ -71,7 +75,7 @@ func (s *TestSuiteFS) TestAuth(c *C) { // TestBucketPolicy - Inserts the bucket policy and verifies it by fetching the policy back. // Deletes the policy and verifies the deletion by fetching it back. -func (s *TestSuiteFS) TestBucketPolicy(c *C) { +func (s *TestSuiteCommon) TestBucketPolicy(c *C) { // Sample bucket policy. bucketPolicyBuf := `{ "Version": "2012-10-17", @@ -172,7 +176,7 @@ func (s *TestSuiteFS) TestBucketPolicy(c *C) { } // TestDeleteBucket - validates DELETE bucket operation. -func (s *TestSuiteFS) TestDeleteBucket(c *C) { +func (s *TestSuiteCommon) TestDeleteBucket(c *C) { bucketName := getRandomBucketName() // HTTP request to create the bucket. @@ -199,7 +203,7 @@ func (s *TestSuiteFS) TestDeleteBucket(c *C) { } // TestDeleteBucketNotEmpty - Validates the operation during an attempt to delete a non-empty bucket. -func (s *TestSuiteFS) TestDeleteBucketNotEmpty(c *C) { +func (s *TestSuiteCommon) TestDeleteBucketNotEmpty(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() @@ -244,7 +248,7 @@ func (s *TestSuiteFS) TestDeleteBucketNotEmpty(c *C) { } // Tests delete object responses and success. -func (s *TestSuiteFS) TestDeleteObject(c *C) { +func (s *TestSuiteCommon) TestDeleteObject(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // HTTP request to create the bucket. @@ -319,7 +323,7 @@ func (s *TestSuiteFS) TestDeleteObject(c *C) { } // TestNonExistentBucket - Asserts response for HEAD on non-existent bucket. -func (s *TestSuiteFS) TestNonExistentBucket(c *C) { +func (s *TestSuiteCommon) TestNonExistentBucket(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // create request to HEAD on the bucket. @@ -337,7 +341,7 @@ func (s *TestSuiteFS) TestNonExistentBucket(c *C) { } // TestEmptyObject - Asserts the response for operation on a 0 byte object. -func (s *TestSuiteFS) TestEmptyObject(c *C) { +func (s *TestSuiteCommon) TestEmptyObject(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // HTTP request to create the bucket. @@ -385,7 +389,7 @@ func (s *TestSuiteFS) TestEmptyObject(c *C) { c.Assert(true, Equals, bytes.Equal(responseBody, buffer.Bytes())) } -func (s *TestSuiteFS) TestBucket(c *C) { +func (s *TestSuiteCommon) TestBucket(c *C) { request, err := newTestRequest("PUT", s.testServer.Server.URL+"/bucket", 0, nil, s.testServer.AccessKey, s.testServer.SecretKey) c.Assert(err, IsNil) @@ -406,7 +410,7 @@ func (s *TestSuiteFS) TestBucket(c *C) { } // TestGetObject - Tests fetching of a small object after its insertion into the bucket. -func (s *TestSuiteFS) TestObjectGet(c *C) { +func (s *TestSuiteCommon) TestObjectGet(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() buffer := bytes.NewReader([]byte("hello world")) @@ -455,7 +459,7 @@ func (s *TestSuiteFS) TestObjectGet(c *C) { } // TestMultipleObjects - Validates upload and fetching of multiple object into the bucket. -func (s *TestSuiteFS) TestMultipleObjects(c *C) { +func (s *TestSuiteCommon) TestMultipleObjects(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // HTTP request to create the bucket. @@ -577,7 +581,7 @@ func (s *TestSuiteFS) TestMultipleObjects(c *C) { } // TestNotImplemented - Validates response for obtaining policy on an non-existent bucket and object. -func (s *TestSuiteFS) TestNotImplemented(c *C) { +func (s *TestSuiteCommon) TestNotImplemented(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() request, err := newTestRequest("GET", s.endPoint+"/"+bucketName+"/object?policy", @@ -591,7 +595,7 @@ func (s *TestSuiteFS) TestNotImplemented(c *C) { } // TestHeader - Validates the error response for an attempt to fetch non-existent object. -func (s *TestSuiteFS) TestHeader(c *C) { +func (s *TestSuiteCommon) TestHeader(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // obtain HTTP request to fetch an object from non-existent bucket/object. @@ -606,7 +610,7 @@ func (s *TestSuiteFS) TestHeader(c *C) { verifyError(c, response, "NoSuchBucket", "The specified bucket does not exist.", http.StatusNotFound) } -func (s *TestSuiteFS) TestPutBucket(c *C) { +func (s *TestSuiteCommon) TestPutBucket(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // Block 1: Testing for racey access @@ -650,7 +654,7 @@ func (s *TestSuiteFS) TestPutBucket(c *C) { // 2. Insert Object. // 3. Use "X-Amz-Copy-Source" header to copy the previously inserted object. // 4. Validate the content of copied object. -func (s *TestSuiteFS) TestCopyObject(c *C) { +func (s *TestSuiteCommon) TestCopyObject(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // HTTP request to create the bucket. @@ -708,7 +712,7 @@ func (s *TestSuiteFS) TestCopyObject(c *C) { } // TestPutObject - Tests successful put object request. -func (s *TestSuiteFS) TestPutObject(c *C) { +func (s *TestSuiteCommon) TestPutObject(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // HTTP request to create the bucket. @@ -755,7 +759,7 @@ func (s *TestSuiteFS) TestPutObject(c *C) { // TestListBuckets - Make request for listing of all buckets. // XML response is parsed. // Its success verifies the format of the response. -func (s *TestSuiteFS) TestListBuckets(c *C) { +func (s *TestSuiteCommon) TestListBuckets(c *C) { // create HTTP request for listing buckets. request, err := newTestRequest("GET", getListBucketURL(s.endPoint), 0, nil, s.accessKey, s.secretKey) @@ -777,7 +781,7 @@ func (s *TestSuiteFS) TestListBuckets(c *C) { // TestNotBeAbleToCreateObjectInNonexistentBucket - Validates the error response // on an attempt to upload an object into a non-existent bucket. -func (s *TestSuiteFS) TestPutObjectLongName(c *C) { +func (s *TestSuiteCommon) TestPutObjectLongName(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // HTTP request to create the bucket. @@ -815,7 +819,7 @@ func (s *TestSuiteFS) TestPutObjectLongName(c *C) { // TestNotBeAbleToCreateObjectInNonexistentBucket - Validates the error response // on an attempt to upload an object into a non-existent bucket. -func (s *TestSuiteFS) TestNotBeAbleToCreateObjectInNonexistentBucket(c *C) { +func (s *TestSuiteCommon) TestNotBeAbleToCreateObjectInNonexistentBucket(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // content of the object to be uploaded. @@ -841,7 +845,7 @@ func (s *TestSuiteFS) TestNotBeAbleToCreateObjectInNonexistentBucket(c *C) { // and If-Unmodified-Since headers set are validated. // If-Modified-Since - Return the object only if it has been modified since the specified time, else return a 304 (not modified). // If-Unmodified-Since - Return the object only if it has not been modified since the specified time, else return a 412 (precondition failed). -func (s *TestSuiteFS) TestHeadOnObjectLastModified(c *C) { +func (s *TestSuiteCommon) TestHeadOnObjectLastModified(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // HTTP request to create the bucket. @@ -911,7 +915,7 @@ func (s *TestSuiteFS) TestHeadOnObjectLastModified(c *C) { // TestHeadOnBucket - Validates response for HEAD on the bucket. // HEAD request on the bucket validates the existence of the bucket. -func (s *TestSuiteFS) TestHeadOnBucket(c *C) { +func (s *TestSuiteCommon) TestHeadOnBucket(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // HTTP request to create the bucket. @@ -937,7 +941,7 @@ func (s *TestSuiteFS) TestHeadOnBucket(c *C) { // TestContentTypePersists - Object upload with different Content-type is first done. // And then a HEAD and GET request on these objects are done to validate if the same Content-Type set during upload persists. -func (s *TestSuiteFS) TestContentTypePersists(c *C) { +func (s *TestSuiteCommon) TestContentTypePersists(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // HTTP request to create the bucket. @@ -1034,7 +1038,7 @@ func (s *TestSuiteFS) TestContentTypePersists(c *C) { // TestPartialContent - Validating for GetObject with partial content request. // By setting the Range header, A request to send specific bytes range of data from an // already uploaded object can be done. -func (s *TestSuiteFS) TestPartialContent(c *C) { +func (s *TestSuiteCommon) TestPartialContent(c *C) { request, err := newTestRequest("PUT", s.testServer.Server.URL+"/partial-content", 0, nil, s.testServer.AccessKey, s.testServer.SecretKey) c.Assert(err, IsNil) @@ -1072,7 +1076,7 @@ func (s *TestSuiteFS) TestPartialContent(c *C) { // TestListObjectsHandlerErrors - Setting invalid parameters to List Objects // and then asserting the error response with the expected one. -func (s *TestSuiteFS) TestListObjectsHandlerErrors(c *C) { +func (s *TestSuiteCommon) TestListObjectsHandlerErrors(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // HTTP request to create the bucket. @@ -1101,7 +1105,7 @@ func (s *TestSuiteFS) TestListObjectsHandlerErrors(c *C) { // TestPutBucketErrors - request for non valid bucket operation // and validate it with expected error result. -func (s *TestSuiteFS) TestPutBucketErrors(c *C) { +func (s *TestSuiteCommon) TestPutBucketErrors(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // generating a HTTP request to create bucket. @@ -1147,7 +1151,7 @@ func (s *TestSuiteFS) TestPutBucketErrors(c *C) { verifyError(c, response, "NotImplemented", "A header you provided implies functionality that is not implemented.", http.StatusNotImplemented) } -func (s *TestSuiteFS) TestGetObjectLarge10MiB(c *C) { +func (s *TestSuiteCommon) TestGetObjectLarge10MiB(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // form HTTP reqest to create the bucket. @@ -1211,7 +1215,7 @@ func (s *TestSuiteFS) TestGetObjectLarge10MiB(c *C) { } // TestGetObjectLarge11MiB - Tests validate fetching of an object of size 11MB. -func (s *TestSuiteFS) TestGetObjectLarge11MiB(c *C) { +func (s *TestSuiteCommon) TestGetObjectLarge11MiB(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // HTTP request to create the bucket. @@ -1280,7 +1284,7 @@ func (s *TestSuiteFS) TestGetObjectLarge11MiB(c *C) { // TestGetPartialObjectMisAligned - tests get object partially mis-aligned. // create a large buffer of mis-aligned data and upload it. // then make partial range requests to while fetching it back and assert the response content. -func (s *TestSuiteFS) TestGetPartialObjectMisAligned(c *C) { +func (s *TestSuiteCommon) TestGetPartialObjectMisAligned(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // HTTP request to create the bucket. @@ -1368,7 +1372,7 @@ func (s *TestSuiteFS) TestGetPartialObjectMisAligned(c *C) { } // TestGetPartialObjectLarge11MiB - Test validates partial content request for a 11MiB object. -func (s *TestSuiteFS) TestGetPartialObjectLarge11MiB(c *C) { +func (s *TestSuiteCommon) TestGetPartialObjectLarge11MiB(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // HTTP request to create the bucket. @@ -1436,7 +1440,7 @@ func (s *TestSuiteFS) TestGetPartialObjectLarge11MiB(c *C) { } // TestGetPartialObjectLarge11MiB - Test validates partial content request for a 10MiB object. -func (s *TestSuiteFS) TestGetPartialObjectLarge10MiB(c *C) { +func (s *TestSuiteCommon) TestGetPartialObjectLarge10MiB(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // HTTP request to create the bucket. @@ -1506,7 +1510,7 @@ func (s *TestSuiteFS) TestGetPartialObjectLarge10MiB(c *C) { } // TestGetObjectErrors - Tests validate error response for invalid object operations. -func (s *TestSuiteFS) TestGetObjectErrors(c *C) { +func (s *TestSuiteCommon) TestGetObjectErrors(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() @@ -1547,7 +1551,7 @@ func (s *TestSuiteFS) TestGetObjectErrors(c *C) { } // TestGetObjectRangeErrors - Validate error response when object is fetched with incorrect byte range value. -func (s *TestSuiteFS) TestGetObjectRangeErrors(c *C) { +func (s *TestSuiteCommon) TestGetObjectRangeErrors(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // HTTP request to create the bucket. @@ -1593,7 +1597,7 @@ func (s *TestSuiteFS) TestGetObjectRangeErrors(c *C) { } // TestObjectMultipartAbort - Test validates abortion of a multipart upload after uploading 2 parts. -func (s *TestSuiteFS) TestObjectMultipartAbort(c *C) { +func (s *TestSuiteCommon) TestObjectMultipartAbort(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // HTTP request to create the bucket. @@ -1661,7 +1665,7 @@ func (s *TestSuiteFS) TestObjectMultipartAbort(c *C) { } // TestBucketMultipartList - Initiates a NewMultipart upload, uploads parts and validates listing of the parts. -func (s *TestSuiteFS) TestBucketMultipartList(c *C) { +func (s *TestSuiteCommon) TestBucketMultipartList(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // HTTP request to create the bucket. @@ -1774,7 +1778,7 @@ func (s *TestSuiteFS) TestBucketMultipartList(c *C) { } // TestValidateObjectMultipartUploadID - Test Initiates a new multipart upload and validates the uploadID. -func (s *TestSuiteFS) TestValidateObjectMultipartUploadID(c *C) { +func (s *TestSuiteCommon) TestValidateObjectMultipartUploadID(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // HTTP request to create the bucket. @@ -1810,7 +1814,7 @@ func (s *TestSuiteFS) TestValidateObjectMultipartUploadID(c *C) { // TestObjectMultipartListError - Initiates a NewMultipart upload, uploads parts and validates // error response for an incorrect max-parts parameter . -func (s *TestSuiteFS) TestObjectMultipartListError(c *C) { +func (s *TestSuiteCommon) TestObjectMultipartListError(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // HTTP request to create the bucket. @@ -1879,7 +1883,7 @@ func (s *TestSuiteFS) TestObjectMultipartListError(c *C) { // TestObjectValidMD5 - First uploads an object with a valid Content-Md5 header and verifies the status, // then upload an object in a wrong Content-Md5 and validate the error response. -func (s *TestSuiteFS) TestObjectValidMD5(c *C) { +func (s *TestSuiteCommon) TestObjectValidMD5(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // HTTP request to create the bucket. @@ -1931,7 +1935,7 @@ func (s *TestSuiteFS) TestObjectValidMD5(c *C) { // TestObjectMultipart - Initiates a NewMultipart upload, uploads 2 parts, // completes the multipart upload and validates the status of the operation. -func (s *TestSuiteFS) TestObjectMultipart(c *C) { +func (s *TestSuiteCommon) TestObjectMultipart(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // HTTP request to create the bucket. diff --git a/server_xl_test.go b/server_xl_test.go index cceb44fed..5350c704a 100644 --- a/server_xl_test.go +++ b/server_xl_test.go @@ -19,21 +19,14 @@ package main import ( "bytes" "crypto/md5" - "encoding/base64" "encoding/hex" - "encoding/xml" - "fmt" - "io" "io/ioutil" - "math/rand" "net/http" - "sync" - "time" . "gopkg.in/check.v1" ) -// API suite container. +// API suite container for XL specific tests. type TestSuiteXL struct { testServer TestServer endPoint string @@ -59,791 +52,6 @@ func (s *TestSuiteXL) TearDownSuite(c *C) { s.testServer.Stop() } -func (s *TestSuiteXL) TestAuth(c *C) { - secretID, err := genSecretAccessKey() - c.Assert(err, IsNil) - - accessID, err := genAccessKeyID() - c.Assert(err, IsNil) - - c.Assert(len(secretID), Equals, minioSecretID) - c.Assert(len(accessID), Equals, minioAccessID) -} - -// TestBucketPolicy - Inserts the bucket policy and verifies it by fetching the policy back. -// Deletes the policy and verifies the deletion by fetching it back. -func (s *TestSuiteXL) TestBucketPolicy(c *C) { - // Sample bucket policy. - bucketPolicyBuf := `{ - "Version": "2012-10-17", - "Statement": [ - { - "Action": [ - "s3:GetBucketLocation", - "s3:ListBucket" - ], - "Effect": "Allow", - "Principal": { - "AWS": [ - "*" - ] - }, - "Resource": [ - "arn:aws:s3:::%s" - ] - }, - { - "Action": [ - "s3:GetObject" - ], - "Effect": "Allow", - "Principal": { - "AWS": [ - "*" - ] - }, - "Resource": [ - "arn:aws:s3:::%s/this*" - ] - } - ] -}` - // generate a random bucket Name. - bucketName := getRandomBucketName() - // create the policy statement string with the randomly generated bucket name. - bucketPolicyStr := fmt.Sprintf(bucketPolicyBuf, bucketName, bucketName) - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the request. - response, err := client.Do(request) - c.Assert(err, IsNil) - // assert the http response status code. - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // Put a new bucket policy. - request, err = newTestRequest("PUT", getPutPolicyURL(s.endPoint, bucketName), - int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request to create bucket. - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusNoContent) - - // Fetch the uploaded policy. - request, err = newTestRequest("GET", getGetPolicyURL(s.endPoint, bucketName), 0, nil, - s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - bucketPolicyReadBuf, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - // Verify if downloaded policy matches with previousy uploaded. - c.Assert(bytes.Equal([]byte(bucketPolicyStr), bucketPolicyReadBuf), Equals, true) - - // Delete policy. - request, err = newTestRequest("DELETE", getDeletePolicyURL(s.endPoint, bucketName), 0, nil, - s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusNoContent) - - // Verify if the policy was indeed deleted. - request, err = newTestRequest("GET", getGetPolicyURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusNotFound) -} - -// TestDeleteBucket - validates DELETE bucket operation. -func (s *TestSuiteXL) TestDeleteBucket(c *C) { - bucketName := getRandomBucketName() - - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - // assert the response status code. - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // construct request to delete the bucket. - request, err = newTestRequest("DELETE", getDeleteBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - // Assert the response status code. - c.Assert(response.StatusCode, Equals, http.StatusNoContent) -} - -// TestDeleteBucketNotEmpty - Validates the operation during an attempt to delete a non-empty bucket. -func (s *TestSuiteXL) TestDeleteBucketNotEmpty(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the request. - response, err := client.Do(request) - c.Assert(err, IsNil) - // assert the response status code. - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // generate http request for an object upload. - // "test-object" is the object name. - objectName := "test-object" - request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the request to complete object upload. - response, err = client.Do(request) - c.Assert(err, IsNil) - // assert the status code of the response. - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // constructing http request to delete the bucket. - // making an attempt to delete an non-empty bucket. - // expected to fail. - request, err = newTestRequest("DELETE", getDeleteBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusConflict) -} - -// TestDeleteObject - uploads the object first. -func (s *TestSuiteXL) TestDeleteObject(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the request. - response, err := client.Do(request) - c.Assert(err, IsNil) - // assert the http response status code. - c.Assert(response.StatusCode, Equals, http.StatusOK) - - objectName := "prefix/myobject" - // obtain http request to upload object. - // object Name contains a prefix. - request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the http request. - response, err = client.Do(request) - c.Assert(err, IsNil) - // assert the status of http response. - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // object name was "prefix/myobject", an attempt to delelte "prefix" - // Should not delete "prefix/myobject" - request, err = newTestRequest("DELETE", getDeleteObjectURL(s.endPoint, bucketName, "prefix"), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusNoContent) - - // create http request to HEAD on the object. - // this helps to validate the existence of the bucket. - request, err = newTestRequest("HEAD", getHeadObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - // Assert the HTTP response status code. - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // create HTTP request to delete the object. - request, err = newTestRequest("DELETE", getDeleteObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - client = http.Client{} - // execute the http request. - response, err = client.Do(request) - c.Assert(err, IsNil) - // assert the http response status code. - c.Assert(response.StatusCode, Equals, http.StatusNoContent) - - // Delete of non-existent data should return success. - request, err = newTestRequest("DELETE", getDeleteObjectURL(s.endPoint, bucketName, "prefix/myobject1"), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - client = http.Client{} - // execute the http request. - response, err = client.Do(request) - c.Assert(err, IsNil) - // assert the http response status. - c.Assert(response.StatusCode, Equals, http.StatusNoContent) -} - -// TestNonExistentBucket - Asserts response for HEAD on non-existent bucket. -func (s *TestSuiteXL) TestNonExistentBucket(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // create request to HEAD on the bucket. - // HEAD on an bucket helps validate the existence of the bucket. - request, err := newTestRequest("HEAD", getHEADBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the http request. - response, err := client.Do(request) - c.Assert(err, IsNil) - // Assert the response. - c.Assert(response.StatusCode, Equals, http.StatusNotFound) -} - -// TestEmptyObject - Asserts the response for operation on a 0 byte object. -func (s *TestSuiteXL) TestEmptyObject(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the http request. - response, err := client.Do(request) - c.Assert(err, IsNil) - // assert the http response status code. - c.Assert(response.StatusCode, Equals, http.StatusOK) - - objectName := "test-object" - // construct http request for uploading the object. - request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the upload request. - response, err = client.Do(request) - c.Assert(err, IsNil) - // assert the http response. - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // make HTTP request to fetch the object. - request, err = newTestRequest("GET", getGetObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the http request to fetch object. - response, err = client.Do(request) - c.Assert(err, IsNil) - // assert the http response status code. - c.Assert(response.StatusCode, Equals, http.StatusOK) - - var buffer bytes.Buffer - // extract the body of the response. - responseBody, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - // assert the http response body content. - c.Assert(true, Equals, bytes.Equal(responseBody, buffer.Bytes())) -} - -// TestBucket - Asserts the response for HEAD on an existing bucket. -func (s *TestSuiteXL) TestBucket(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // generating HEAD http request on the bucket. - // this helps verify existence of the bucket. - request, err = newTestRequest("HEAD", getHEADBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - // execute the request. - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - // assert the response http status code. - c.Assert(response.StatusCode, Equals, http.StatusOK) -} - -// TestGetObject - Tests fetching of a small object after its insertion into the bucket. -func (s *TestSuiteXL) TestGetObject(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - buffer := bytes.NewReader([]byte("hello world")) - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the make bucket http request. - response, err := client.Do(request) - c.Assert(err, IsNil) - // assert the response http status code. - c.Assert(response.StatusCode, Equals, http.StatusOK) - - objectName := "testObject" - // create HTTP request to upload the object. - request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName), - int64(buffer.Len()), buffer, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request to upload the object. - response, err = client.Do(request) - c.Assert(err, IsNil) - // assert the HTTP response status code. - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // create HTTP request to fetch the object. - request, err = newTestRequest("GET", getGetObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the http request to fetch the object. - response, err = client.Do(request) - c.Assert(err, IsNil) - // assert the http response status code. - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // extract response body content. - responseBody, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - // assert the HTTP response body content with the expected content. - c.Assert(responseBody, DeepEquals, []byte("hello world")) - -} - -// TestMultipleObjects - Validates upload and fetching of multiple object into the bucket. -func (s *TestSuiteXL) TestMultipleObjects(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request to create the bucket. - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // constructing HTTP request to fetch a non-existent object. - // expected to fail, error response asserted for expected error values later. - objectName := "testObject" - request, err = newTestRequest("GET", getGetObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request. - response, err = client.Do(request) - c.Assert(err, IsNil) - // Asserting the error response with the expected values. - verifyError(c, response, "NoSuchKey", "The specified key does not exist.", http.StatusNotFound) - - objectName = "testObject1" - // content for the object to be uploaded. - buffer1 := bytes.NewReader([]byte("hello one")) - // create HTTP request for the object upload. - request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName), - int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request for object upload. - response, err = client.Do(request) - c.Assert(err, IsNil) - // assert the returned values. - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // create HTTP request to fetch the object which was uploaded above. - request, err = newTestRequest("GET", getGetObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request. - response, err = client.Do(request) - c.Assert(err, IsNil) - // assert whether 200 OK response status is obtained. - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // extract the response body. - responseBody, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - // assert the content body for the expected object data. - c.Assert(true, Equals, bytes.Equal(responseBody, []byte("hello one"))) - - // data for new object to be uploaded. - buffer2 := bytes.NewReader([]byte("hello two")) - objectName = "testObject2" - request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName), - int64(buffer2.Len()), buffer2, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request for object upload. - response, err = client.Do(request) - c.Assert(err, IsNil) - // assert the response status code for expected value 200 OK. - c.Assert(response.StatusCode, Equals, http.StatusOK) - // fetch the object which was uploaded above. - request, err = newTestRequest("GET", getGetObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request to fetch the object. - response, err = client.Do(request) - c.Assert(err, IsNil) - // assert the response status code for expected value 200 OK. - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // verify response data - responseBody, err = ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - c.Assert(true, Equals, bytes.Equal(responseBody, []byte("hello two"))) - - // data for new object to be uploaded. - buffer3 := bytes.NewReader([]byte("hello three")) - objectName = "testObject3" - request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName), - int64(buffer3.Len()), buffer3, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute HTTP request. - response, err = client.Do(request) - c.Assert(err, IsNil) - // verify the response code with the expected value of 200 OK. - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // fetch the object which was uploaded above. - request, err = newTestRequest("GET", getPutObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // verify object. - responseBody, err = ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - c.Assert(true, Equals, bytes.Equal(responseBody, []byte("hello three"))) -} - -// TestNotImplemented - Validates response for obtaining policy on an non-existent bucket and object. -func (s *TestSuiteXL) TestNotImplemented(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - request, err := newTestRequest("GET", s.endPoint+"/"+bucketName+"/object?policy", - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusNotImplemented) -} - -// TestHeader - Validates the error response for an attempt to fetch non-existent object. -func (s *TestSuiteXL) TestHeader(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // obtain HTTP request to fetch an object from non-existent bucket/object. - request, err := newTestRequest("GET", getGetObjectURL(s.endPoint, bucketName, "testObject"), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - // asserting for the expected error response. - verifyError(c, response, "NoSuchBucket", "The specified bucket does not exist.", http.StatusNotFound) -} - -// TestPutBucket - Validating bucket creation. -func (s *TestSuiteXL) TestPutBucket(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // Block 1: Testing for racey access - // The assertion is removed from this block since the purpose of this block is to find races - // The purpose this block is not to check for correctness of functionality - // Run the test with -race flag to utilize this - var wg sync.WaitGroup - for i := 0; i < ConcurrencyLevel; i++ { - wg.Add(1) - go func() { - defer wg.Done() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - response, err := client.Do(request) - defer response.Body.Close() - }() - } - wg.Wait() - - bucketName = getRandomBucketName() - //Block 2: testing for correctness of the functionality - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - response.Body.Close() -} - -// TestCopyObject - Validates copy object. -// The following is the test flow. -// 1. Create bucket. -// 2. Insert Object. -// 3. Use "X-Amz-Copy-Source" header to copy the previously inserted object. -// 4. Validate the content of copied object. -func (s *TestSuiteXL) TestCopyObject(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request to create bucket. - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // content for the object to be inserted. - buffer1 := bytes.NewReader([]byte("hello world")) - objectName := "testObject" - // create HTTP request for object upload. - request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName), - int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey) - request.Header.Set("Content-Type", "application/json") - c.Assert(err, IsNil) - // execute the HTTP request for object upload. - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - objectName2 := "testObject2" - // creating HTTP request for uploading the object. - request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName2), - 0, nil, s.accessKey, s.secretKey) - // setting the "X-Amz-Copy-Source" to allow copying the content of - // previously uploaded object. - request.Header.Set("X-Amz-Copy-Source", "/"+bucketName+"/"+objectName) - c.Assert(err, IsNil) - // execute the HTTP request. - // the content is expected to have the content of previous disk. - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // creating HTTP request to fetch the previously uploaded object. - request, err = newTestRequest("GET", getGetObjectURL(s.endPoint, bucketName, objectName2), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - // executing the HTTP request. - response, err = client.Do(request) - c.Assert(err, IsNil) - // validating the response status code. - c.Assert(response.StatusCode, Equals, http.StatusOK) - // reading the response body. - // response body is expected to have the copied content of the first uploaded object. - object, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - c.Assert(string(object), Equals, "hello world") - c.Assert(response.Header.Get("Content-Type"), Equals, "application/json") -} - -// TestPutObject - Tests successful put object request. -func (s *TestSuiteXL) TestPutObject(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request to create bucket. - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // content for new object upload. - buffer1 := bytes.NewReader([]byte("hello world")) - objectName := "testObject" - // creating HTTP request for object upload. - request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName), - int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - // execute the HTTP request for object upload. - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // fetch the object back and verify its contents. - request, err = newTestRequest("GET", getGetObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - // execute the HTTP request to fetch the object. - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - c.Assert(response.ContentLength, Equals, int64(len([]byte("hello world")))) - var buffer2 bytes.Buffer - // retrive the contents of response body. - n, err := io.Copy(&buffer2, response.Body) - c.Assert(err, IsNil) - c.Assert(n, Equals, int64(len([]byte("hello world")))) - // asserted the contents of the fetched object with the expected result. - c.Assert(true, Equals, bytes.Equal(buffer2.Bytes(), []byte("hello world"))) -} - -// TestPutObjectLongName - Long Object name strings are created and uploaded, validated for success. -func (s *TestSuiteXL) TestPutObjectLongName(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request to create bucket. - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - // content for the object to be uploaded. - buffer := bytes.NewReader([]byte("hello world")) - // make long object name. - longObjName := fmt.Sprintf("%0255d/%0255d/%0255d", 1, 1, 1) - // create new HTTP request to insert the object. - request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, longObjName), - int64(buffer.Len()), buffer, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - // execute the HTTP request. - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - // make long object name. - longObjName = fmt.Sprintf("%0256d", 1) - request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, longObjName), - int64(buffer.Len()), buffer, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusNotFound) -} - -// TestListBuckets - Make request for listing of all buckets. -// XML response is parsed. -// Its success verifies the format of the response. -func (s *TestSuiteXL) TestListBuckets(c *C) { - // create HTTP request for listing buckets. - request, err := newTestRequest("GET", getListBucketURL(s.endPoint), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request to list buckets. - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - var results ListBucketsResponse - // parse the list bucket response. - decoder := xml.NewDecoder(response.Body) - err = decoder.Decode(&results) - // validating that the xml-decoding/parsing was successfull. - c.Assert(err, IsNil) -} - -// TestNotBeAbleToCreateObjectInNonexistentBucket - Validates the error response -// on an attempt to upload an object into a non-existent bucket. -func (s *TestSuiteXL) TestNotBeAbleToCreateObjectInNonexistentBucket(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // content of the object to be uploaded. - buffer1 := bytes.NewReader([]byte("hello world")) - - // preparing for upload by generating the upload URL. - objectName := "test-object" - request, err := newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName), - int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // Execute the HTTP request. - response, err := client.Do(request) - c.Assert(err, IsNil) - // Assert the response error message. - verifyError(c, response, "NoSuchBucket", "The specified bucket does not exist.", http.StatusNotFound) -} - // TestGetOnObject - Asserts properties for GET on an object. // GET requests on an object retrieves the object from server. // Tests behaviour when If-Match/If-None-Match headers are set on the request. @@ -915,13 +123,13 @@ func (s *TestSuiteXL) TestGetOnObject(c *C) { c.Assert(response.StatusCode, Equals, http.StatusNotModified) } -// TestHeadOnObjectLastModified - Asserts response for HEAD on an object. -// HEAD requests on an object validates the existence of the object. -// The responses for fetching the object when If-Modified-Since -// and If-Unmodified-Since headers set are validated. -// If-Modified-Since - Return the object only if it has been modified since the specified time, else return a 304 (not modified). -// If-Unmodified-Since - Return the object only if it has not been modified since the specified time, else return a 412 (precondition failed). -func (s *TestSuiteXL) TestHeadOnObjectLastModified(c *C) { +// TestCopyObject - Validates copy object. +// The following is the test flow. +// 1. Create bucket. +// 2. Insert Object. +// 3. Use "X-Amz-Copy-Source" header to copy the previously inserted object. +// 4. Validate the content of copied object. +func (s *TestSuiteXL) TestCopyObject(c *C) { // generate a random bucket name. bucketName := getRandomBucketName() // HTTP request to create the bucket. @@ -935,84 +143,48 @@ func (s *TestSuiteXL) TestHeadOnObjectLastModified(c *C) { c.Assert(err, IsNil) c.Assert(response.StatusCode, Equals, http.StatusOK) - // preparing for object upload. - objectName := "test-object" - // content for the object to be uploaded. + // content for the object to be inserted. buffer1 := bytes.NewReader([]byte("hello world")) - // obtaining URL for uploading the object. + objectName := "testObject" + // create HTTP request for object upload. request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName), int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey) + request.Header.Set("Content-Type", "application/json") c.Assert(err, IsNil) - - // executing the HTTP request to download the object. + // execute the HTTP request for object upload. response, err = client.Do(request) c.Assert(err, IsNil) c.Assert(response.StatusCode, Equals, http.StatusOK) - // make HTTP request to obtain object info. - request, err = newTestRequest("HEAD", getHeadObjectURL(s.endPoint, bucketName, objectName), + + objectName2 := "testObject2" + // creating HTTP request for uploading the object. + request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName2), 0, nil, s.accessKey, s.secretKey) + // setting the "X-Amz-Copy-Source" to allow copying the content of + // previously uploaded object. + request.Header.Set("X-Amz-Copy-Source", "/"+bucketName+"/"+objectName) c.Assert(err, IsNil) // execute the HTTP request. + // the content is expected to have the content of previous disk. response, err = client.Do(request) c.Assert(err, IsNil) - // verify the status of the HTTP response. c.Assert(response.StatusCode, Equals, http.StatusOK) - // retrive the info of last modification time of the object from the response header. - lastModified := response.Header.Get("Last-Modified") - // Parse it into time.Time structure. - t, err := time.Parse(http.TimeFormat, lastModified) - c.Assert(err, IsNil) - - // make HTTP request to obtain object info. - // But this time set the "If-Modified-Since" header to be a minute more than the actual - // last modified time of the object. - request, err = newTestRequest("HEAD", getHeadObjectURL(s.endPoint, bucketName, objectName), + // creating HTTP request to fetch the previously uploaded object. + request, err = newTestRequest("GET", getGetObjectURL(s.endPoint, bucketName, objectName2), 0, nil, s.accessKey, s.secretKey) c.Assert(err, IsNil) - request.Header.Set("If-Modified-Since", t.Add(1*time.Minute).UTC().Format(http.TimeFormat)) + // executing the HTTP request. response, err = client.Do(request) c.Assert(err, IsNil) - // Since the "If-Modified-Since" header was ahead in time compared to the actual modified time of the object - // expecting the response status to be http.StatusNotModified. - c.Assert(response.StatusCode, Equals, http.StatusNotModified) - - // Again, obtain the object info. - // This time setting "If-Unmodified-Since" to a time after the object is modified. - // As documented above, expecting http.StatusPreconditionFailed. - request, err = newTestRequest("HEAD", getHeadObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - request.Header.Set("If-Unmodified-Since", t.Add(-1*time.Minute).UTC().Format(http.TimeFormat)) - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusPreconditionFailed) -} - -// TestHeadOnBucket - Validates response for HEAD on the bucket. -// HEAD request on the bucket validates the existence of the bucket. -func (s *TestSuiteXL) TestHeadOnBucket(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getHEADBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request to create bucket. - response, err := client.Do(request) - c.Assert(err, IsNil) + // validating the response status code. c.Assert(response.StatusCode, Equals, http.StatusOK) - // make HEAD request on the bucket. - request, err = newTestRequest("HEAD", getHEADBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) + // reading the response body. + // response body is expected to have the copied content of the first uploaded object. + object, err := ioutil.ReadAll(response.Body) c.Assert(err, IsNil) - // execute the HTTP request. - response, err = client.Do(request) - c.Assert(err, IsNil) - // Asserting the response status for expected value of http.StatusOK. - c.Assert(response.StatusCode, Equals, http.StatusOK) + c.Assert(string(object), Equals, "hello world") + c.Assert(response.Header.Get("Content-Type"), Equals, "application/json") } // TestContentTypePersists - Object upload with different Content-type is first done. @@ -1110,1256 +282,3 @@ func (s *TestSuiteXL) TestContentTypePersists(c *C) { // Assert if the content-type header set during the object upload persists. c.Assert(response.Header.Get("Content-Type"), Equals, "application/json") } - -// TestPartialContent - Validating for GetObject with partial content request. -// By setting the Range header, A request to send specific bytes range of data from an -// already uploaded object can be done. -func (s *TestSuiteXL) TestPartialContent(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP Request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request to create the bucket. - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - // content for the object to be uploaded. - buffer1 := bytes.NewReader([]byte("Hello World")) - objectName := "test-object" - // make HTTP request to upload the object. - request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName), - int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // Execute the HTTP request for object upload. - response, err = client.Do(request) - // verify that the upload was successfull. - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // Set of cases which will be used to create Get Object requests with specific content range. - var table = []struct { - byteRange string - expectedString string - }{ - // Request bytes 6-7 of the uploaded object. - // Since the object content was "Hello World", expecting "Wo" in the response. - {"6-7", "Wo"}, - // Request for first 6 bytes of object content. - {"6-", "World"}, - // Request for last 7 bytes of the object content. - {"-7", "o World"}, - } - for _, t := range table { - // make HTTP request to fetch the object. - request, err = newTestRequest("GET", getGetObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - // Set the Range header with range of bytes data of the uploaded object to be fetched. - request.Header.Add("Range", "bytes="+t.byteRange) - - client = http.Client{} - // execute the HTTP request. - response, err = client.Do(request) - c.Assert(err, IsNil) - // since the complete object is not fetched, - // http.StatusPartialContent is expected to be the response status. - c.Assert(response.StatusCode, Equals, http.StatusPartialContent) - // parse the response body to obtain the partial content requested for. - partialObject, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - // asserting the obtained partial content with the expected data. - c.Assert(string(partialObject), Equals, t.expectedString) - } -} - -// TestListObjectsHandlerErrors - Setting invalid parameters to List Objects -// and then asserting the error response with the expected one. -func (s *TestSuiteXL) TestListObjectsHandlerErrors(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request to create bucket. - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // create HTTP request with invalid value of max-keys parameter. - // max-keys is set to -2. - request, err = newTestRequest("GET", getListObjectsURL(s.endPoint, bucketName, "-2"), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - client = http.Client{} - // execute the HTTP request. - response, err = client.Do(request) - c.Assert(err, IsNil) - // validating the error response. - verifyError(c, response, "InvalidArgument", "Argument maxKeys must be an integer between 0 and 2147483647.", http.StatusBadRequest) -} - -// TestPutBucketErrors - request for non valid bucket operation -// and validate it with expected error result. -func (s *TestSuiteXL) TestPutBucketErrors(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // generating a HTTP request to create bucket. - // using invalid bucket name. - request, err := newTestRequest("PUT", s.endPoint+"/putbucket-.", - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - // expected to fail with error message "InvalidBucketName". - verifyError(c, response, "InvalidBucketName", "The specified bucket is not valid.", http.StatusBadRequest) - // HTTP request to create the bucket. - request, err = newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request to create bucket. - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - // make HTTP request to create the same bucket again. - // expected to fail with error message "BucketAlreadyOwnedByYou". - request, err = newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - response, err = client.Do(request) - c.Assert(err, IsNil) - verifyError(c, response, "BucketAlreadyOwnedByYou", "Your previous request to create the named bucket succeeded and you already own it.", - http.StatusConflict) - - // request for ACL. - // Since Minio server doesn't support ACL's the request is expected to fail with "NotImplemented" error message. - request, err = newTestRequest("PUT", s.endPoint+"/"+bucketName+"?acl", - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - response, err = client.Do(request) - c.Assert(err, IsNil) - verifyError(c, response, "NotImplemented", "A header you provided implies functionality that is not implemented.", http.StatusNotImplemented) -} - -// TestGetObjectLarge10MiB - Tests validate fetching of an object of size 10MB. -func (s *TestSuiteXL) TestGetObjectLarge10MiB(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // form HTTP reqest to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request to create the bucket. - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - var buffer bytes.Buffer - line := `1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,123"` - // Create 10MiB content where each line contains 1024 characters. - for i := 0; i < 10*1024; i++ { - buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line)) - } - putContent := buffer.String() - - buf := bytes.NewReader([]byte(putContent)) - - objectName := "test-big-object" - // create HTTP request for object upload. - request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName), - int64(buf.Len()), buf, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request. - response, err = client.Do(request) - c.Assert(err, IsNil) - // Assert the status code to verify successful upload. - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // prepare HTTP requests to download the object. - request, err = newTestRequest("GET", getPutObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request to download the object. - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - // extract the content from response body. - getContent, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - - // Compare putContent and getContent. - c.Assert(string(getContent), Equals, putContent) -} - -// TestGetObjectLarge11MiB - Tests validate fetching of an object of size 10MB. -func (s *TestSuiteXL) TestGetObjectLarge11MiB(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request. - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - var buffer bytes.Buffer - line := `1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,123` - // Create 11MiB content where each line contains 1024 characters. - for i := 0; i < 11*1024; i++ { - buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line)) - } - putMD5 := sumMD5(buffer.Bytes()) - - objectName := "test-11Mb-object" - // Put object - buf := bytes.NewReader(buffer.Bytes()) - // create HTTP request foe object upload. - request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName), - int64(buf.Len()), buf, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request for object upload. - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // create HTTP request to download the object. - request, err = newTestRequest("GET", getGetObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request. - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - // fetch the content from response body. - getContent, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - - // Get md5Sum of the response content. - getMD5 := sumMD5(getContent) - - // Compare putContent and getContent. - c.Assert(hex.EncodeToString(putMD5), Equals, hex.EncodeToString(getMD5)) -} - -// TestGetPartialObjectMisAligned - tests get object partially mis-aligned. -// create a large buffer of mis-aligned data and upload it. -// then make partial range requests to while fetching it back and assert the response content. -func (s *TestSuiteXL) TestGetPartialObjectMisAligned(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.testServer.AccessKey, s.testServer.SecretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request to create the bucket. - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - var buffer bytes.Buffer - line := `1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,123` - - rand.Seed(time.Now().UTC().UnixNano()) - // Create a misalgined data. - for i := 0; i < 13*rand.Intn(1<<16); i++ { - buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line[:rand.Intn(1<<8)])) - } - putContent := buffer.String() - buf := bytes.NewReader([]byte(putContent)) - - objectName := "test-big-file" - // HTTP request to upload the object. - request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName), - int64(buf.Len()), buf, s.testServer.AccessKey, s.testServer.SecretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request to upload the object. - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // test Cases containing data to make partial range requests. - // also has expected response data. - var testCases = []struct { - byteRange string - expectedString string - }{ - // request for byte range 10-11. - // expecting the result to contain only putContent[10:12] bytes. - {"10-11", putContent[10:12]}, - // request for object data after the first byte. - {"1-", putContent[1:]}, - // request for object data after the first byte. - {"6-", putContent[6:]}, - // request for last 2 bytes of th object. - {"-2", putContent[len(putContent)-2:]}, - // request for last 7 bytes of the object. - {"-7", putContent[len(putContent)-7:]}, - } - for _, t := range testCases { - // HTTP request to download the object. - request, err = newTestRequest("GET", getGetObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.testServer.AccessKey, s.testServer.SecretKey) - c.Assert(err, IsNil) - // Get partial content based on the byte range set. - request.Header.Add("Range", "bytes="+t.byteRange) - - client = http.Client{} - // execute the HTTP request. - response, err = client.Do(request) - c.Assert(err, IsNil) - // Since only part of the object is requested, expecting response status to be http.StatusPartialContent . - c.Assert(response.StatusCode, Equals, http.StatusPartialContent) - // parse the HTTP response body. - getContent, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - - // Compare putContent and getContent. - c.Assert(string(getContent), Equals, t.expectedString) - } -} - -// TestGetPartialObjectLarge11MiB - Test validates partial content request for a 11MiB object. -func (s *TestSuiteXL) TestGetPartialObjectLarge11MiB(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request to create the bucket. - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - var buffer bytes.Buffer - line := `234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,123` - // Create 11MiB content where each line contains 1024 - // characters. - for i := 0; i < 11*1024; i++ { - buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line)) - } - putContent := buffer.String() - - objectName := "test-large-11Mb-object" - - buf := bytes.NewReader([]byte(putContent)) - // HTTP request to upload the object. - request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName), - int64(buf.Len()), buf, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request to upload the object. - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // HTTP request to download the object. - request, err = newTestRequest("GET", getGetObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - // This range spans into first two blocks. - request.Header.Add("Range", "bytes=10485750-10485769") - - client = http.Client{} - // execute the HTTP request. - response, err = client.Do(request) - c.Assert(err, IsNil) - // Since only part of the object is requested, expecting response status to be http.StatusPartialContent . - c.Assert(response.StatusCode, Equals, http.StatusPartialContent) - // read the downloaded content from the response body. - getContent, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - - // Compare putContent and getContent. - c.Assert(string(getContent), Equals, putContent[10485750:10485770]) -} - -// TestGetPartialObjectLarge11MiB - Test validates partial content request for a 10MiB object. -func (s *TestSuiteXL) TestGetPartialObjectLarge10MiB(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request to create bucket. - response, err := client.Do(request) - // expecting the error to be nil. - c.Assert(err, IsNil) - // expecting the HTTP response status code to 200 OK. - c.Assert(response.StatusCode, Equals, http.StatusOK) - - var buffer bytes.Buffer - line := `1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, - 1234567890,1234567890,1234567890,123` - // Create 10MiB content where each line contains 1024 characters. - for i := 0; i < 10*1024; i++ { - buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line)) - } - - putContent := buffer.String() - buf := bytes.NewReader([]byte(putContent)) - - objectName := "test-big-10Mb-file" - // HTTP request to upload the object. - request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName), - int64(buf.Len()), buf, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request to upload the object. - response, err = client.Do(request) - c.Assert(err, IsNil) - // verify whether upload was successfull. - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // HTTP request to download the object. - request, err = newTestRequest("GET", getGetObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - // Get partial content based on the byte range set. - request.Header.Add("Range", "bytes=2048-2058") - - client = http.Client{} - // execute the HTTP request to download the partila content. - response, err = client.Do(request) - c.Assert(err, IsNil) - // Since only part of the object is requested, expecting response status to be http.StatusPartialContent . - c.Assert(response.StatusCode, Equals, http.StatusPartialContent) - // read the downloaded content from the response body. - getContent, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - - // Compare putContent and getContent. - c.Assert(string(getContent), Equals, putContent[2048:2059]) -} - -// TestGetObjectErrors - Tests validate error response for invalid object operations. -func (s *TestSuiteXL) TestGetObjectErrors(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request to create bucket. - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - objectName := "test-non-exitent-object" - // HTTP request to download the object. - // Since the specified object doesn't exist in the given bucket, - // expected to fail with error message "NoSuchKey" - request, err = newTestRequest("GET", getGetObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - verifyError(c, response, "NoSuchKey", "The specified key does not exist.", http.StatusNotFound) - - // request to download an object, but an invalid bucket name is set. - request, err = newTestRequest("GET", getGetObjectURL(s.endPoint, "/getobjecterrors-.", objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - // execute the HTTP request. - response, err = client.Do(request) - c.Assert(err, IsNil) - // expected to fail with "InvalidBucketName". - verifyError(c, response, "InvalidBucketName", "The specified bucket is not valid.", http.StatusBadRequest) -} - -// TestGetObjectRangeErrors - Validate error response when object is fetched with incorrect byte range value. -func (s *TestSuiteXL) TestGetObjectRangeErrors(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request to create bucket. - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // content for the object to be uploaded. - buffer1 := bytes.NewReader([]byte("Hello World")) - - objectName := "test-object" - // HTTP request to upload the object. - request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName), - int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request to upload the object. - response, err = client.Do(request) - c.Assert(err, IsNil) - // verify whether upload was successfull. - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // HTTP request to download the object. - request, err = newTestRequest("GET", getGetObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - // invalid byte range set. - request.Header.Add("Range", "bytes=7-6") - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request. - response, err = client.Do(request) - c.Assert(err, IsNil) - // expected to fail with "InvalidRange" error message. - verifyError(c, response, "InvalidRange", "The requested range cannot be satisfied.", http.StatusRequestedRangeNotSatisfiable) -} - -// TestObjectMultipartAbort - Test validates abortion of a multipart upload after uploading 2 parts. -func (s *TestSuiteXL) TestObjectMultipartAbort(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request to create bucket. - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - objectName := "test-multipart-object" - // construct HTTP request to initiate a NewMultipart upload. - request, err = newTestRequest("POST", getNewMultipartURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - // execute the HTTP request initiating the new multipart upload. - response, err = client.Do(request) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // parse the response body and obtain the new upload ID. - decoder := xml.NewDecoder(response.Body) - newResponse := &InitiateMultipartUploadResponse{} - - err = decoder.Decode(newResponse) - c.Assert(err, IsNil) - c.Assert(len(newResponse.UploadID) > 0, Equals, true) - // uploadID to be used for rest of the multipart operations on the object. - uploadID := newResponse.UploadID - - // content for the part to be uploaded. - buffer1 := bytes.NewReader([]byte("hello world")) - // HTTP request for the part to be uploaded. - request, err = newTestRequest("PUT", getPartUploadURL(s.endPoint, bucketName, objectName, uploadID, "1"), - int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - // execute the HTTP request to upload the first part. - response1, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response1.StatusCode, Equals, http.StatusOK) - - // content for the second part to be uploaded. - buffer2 := bytes.NewReader([]byte("hello world")) - // HTTP request for the second part to be uploaded. - request, err = newTestRequest("PUT", getPartUploadURL(s.endPoint, bucketName, objectName, uploadID, "2"), - int64(buffer2.Len()), buffer2, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - // execute the HTTP request to upload the second part. - response2, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response2.StatusCode, Equals, http.StatusOK) - // HTTP request for aborting the multipart upload. - request, err = newTestRequest("DELETE", getAbortMultipartUploadURL(s.endPoint, bucketName, objectName, uploadID), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - // execute the HTTP request to abort the multipart upload. - response3, err := client.Do(request) - c.Assert(err, IsNil) - // expecting the response status code to be http.StatusNoContent. - // The assertion validates the success of Abort Multipart operation. - c.Assert(response3.StatusCode, Equals, http.StatusNoContent) -} - -// TestBucketMultipartList - Initiates a NewMultipart upload, uploads parts and validates listing of the parts. -func (s *TestSuiteXL) TestBucketMultipartList(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), 0, - nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request to create bucket. - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, 200) - - objectName := "test-multipart-object" - // construct HTTP request to initiate a NewMultipart upload. - request, err = newTestRequest("POST", getNewMultipartURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - // execute the HTTP request initiating the new multipart upload. - response, err = client.Do(request) - c.Assert(err, IsNil) - // expecting the response status code to be http.StatusOK(200 OK) . - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // parse the response body and obtain the new upload ID. - decoder := xml.NewDecoder(response.Body) - newResponse := &InitiateMultipartUploadResponse{} - - err = decoder.Decode(newResponse) - c.Assert(err, IsNil) - c.Assert(len(newResponse.UploadID) > 0, Equals, true) - // uploadID to be used for rest of the multipart operations on the object. - uploadID := newResponse.UploadID - - // content for the part to be uploaded. - buffer1 := bytes.NewReader([]byte("hello world")) - // HTTP request for the part to be uploaded. - request, err = newTestRequest("PUT", getPartUploadURL(s.endPoint, bucketName, objectName, uploadID, "1"), - int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - // execute the HTTP request to upload the first part. - response1, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response1.StatusCode, Equals, http.StatusOK) - - // content for the second part to be uploaded. - buffer2 := bytes.NewReader([]byte("hello world")) - // HTTP request for the second part to be uploaded. - request, err = newTestRequest("PUT", getPartUploadURL(s.endPoint, bucketName, objectName, uploadID, "2"), - int64(buffer2.Len()), buffer2, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - // execute the HTTP request to upload the second part. - response2, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response2.StatusCode, Equals, http.StatusOK) - - // HTTP request to ListMultipart Uploads. - request, err = newTestRequest("GET", getListMultipartURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - // execute the HTTP request. - response3, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response3.StatusCode, Equals, http.StatusOK) - - // The reason to duplicate this structure here is to verify if the - // unmarshalling works from a client perspective, specifically - // while unmarshalling time.Time type for 'Initiated' field. - // time.Time does not honor xml marshaler, it means that we need - // to encode/format it before giving it to xml marshalling. - - // This below check adds client side verification to see if its - // truly parseable. - - // listMultipartUploadsResponse - format for list multipart uploads response. - type listMultipartUploadsResponse struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult" json:"-"` - - Bucket string - KeyMarker string - UploadIDMarker string `xml:"UploadIdMarker"` - NextKeyMarker string - NextUploadIDMarker string `xml:"NextUploadIdMarker"` - EncodingType string - MaxUploads int - IsTruncated bool - // All the in progress multipart uploads. - Uploads []struct { - Key string - UploadID string `xml:"UploadId"` - Initiator Initiator - Owner Owner - StorageClass string - Initiated time.Time // Keep this native to be able to parse properly. - } - Prefix string - Delimiter string - CommonPrefixes []CommonPrefix - } - - // parse the response body. - decoder = xml.NewDecoder(response3.Body) - newResponse3 := &listMultipartUploadsResponse{} - err = decoder.Decode(newResponse3) - c.Assert(err, IsNil) - // Assert the bucket name in the response with the expected bucketName. - c.Assert(newResponse3.Bucket, Equals, bucketName) - // Assert the bucket name in the response with the expected bucketName. - c.Assert(newResponse3.IsTruncated, Equals, false) -} - -// TestMakeBucketLocation - tests make bucket location header response. -func (s *TestSuiteXL) TestMakeBucketLocation(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request to create bucket. - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, 200) - // Validate location header value equals proper bucket name. - c.Assert(response.Header.Get("Location"), Equals, "/"+bucketName) -} - -// TestValidateObjectMultipartUploadID - Test Initiates a new multipart upload and validates the uploadID. -func (s *TestSuiteXL) TestValidateObjectMultipartUploadID(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request to create bucket. - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, 200) - - objectName := "directory1/directory2/object" - // construct HTTP request to initiate a NewMultipart upload. - request, err = newTestRequest("POST", getNewMultipartURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - // execute the HTTP request initiating the new multipart upload. - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // parse the response body and obtain the new upload ID. - decoder := xml.NewDecoder(response.Body) - newResponse := &InitiateMultipartUploadResponse{} - err = decoder.Decode(newResponse) - // expecting the decoding error to be nil. - c.Assert(err, IsNil) - // Verifying for Upload ID value to be greater than 0. - c.Assert(len(newResponse.UploadID) > 0, Equals, true) -} - -// TestObjectMultipartListError - Initiates a NewMultipart upload, uploads parts and validates -// error response for an incorrect max-parts parameter . -func (s *TestSuiteXL) TestObjectMultipartListError(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request to create bucket. - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, 200) - - objectName := "test-multipart-object" - // construct HTTP request to initiate a NewMultipart upload. - request, err = newTestRequest("POST", getNewMultipartURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - // execute the HTTP request initiating the new multipart upload. - response, err = client.Do(request) - c.Assert(response.StatusCode, Equals, http.StatusOK) - // parse the response body and obtain the new upload ID. - decoder := xml.NewDecoder(response.Body) - newResponse := &InitiateMultipartUploadResponse{} - - err = decoder.Decode(newResponse) - c.Assert(err, IsNil) - c.Assert(len(newResponse.UploadID) > 0, Equals, true) - // uploadID to be used for rest of the multipart operations on the object. - uploadID := newResponse.UploadID - - // content for the part to be uploaded. - buffer1 := bytes.NewReader([]byte("hello world")) - // HTTP request for the part to be uploaded. - request, err = newTestRequest("PUT", getPartUploadURL(s.endPoint, bucketName, objectName, uploadID, "1"), - int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - // execute the HTTP request to upload the first part. - response1, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response1.StatusCode, Equals, http.StatusOK) - - // content for the second part to be uploaded. - buffer2 := bytes.NewReader([]byte("hello world")) - // HTTP request for the second part to be uploaded. - request, err = newTestRequest("PUT", getPartUploadURL(s.endPoint, bucketName, objectName, uploadID, "2"), - int64(buffer2.Len()), buffer2, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - // execute the HTTP request to upload the second part. - response2, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response2.StatusCode, Equals, http.StatusOK) - // HTTP request to ListMultipart Uploads. - // max-keys is set to invalid value of -2. - request, err = newTestRequest("GET", getListMultipartURLWithParams(s.endPoint, bucketName, objectName, uploadID, "-2"), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - // execute the HTTP request. - response4, err := client.Do(request) - c.Assert(err, IsNil) - // Since max-keys parameter in the ListMultipart request set to invalid value of -2, - // its expected to fail with error message "InvalidArgument". - verifyError(c, response4, "InvalidArgument", "Argument maxParts must be an integer between 1 and 10000.", http.StatusBadRequest) -} - -// TestMultipartErrorEntityTooSmall - initiates a new multipart upload, -// uploads 2 parts of size less than 5MB, upon complete multipart upload -// validates EntityTooSmall error returned by the operation. -func (s *TestSuiteXL) TestMultipartErrorEntityTooSmall(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request to create bucket. - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, 200) - - objectName := "test-multipart-object" - // construct HTTP request to initiate a NewMultipart upload. - request, err = newTestRequest("POST", getNewMultipartURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request initiating the new multipart upload. - response, err = client.Do(request) - c.Assert(err, IsNil) - // expecting the response status code to be http.StatusOK(200 OK). - c.Assert(response.StatusCode, Equals, http.StatusOK) - // parse the response body and obtain the new upload ID. - decoder := xml.NewDecoder(response.Body) - newResponse := &InitiateMultipartUploadResponse{} - - err = decoder.Decode(newResponse) - c.Assert(err, IsNil) - c.Assert(len(newResponse.UploadID) > 0, Equals, true) - // uploadID to be used for rest of the multipart operations on the object. - uploadID := newResponse.UploadID - - // content for the part to be uploaded. - // Create a byte array of 4MB. - data := bytes.Repeat([]byte("0123456789abcdef"), 4*1024*1024/16) - // calculate md5Sum of the data. - hasher := md5.New() - hasher.Write(data) - md5Sum := hasher.Sum(nil) - - buffer1 := bytes.NewReader(data) - // HTTP request for the part to be uploaded. - request, err = newTestRequest("PUT", getPartUploadURL(s.endPoint, bucketName, objectName, uploadID, "1"), - int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey) - // set the Content-Md5 header to the base64 encoding the md5Sum of the content. - request.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(md5Sum)) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request to upload the first part. - response1, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response1.StatusCode, Equals, http.StatusOK) - - // content for the second part to be uploaded will be same as first part. - hasher = md5.New() - hasher.Write(data) - // calculate md5Sum of the data. - md5Sum = hasher.Sum(nil) - - buffer2 := bytes.NewReader(data) - // HTTP request for the second part to be uploaded. - request, err = newTestRequest("PUT", getPartUploadURL(s.endPoint, bucketName, objectName, uploadID, "2"), - int64(buffer2.Len()), buffer2, s.accessKey, s.secretKey) - // set the Content-Md5 header to the base64 encoding the md5Sum of the content. - request.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(md5Sum)) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request to upload the second part. - response2, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response2.StatusCode, Equals, http.StatusOK) - - // Complete multipart upload - completeUploads := &completeMultipartUpload{ - Parts: []completePart{ - { - PartNumber: 1, - ETag: response1.Header.Get("ETag"), - }, - { - PartNumber: 2, - ETag: response2.Header.Get("ETag"), - }, - }, - } - - completeBytes, err := xml.Marshal(completeUploads) - c.Assert(err, IsNil) - // Indicating that all parts are uploaded and initiating completeMultipartUpload. - request, err = newTestRequest("POST", getCompleteMultipartUploadURL(s.endPoint, bucketName, objectName, uploadID), int64(len(completeBytes)), bytes.NewReader(completeBytes), s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - // Execute the complete multipart request. - response, err = client.Do(request) - c.Assert(err, IsNil) - // verify whether complete multipart was successfull. - verifyError(c, response, "EntityTooSmall", "Your proposed upload is smaller than the minimum allowed object size.", http.StatusOK) -} - -// TestObjectMultipart - Initiates a NewMultipart upload, uploads 2 parts, -// completes the multipart upload and validates the status of the operation. -func (s *TestSuiteXL) TestObjectMultipart(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request to create bucket. - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, 200) - - objectName := "test-multipart-object" - // construct HTTP request to initiate a NewMultipart upload. - request, err = newTestRequest("POST", getNewMultipartURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request initiating the new multipart upload. - response, err = client.Do(request) - c.Assert(err, IsNil) - // expecting the response status code to be http.StatusOK(200 OK). - c.Assert(response.StatusCode, Equals, http.StatusOK) - // parse the response body and obtain the new upload ID. - decoder := xml.NewDecoder(response.Body) - newResponse := &InitiateMultipartUploadResponse{} - - err = decoder.Decode(newResponse) - c.Assert(err, IsNil) - c.Assert(len(newResponse.UploadID) > 0, Equals, true) - // uploadID to be used for rest of the multipart operations on the object. - uploadID := newResponse.UploadID - - // content for the part to be uploaded. - // Create a byte array of 5MB. - data := bytes.Repeat([]byte("0123456789abcdef"), 5*1024*1024/16) - // calculate md5Sum of the data. - hasher := md5.New() - hasher.Write(data) - md5Sum := hasher.Sum(nil) - - buffer1 := bytes.NewReader(data) - // HTTP request for the part to be uploaded. - request, err = newTestRequest("PUT", getPartUploadURL(s.endPoint, bucketName, objectName, uploadID, "1"), - int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey) - // set the Content-Md5 header to the base64 encoding the md5Sum of the content. - request.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(md5Sum)) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request to upload the first part. - response1, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response1.StatusCode, Equals, http.StatusOK) - - // content for the second part to be uploaded. - // Create a byte array of 1 byte. - data = []byte("0") - - hasher = md5.New() - hasher.Write(data) - // calculate md5Sum of the data. - md5Sum = hasher.Sum(nil) - - buffer2 := bytes.NewReader(data) - // HTTP request for the second part to be uploaded. - request, err = newTestRequest("PUT", getPartUploadURL(s.endPoint, bucketName, objectName, uploadID, "2"), - int64(buffer2.Len()), buffer2, s.accessKey, s.secretKey) - // set the Content-Md5 header to the base64 encoding the md5Sum of the content. - request.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(md5Sum)) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request to upload the second part. - response2, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response2.StatusCode, Equals, http.StatusOK) - - // Complete multipart upload - completeUploads := &completeMultipartUpload{ - Parts: []completePart{ - { - PartNumber: 1, - ETag: response1.Header.Get("ETag"), - }, - { - PartNumber: 2, - ETag: response2.Header.Get("ETag"), - }, - }, - } - - completeBytes, err := xml.Marshal(completeUploads) - c.Assert(err, IsNil) - // Indicating that all parts are uploaded and initiating completeMultipartUpload. - request, err = newTestRequest("POST", getCompleteMultipartUploadURL(s.endPoint, bucketName, objectName, uploadID), - int64(len(completeBytes)), bytes.NewReader(completeBytes), s.accessKey, s.secretKey) - c.Assert(err, IsNil) - // Execute the complete multipart request. - response, err = client.Do(request) - c.Assert(err, IsNil) - // verify whether complete multipart was successfull. - c.Assert(response.StatusCode, Equals, http.StatusOK) -} - -// TestObjectMultipartOverwriteSinglePut - Initiates a NewMultipart upload, uploads 2 parts, -// completes the multipart upload and validates the status of the operation. -// then, after PutObject with same object name on the bucket, -// test validates for successful overwrite. -func (s *TestSuiteXL) TestObjectMultipartOverwriteSinglePut(c *C) { - // generate a random bucket name. - bucketName := getRandomBucketName() - // HTTP request to create the bucket. - request, err := newTestRequest("PUT", getMakeBucketURL(s.endPoint, bucketName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client := http.Client{} - // execute the HTTP request to create bucket. - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, 200) - - objectName := "test-multipart-object" - - request, err = newTestRequest("POST", getNewMultipartURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request initiating the new multipart upload. - response, err = client.Do(request) - c.Assert(err, IsNil) - // expecting the response status code to be http.StatusOK(200 OK). - c.Assert(response.StatusCode, Equals, http.StatusOK) - // parse the response body and obtain the new upload ID. - decoder := xml.NewDecoder(response.Body) - newResponse := &InitiateMultipartUploadResponse{} - - err = decoder.Decode(newResponse) - c.Assert(err, IsNil) - c.Assert(len(newResponse.UploadID) > 0, Equals, true) - // uploadID to be used for rest of the multipart operations on the object. - uploadID := newResponse.UploadID - - // content for the part to be uploaded. - // Create a byte array of 5MB. - data := bytes.Repeat([]byte("0123456789abcdef"), 5*1024*1024/16) - // calculate md5Sum of the data. - hasher := md5.New() - hasher.Write(data) - md5Sum := hasher.Sum(nil) - - buffer1 := bytes.NewReader(data) - // HTTP request for the part to be uploaded. - request, err = newTestRequest("PUT", getPartUploadURL(s.endPoint, bucketName, objectName, uploadID, "1"), - int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey) - // set the Content-Md5 header to the base64 encoding the md5Sum of the content. - request.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(md5Sum)) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request to upload the first part. - response1, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response1.StatusCode, Equals, http.StatusOK) - - // Create a byte array of 1 byte. - // content for the second part to be uploaded. - data = []byte("0") - - hasher = md5.New() - hasher.Write(data) - // calculate md5Sum of the data. - md5Sum = hasher.Sum(nil) - - buffer2 := bytes.NewReader(data) - // HTTP request for the second part to be uploaded. - request, err = newTestRequest("PUT", getPartUploadURL(s.endPoint, bucketName, objectName, uploadID, "2"), - int64(buffer2.Len()), buffer2, s.accessKey, s.secretKey) - // set the Content-Md5 header to the base64 encoding the md5Sum of the content. - request.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(md5Sum)) - c.Assert(err, IsNil) - - client = http.Client{} - // execute the HTTP request to upload the second part. - response2, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response2.StatusCode, Equals, http.StatusOK) - - // Complete multipart upload. - completeUploads := &completeMultipartUpload{ - Parts: []completePart{ - { - PartNumber: 1, - ETag: response1.Header.Get("ETag"), - }, - { - PartNumber: 2, - ETag: response2.Header.Get("ETag"), - }, - }, - } - - completeBytes, err := xml.Marshal(completeUploads) - c.Assert(err, IsNil) - // Indicating that all parts are uploaded and initiating completeMultipartUpload. - request, err = newTestRequest("POST", getCompleteMultipartUploadURL(s.endPoint, bucketName, objectName, uploadID), - int64(len(completeBytes)), bytes.NewReader(completeBytes), s.accessKey, s.secretKey) - c.Assert(err, IsNil) - - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // A new object is uploaded with same key as before. - // validation for successful overwrite is done by downloading back the object and reading its content. - buffer1 = bytes.NewReader([]byte("hello world")) - // HTTP request for uploading the object. - request, err = newTestRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objectName), - int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - // execute the HTTP request to upload the object. - response, err = client.Do(request) - c.Assert(err, IsNil) - // verify whether upload was successfull. - c.Assert(response.StatusCode, Equals, http.StatusOK) - // HTTP request for downloading the object. - request, err = newTestRequest("GET", getGetObjectURL(s.endPoint, bucketName, objectName), - 0, nil, s.accessKey, s.secretKey) - c.Assert(err, IsNil) - // execute the HTTP request to download the object. - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - // validate the response content length. - c.Assert(response.ContentLength, Equals, int64(len([]byte("hello world")))) - var buffer3 bytes.Buffer - // read the response body to obtain the content of downloaded object. - n, err := io.Copy(&buffer3, response.Body) - c.Assert(err, IsNil) - // validate the downloaded content. - // verify successful overwrite with the new content. - c.Assert(n, Equals, int64(len([]byte("hello world")))) - c.Assert(true, Equals, bytes.Equal(buffer3.Bytes(), []byte("hello world"))) -}