From ad779a8ba4fc44d08ae9e956cf7a233c589f27d0 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 20 Jun 2016 22:08:36 -0700 Subject: [PATCH] XL: Enable tests for large GetObject. (#1947) Ref #1946 --- server_test.go | 130 --------------------------------- server_xl_test.go | 178 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 178 insertions(+), 130 deletions(-) diff --git a/server_test.go b/server_test.go index 8fb984cbb..ce81d4c1d 100644 --- a/server_test.go +++ b/server_test.go @@ -1158,136 +1158,6 @@ func (s *MyAPISuite) TestGetObjectRangeErrors(c *C) { verifyError(c, response, "InvalidRange", "The requested range cannot be satisfied.", http.StatusRequestedRangeNotSatisfiable) } -func (s *MyAPISuite) TestGetObjectLarge10MiB(c *C) { - // Make bucket for this test. - request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/test-bucket-10", 0, nil) - c.Assert(err, IsNil) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - var buffer bytes.Buffer - line := "1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,123" - // Create 10MiB content where each line contains 1024 characters. - for i := 0; i < 10*1024; i++ { - buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line)) - } - putContent := buffer.String() - - // Put object - buf := bytes.NewReader([]byte(putContent)) - request, err = s.newRequest("PUT", testAPIFSCacheServer.URL+"/test-bucket-10/big-file-10", int64(buf.Len()), buf) - c.Assert(err, IsNil) - - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // Get object - request, err = s.newRequest("GET", testAPIFSCacheServer.URL+"/test-bucket-10/big-file-10", 0, nil) - c.Assert(err, IsNil) - - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - getContent, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - - // Compare putContent and getContent - c.Assert(string(getContent), Equals, putContent) -} - -func (s *MyAPISuite) TestGetObjectLarge11MiB(c *C) { - // Make bucket for this test. - request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/test-bucket-11", 0, nil) - c.Assert(err, IsNil) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - var buffer bytes.Buffer - line := "1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,123" - // Create 11MiB content where each line contains 1024 characters. - for i := 0; i < 11*1024; i++ { - buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line)) - } - putContent := buffer.String() - - // Put object - buf := bytes.NewReader([]byte(putContent)) - request, err = s.newRequest("PUT", testAPIFSCacheServer.URL+"/test-bucket-11/big-file-11", int64(buf.Len()), buf) - c.Assert(err, IsNil) - - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // Get object - request, err = s.newRequest("GET", testAPIFSCacheServer.URL+"/test-bucket-11/big-file-11", 0, nil) - c.Assert(err, IsNil) - - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - getContent, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - - // Compare putContent and getContent - c.Assert(string(getContent), Equals, putContent) -} - -func (s *MyAPISuite) TestGetPartialObjectLarge10MiB(c *C) { - // Make bucket for this test. - request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/test-bucket-10p", 0, nil) - c.Assert(err, IsNil) - - client := http.Client{} - response, err := client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - var buffer bytes.Buffer - line := "1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,123" - // Create 10MiB content where each line contains 1024 characters. - for i := 0; i < 10*1024; i++ { - buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line)) - } - putContent := buffer.String() - - // Put object - buf := bytes.NewReader([]byte(putContent)) - request, err = s.newRequest("PUT", testAPIFSCacheServer.URL+"/test-bucket-10p/big-file-10", int64(buf.Len()), buf) - c.Assert(err, IsNil) - - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - - // Get object - request, err = s.newRequest("GET", testAPIFSCacheServer.URL+"/test-bucket-10p/big-file-10", 0, nil) - c.Assert(err, IsNil) - request.Header.Add("Range", "bytes=2048-2058") - - client = http.Client{} - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusPartialContent) - getContent, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - - // Compare putContent and getContent - c.Assert(string(getContent), Equals, putContent[2048:2059]) -} - func (s *MyAPISuite) TestObjectMultipartAbort(c *C) { request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/objectmultipartabort", 0, nil) c.Assert(err, IsNil) diff --git a/server_xl_test.go b/server_xl_test.go index 49b4cee93..62149d56a 100644 --- a/server_xl_test.go +++ b/server_xl_test.go @@ -1137,6 +1137,184 @@ func (s *MyAPIXLSuite) TestPutBucketErrors(c *C) { verifyError(c, response, "NotImplemented", "A header you provided implies functionality that is not implemented.", http.StatusNotImplemented) } +func (s *MyAPIXLSuite) TestGetObjectLarge10MiB(c *C) { + // Make bucket for this test. + request, err := s.newRequest("PUT", testAPIXLServer.URL+"/test-bucket-10", 0, nil) + c.Assert(err, IsNil) + + client := http.Client{} + response, err := client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusOK) + + var buffer bytes.Buffer + line := "1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,123" + // Create 10MiB content where each line contains 1024 characters. + for i := 0; i < 10*1024; i++ { + buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line)) + } + putContent := buffer.String() + + // Put object + buf := bytes.NewReader([]byte(putContent)) + request, err = s.newRequest("PUT", testAPIXLServer.URL+"/test-bucket-10/big-file-10", int64(buf.Len()), buf) + c.Assert(err, IsNil) + + client = http.Client{} + response, err = client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusOK) + + // Get object + request, err = s.newRequest("GET", testAPIXLServer.URL+"/test-bucket-10/big-file-10", 0, nil) + c.Assert(err, IsNil) + + client = http.Client{} + response, err = client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusOK) + getContent, err := ioutil.ReadAll(response.Body) + c.Assert(err, IsNil) + + // Compare putContent and getContent + c.Assert(string(getContent), Equals, putContent) +} + +func (s *MyAPIXLSuite) TestGetObjectLarge11MiB(c *C) { + // Make bucket for this test. + request, err := s.newRequest("PUT", testAPIXLServer.URL+"/test-bucket-11", 0, nil) + c.Assert(err, IsNil) + + client := http.Client{} + response, err := client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusOK) + + var buffer bytes.Buffer + line := "1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,123" + // Create 11MiB content where each line contains 1024 characters. + for i := 0; i < 11*1024; i++ { + buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line)) + } + putMD5 := sumMD5(buffer.Bytes()) + + // Put object + buf := bytes.NewReader(buffer.Bytes()) + request, err = s.newRequest("PUT", testAPIXLServer.URL+"/test-bucket-11/big-file-11", int64(buf.Len()), buf) + c.Assert(err, IsNil) + + client = http.Client{} + response, err = client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusOK) + + // Get object + request, err = s.newRequest("GET", testAPIXLServer.URL+"/test-bucket-11/big-file-11", 0, nil) + c.Assert(err, IsNil) + + client = http.Client{} + response, err = client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusOK) + getContent, err := ioutil.ReadAll(response.Body) + c.Assert(err, IsNil) + + getMD5 := sumMD5(getContent) // Get md5. + + // Compare putContent and getContent + c.Assert(hex.EncodeToString(putMD5), Equals, hex.EncodeToString(getMD5)) +} + +func (s *MyAPIXLSuite) TestGetPartialObjectLarge11MiB(c *C) { + // Make bucket for this test. + request, err := s.newRequest("PUT", testAPIXLServer.URL+"/test-bucket-11p", 0, nil) + c.Assert(err, IsNil) + + client := http.Client{} + response, err := client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusOK) + + var buffer bytes.Buffer + line := "1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,123" + // Create 11MiB content where each line contains 1024 + // characters. + for i := 0; i < 11*1024; i++ { + buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line)) + } + putContent := buffer.String() + + // Put object + buf := bytes.NewReader([]byte(putContent)) + request, err = s.newRequest("PUT", testAPIXLServer.URL+"/test-bucket-11p/big-file-11", int64(buf.Len()), buf) + c.Assert(err, IsNil) + + client = http.Client{} + response, err = client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusOK) + + // Get object + request, err = s.newRequest("GET", testAPIXLServer.URL+"/test-bucket-11p/big-file-11", 0, nil) + c.Assert(err, IsNil) + // This range spans into first two blocks. + request.Header.Add("Range", "bytes=10485750-10485769") + + client = http.Client{} + response, err = client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusPartialContent) + getContent, err := ioutil.ReadAll(response.Body) + c.Assert(err, IsNil) + + // Compare putContent and getContent + c.Assert(string(getContent), Equals, putContent[10485750:10485770]) +} + +func (s *MyAPIXLSuite) TestGetPartialObjectLarge10MiB(c *C) { + // Make bucket for this test. + request, err := s.newRequest("PUT", testAPIXLServer.URL+"/test-bucket-10p", 0, nil) + c.Assert(err, IsNil) + + client := http.Client{} + response, err := client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusOK) + + var buffer bytes.Buffer + line := "1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,123" + // Create 10MiB content where each line contains 1024 characters. + for i := 0; i < 10*1024; i++ { + buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line)) + } + putContent := buffer.String() + + // Put object + buf := bytes.NewReader([]byte(putContent)) + request, err = s.newRequest("PUT", testAPIXLServer.URL+"/test-bucket-10p/big-file-10", int64(buf.Len()), buf) + c.Assert(err, IsNil) + + client = http.Client{} + response, err = client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusOK) + + // Get object + request, err = s.newRequest("GET", testAPIXLServer.URL+"/test-bucket-10p/big-file-10", 0, nil) + c.Assert(err, IsNil) + request.Header.Add("Range", "bytes=2048-2058") + + client = http.Client{} + response, err = client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusPartialContent) + getContent, err := ioutil.ReadAll(response.Body) + c.Assert(err, IsNil) + + // Compare putContent and getContent + c.Assert(string(getContent), Equals, putContent[2048:2059]) +} + func (s *MyAPIXLSuite) TestGetObjectErrors(c *C) { request, err := s.newRequest("GET", testAPIXLServer.URL+"/getobjecterrors", 0, nil) c.Assert(err, IsNil)