From 8559e894946c4628123da25688e612552e105ee4 Mon Sep 17 00:00:00 2001 From: Bala FA Date: Tue, 21 Jun 2016 02:53:25 +0530 Subject: [PATCH] XL: fix getBlockInfo() to return correct end block (#1941) If requested offset/length of an object is equal to erasureInfo.BlockSize, getBlockInfo() returns one more block added to actual end block. This patch fixes the issue. This patch also adds unit test for get objects with big files. --- erasure-utils.go | 3 ++ server_test.go | 130 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/erasure-utils.go b/erasure-utils.go index dac2e3dc4..c4dcf8a5e 100644 --- a/erasure-utils.go +++ b/erasure-utils.go @@ -96,6 +96,9 @@ func getBlockInfo(offset, length, blockSize int64) (startBlock, bytesToSkip, end // Calculate end block for given size to read endBlock = (offset + length) / blockSize + if endBlock > 0 && (offset+length)%blockSize == 0 { + endBlock-- + } return } diff --git a/server_test.go b/server_test.go index ce81d4c1d..8fb984cbb 100644 --- a/server_test.go +++ b/server_test.go @@ -1158,6 +1158,136 @@ func (s *MyAPISuite) TestGetObjectRangeErrors(c *C) { verifyError(c, response, "InvalidRange", "The requested range cannot be satisfied.", http.StatusRequestedRangeNotSatisfiable) } +func (s *MyAPISuite) TestGetObjectLarge10MiB(c *C) { + // Make bucket for this test. + request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/test-bucket-10", 0, nil) + c.Assert(err, IsNil) + + client := http.Client{} + response, err := client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusOK) + + var buffer bytes.Buffer + line := "1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,123" + // Create 10MiB content where each line contains 1024 characters. + for i := 0; i < 10*1024; i++ { + buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line)) + } + putContent := buffer.String() + + // Put object + buf := bytes.NewReader([]byte(putContent)) + request, err = s.newRequest("PUT", testAPIFSCacheServer.URL+"/test-bucket-10/big-file-10", int64(buf.Len()), buf) + c.Assert(err, IsNil) + + client = http.Client{} + response, err = client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusOK) + + // Get object + request, err = s.newRequest("GET", testAPIFSCacheServer.URL+"/test-bucket-10/big-file-10", 0, nil) + c.Assert(err, IsNil) + + client = http.Client{} + response, err = client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusOK) + getContent, err := ioutil.ReadAll(response.Body) + c.Assert(err, IsNil) + + // Compare putContent and getContent + c.Assert(string(getContent), Equals, putContent) +} + +func (s *MyAPISuite) TestGetObjectLarge11MiB(c *C) { + // Make bucket for this test. + request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/test-bucket-11", 0, nil) + c.Assert(err, IsNil) + + client := http.Client{} + response, err := client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusOK) + + var buffer bytes.Buffer + line := "1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,123" + // Create 11MiB content where each line contains 1024 characters. + for i := 0; i < 11*1024; i++ { + buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line)) + } + putContent := buffer.String() + + // Put object + buf := bytes.NewReader([]byte(putContent)) + request, err = s.newRequest("PUT", testAPIFSCacheServer.URL+"/test-bucket-11/big-file-11", int64(buf.Len()), buf) + c.Assert(err, IsNil) + + client = http.Client{} + response, err = client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusOK) + + // Get object + request, err = s.newRequest("GET", testAPIFSCacheServer.URL+"/test-bucket-11/big-file-11", 0, nil) + c.Assert(err, IsNil) + + client = http.Client{} + response, err = client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusOK) + getContent, err := ioutil.ReadAll(response.Body) + c.Assert(err, IsNil) + + // Compare putContent and getContent + c.Assert(string(getContent), Equals, putContent) +} + +func (s *MyAPISuite) TestGetPartialObjectLarge10MiB(c *C) { + // Make bucket for this test. + request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/test-bucket-10p", 0, nil) + c.Assert(err, IsNil) + + client := http.Client{} + response, err := client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusOK) + + var buffer bytes.Buffer + line := "1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,123" + // Create 10MiB content where each line contains 1024 characters. + for i := 0; i < 10*1024; i++ { + buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line)) + } + putContent := buffer.String() + + // Put object + buf := bytes.NewReader([]byte(putContent)) + request, err = s.newRequest("PUT", testAPIFSCacheServer.URL+"/test-bucket-10p/big-file-10", int64(buf.Len()), buf) + c.Assert(err, IsNil) + + client = http.Client{} + response, err = client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusOK) + + // Get object + request, err = s.newRequest("GET", testAPIFSCacheServer.URL+"/test-bucket-10p/big-file-10", 0, nil) + c.Assert(err, IsNil) + request.Header.Add("Range", "bytes=2048-2058") + + client = http.Client{} + response, err = client.Do(request) + c.Assert(err, IsNil) + c.Assert(response.StatusCode, Equals, http.StatusPartialContent) + getContent, err := ioutil.ReadAll(response.Body) + c.Assert(err, IsNil) + + // Compare putContent and getContent + c.Assert(string(getContent), Equals, putContent[2048:2059]) +} + func (s *MyAPISuite) TestObjectMultipartAbort(c *C) { request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/objectmultipartabort", 0, nil) c.Assert(err, IsNil)