mirror of
https://github.com/minio/minio.git
synced 2024-12-26 23:25:54 -05:00
b55922effe
Optimizing List Objects by using binary sort to discard entries in cases where prefix or marker is set. Adding test coverage to ListObjects. Adding benchmark to ListObjects.
457 lines
16 KiB
Go
457 lines
16 KiB
Go
/*
|
|
* Minimalist Object Storage, (C) 2015, 2016 Minio, Inc.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package fs
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/md5"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"encoding/xml"
|
|
"math/rand"
|
|
"strconv"
|
|
|
|
"gopkg.in/check.v1"
|
|
)
|
|
|
|
// APITestSuite - collection of API tests
|
|
func APITestSuite(c *check.C, create func() Filesystem) {
|
|
testMakeBucket(c, create)
|
|
testMultipleObjectCreation(c, create)
|
|
testPaging(c, create)
|
|
testObjectOverwriteWorks(c, create)
|
|
testNonExistantBucketOperations(c, create)
|
|
testBucketRecreateFails(c, create)
|
|
testPutObjectInSubdir(c, create)
|
|
testListBuckets(c, create)
|
|
testListBucketsOrder(c, create)
|
|
testListObjectsTestsForNonExistantBucket(c, create)
|
|
testNonExistantObjectInBucket(c, create)
|
|
testGetDirectoryReturnsObjectNotFound(c, create)
|
|
testDefaultContentType(c, create)
|
|
testMultipartObjectCreation(c, create)
|
|
testMultipartObjectAbort(c, create)
|
|
}
|
|
|
|
func testMakeBucket(c *check.C, create func() Filesystem) {
|
|
fs := create()
|
|
err := fs.MakeBucket("bucket")
|
|
c.Assert(err, check.IsNil)
|
|
}
|
|
|
|
func testMultipartObjectCreation(c *check.C, create func() Filesystem) {
|
|
fs := create()
|
|
err := fs.MakeBucket("bucket")
|
|
c.Assert(err, check.IsNil)
|
|
uploadID, err := fs.NewMultipartUpload("bucket", "key")
|
|
c.Assert(err, check.IsNil)
|
|
|
|
completedParts := CompleteMultipartUpload{}
|
|
completedParts.Part = make([]CompletePart, 0)
|
|
for i := 1; i <= 10; i++ {
|
|
randomPerm := rand.Perm(10)
|
|
randomString := ""
|
|
for _, num := range randomPerm {
|
|
randomString = randomString + strconv.Itoa(num)
|
|
}
|
|
|
|
hasher := md5.New()
|
|
hasher.Write([]byte(randomString))
|
|
expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
|
|
expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
|
|
|
|
var calculatedmd5sum string
|
|
calculatedmd5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, expectedmd5Sum, i, int64(len(randomString)),
|
|
bytes.NewBufferString(randomString), nil)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(calculatedmd5sum, check.Equals, expectedmd5Sumhex)
|
|
completedParts.Part = append(completedParts.Part, CompletePart{PartNumber: i, ETag: calculatedmd5sum})
|
|
}
|
|
completedPartsBytes, e := xml.Marshal(completedParts)
|
|
c.Assert(e, check.IsNil)
|
|
objectInfo, err := fs.CompleteMultipartUpload("bucket", "key", uploadID, bytes.NewReader(completedPartsBytes), nil)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(objectInfo.MD5Sum, check.Equals, "9b7d6f13ba00e24d0b02de92e814891b-10")
|
|
}
|
|
|
|
func testMultipartObjectAbort(c *check.C, create func() Filesystem) {
|
|
fs := create()
|
|
err := fs.MakeBucket("bucket")
|
|
c.Assert(err, check.IsNil)
|
|
uploadID, err := fs.NewMultipartUpload("bucket", "key")
|
|
c.Assert(err, check.IsNil)
|
|
|
|
parts := make(map[int]string)
|
|
for i := 1; i <= 10; i++ {
|
|
randomPerm := rand.Perm(10)
|
|
randomString := ""
|
|
for _, num := range randomPerm {
|
|
randomString = randomString + strconv.Itoa(num)
|
|
}
|
|
|
|
hasher := md5.New()
|
|
hasher.Write([]byte(randomString))
|
|
expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
|
|
expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
|
|
|
|
var calculatedmd5sum string
|
|
calculatedmd5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, expectedmd5Sum, i, int64(len(randomString)),
|
|
bytes.NewBufferString(randomString), nil)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(calculatedmd5sum, check.Equals, expectedmd5Sumhex)
|
|
parts[i] = calculatedmd5sum
|
|
}
|
|
err = fs.AbortMultipartUpload("bucket", "key", uploadID)
|
|
c.Assert(err, check.IsNil)
|
|
}
|
|
|
|
func testMultipleObjectCreation(c *check.C, create func() Filesystem) {
|
|
objects := make(map[string][]byte)
|
|
fs := create()
|
|
err := fs.MakeBucket("bucket")
|
|
c.Assert(err, check.IsNil)
|
|
for i := 0; i < 10; i++ {
|
|
randomPerm := rand.Perm(10)
|
|
randomString := ""
|
|
for _, num := range randomPerm {
|
|
randomString = randomString + strconv.Itoa(num)
|
|
}
|
|
|
|
hasher := md5.New()
|
|
hasher.Write([]byte(randomString))
|
|
expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
|
|
expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
|
|
|
|
key := "obj" + strconv.Itoa(i)
|
|
objects[key] = []byte(randomString)
|
|
objectInfo, err := fs.CreateObject("bucket", key, expectedmd5Sum, int64(len(randomString)), bytes.NewBufferString(randomString), nil)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(objectInfo.MD5Sum, check.Equals, expectedmd5Sumhex)
|
|
}
|
|
|
|
for key, value := range objects {
|
|
var byteBuffer bytes.Buffer
|
|
_, err := fs.GetObject(&byteBuffer, "bucket", key, 0, 0)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(byteBuffer.Bytes(), check.DeepEquals, value)
|
|
|
|
metadata, err := fs.GetObjectInfo("bucket", key)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(metadata.Size, check.Equals, int64(len(value)))
|
|
}
|
|
}
|
|
|
|
func testPaging(c *check.C, create func() Filesystem) {
|
|
fs := create()
|
|
fs.MakeBucket("bucket")
|
|
result, err := fs.ListObjects("bucket", "", "", "", 0)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(len(result.Objects), check.Equals, 0)
|
|
c.Assert(result.IsTruncated, check.Equals, false)
|
|
// check before paging occurs
|
|
for i := 0; i < 5; i++ {
|
|
key := "obj" + strconv.Itoa(i)
|
|
_, err = fs.CreateObject("bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil)
|
|
c.Assert(err, check.IsNil)
|
|
result, err = fs.ListObjects("bucket", "", "", "", 5)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(len(result.Objects), check.Equals, i+1)
|
|
c.Assert(result.IsTruncated, check.Equals, false)
|
|
}
|
|
// check after paging occurs pages work
|
|
for i := 6; i <= 10; i++ {
|
|
key := "obj" + strconv.Itoa(i)
|
|
_, err = fs.CreateObject("bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil)
|
|
c.Assert(err, check.IsNil)
|
|
result, err = fs.ListObjects("bucket", "obj", "", "", 5)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(len(result.Objects), check.Equals, 5)
|
|
c.Assert(result.IsTruncated, check.Equals, true)
|
|
}
|
|
// check paging with prefix at end returns less objects
|
|
{
|
|
_, err = fs.CreateObject("bucket", "newPrefix", "", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
|
|
c.Assert(err, check.IsNil)
|
|
_, err = fs.CreateObject("bucket", "newPrefix2", "", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
|
|
c.Assert(err, check.IsNil)
|
|
result, err = fs.ListObjects("bucket", "new", "", "", 5)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(len(result.Objects), check.Equals, 2)
|
|
}
|
|
|
|
// check ordering of pages
|
|
{
|
|
result, err = fs.ListObjects("bucket", "", "", "", 1000)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(result.Objects[0].Name, check.Equals, "newPrefix")
|
|
c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2")
|
|
c.Assert(result.Objects[2].Name, check.Equals, "obj0")
|
|
c.Assert(result.Objects[3].Name, check.Equals, "obj1")
|
|
c.Assert(result.Objects[4].Name, check.Equals, "obj10")
|
|
}
|
|
|
|
// check delimited results with delimiter and prefix
|
|
{
|
|
_, err = fs.CreateObject("bucket", "this/is/delimited", "", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
|
|
c.Assert(err, check.IsNil)
|
|
_, err = fs.CreateObject("bucket", "this/is/also/a/delimited/file", "", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
|
|
c.Assert(err, check.IsNil)
|
|
result, err = fs.ListObjects("bucket", "this/is/", "", "/", 10)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(len(result.Objects), check.Equals, 1)
|
|
c.Assert(result.Prefixes[0], check.Equals, "this/is/also/")
|
|
}
|
|
|
|
// check delimited results with delimiter without prefix
|
|
{
|
|
result, err = fs.ListObjects("bucket", "", "", "/", 1000)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(result.Objects[0].Name, check.Equals, "newPrefix")
|
|
c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2")
|
|
c.Assert(result.Objects[2].Name, check.Equals, "obj0")
|
|
c.Assert(result.Objects[3].Name, check.Equals, "obj1")
|
|
c.Assert(result.Objects[4].Name, check.Equals, "obj10")
|
|
c.Assert(result.Prefixes[0], check.Equals, "this/")
|
|
}
|
|
|
|
// check results with Marker
|
|
{
|
|
result, err = fs.ListObjects("bucket", "", "newPrefix", "", 3)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(result.Objects[0].Name, check.Equals, "newPrefix2")
|
|
c.Assert(result.Objects[1].Name, check.Equals, "obj0")
|
|
c.Assert(result.Objects[2].Name, check.Equals, "obj1")
|
|
}
|
|
// check ordering of results with prefix
|
|
{
|
|
result, err = fs.ListObjects("bucket", "obj", "", "", 1000)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(result.Objects[0].Name, check.Equals, "obj0")
|
|
c.Assert(result.Objects[1].Name, check.Equals, "obj1")
|
|
c.Assert(result.Objects[2].Name, check.Equals, "obj10")
|
|
c.Assert(result.Objects[3].Name, check.Equals, "obj2")
|
|
c.Assert(result.Objects[4].Name, check.Equals, "obj3")
|
|
}
|
|
// check ordering of results with prefix and no paging
|
|
{
|
|
result, err = fs.ListObjects("bucket", "new", "", "", 5)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(result.Objects[0].Name, check.Equals, "newPrefix")
|
|
c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2")
|
|
}
|
|
}
|
|
|
|
func testObjectOverwriteWorks(c *check.C, create func() Filesystem) {
|
|
fs := create()
|
|
err := fs.MakeBucket("bucket")
|
|
c.Assert(err, check.IsNil)
|
|
|
|
hasher1 := md5.New()
|
|
hasher1.Write([]byte("one"))
|
|
md5Sum1 := base64.StdEncoding.EncodeToString(hasher1.Sum(nil))
|
|
md5Sum1hex := hex.EncodeToString(hasher1.Sum(nil))
|
|
objectInfo, err := fs.CreateObject("bucket", "object", md5Sum1, int64(len("one")), bytes.NewBufferString("one"), nil)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(md5Sum1hex, check.Equals, objectInfo.MD5Sum)
|
|
|
|
hasher2 := md5.New()
|
|
hasher2.Write([]byte("three"))
|
|
md5Sum2 := base64.StdEncoding.EncodeToString(hasher2.Sum(nil))
|
|
_, err = fs.CreateObject("bucket", "object", md5Sum2, int64(len("three")), bytes.NewBufferString("three"), nil)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
var bytesBuffer bytes.Buffer
|
|
length, err := fs.GetObject(&bytesBuffer, "bucket", "object", 0, 0)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(length, check.Equals, int64(len("three")))
|
|
c.Assert(string(bytesBuffer.Bytes()), check.Equals, "three")
|
|
}
|
|
|
|
func testNonExistantBucketOperations(c *check.C, create func() Filesystem) {
|
|
fs := create()
|
|
_, err := fs.CreateObject("bucket", "object", "", int64(len("one")), bytes.NewBufferString("one"), nil)
|
|
c.Assert(err, check.Not(check.IsNil))
|
|
}
|
|
|
|
func testBucketRecreateFails(c *check.C, create func() Filesystem) {
|
|
fs := create()
|
|
err := fs.MakeBucket("string")
|
|
c.Assert(err, check.IsNil)
|
|
err = fs.MakeBucket("string")
|
|
c.Assert(err, check.Not(check.IsNil))
|
|
}
|
|
|
|
func testPutObjectInSubdir(c *check.C, create func() Filesystem) {
|
|
fs := create()
|
|
err := fs.MakeBucket("bucket")
|
|
c.Assert(err, check.IsNil)
|
|
|
|
hasher := md5.New()
|
|
hasher.Write([]byte("hello world"))
|
|
md5Sum1 := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
|
|
md5Sum1hex := hex.EncodeToString(hasher.Sum(nil))
|
|
objectInfo, err := fs.CreateObject("bucket", "dir1/dir2/object", md5Sum1, int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(objectInfo.MD5Sum, check.Equals, md5Sum1hex)
|
|
|
|
var bytesBuffer bytes.Buffer
|
|
length, err := fs.GetObject(&bytesBuffer, "bucket", "dir1/dir2/object", 0, 0)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(len(bytesBuffer.Bytes()), check.Equals, len("hello world"))
|
|
c.Assert(int64(len(bytesBuffer.Bytes())), check.Equals, length)
|
|
}
|
|
|
|
func testListBuckets(c *check.C, create func() Filesystem) {
|
|
fs := create()
|
|
|
|
// test empty list
|
|
buckets, err := fs.ListBuckets()
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(len(buckets), check.Equals, 0)
|
|
|
|
// add one and test exists
|
|
err = fs.MakeBucket("bucket1")
|
|
c.Assert(err, check.IsNil)
|
|
|
|
buckets, err = fs.ListBuckets()
|
|
c.Assert(len(buckets), check.Equals, 1)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
// add two and test exists
|
|
err = fs.MakeBucket("bucket2")
|
|
c.Assert(err, check.IsNil)
|
|
|
|
buckets, err = fs.ListBuckets()
|
|
c.Assert(len(buckets), check.Equals, 2)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
// add three and test exists + prefix
|
|
err = fs.MakeBucket("bucket22")
|
|
|
|
buckets, err = fs.ListBuckets()
|
|
c.Assert(len(buckets), check.Equals, 3)
|
|
c.Assert(err, check.IsNil)
|
|
}
|
|
|
|
func testListBucketsOrder(c *check.C, create func() Filesystem) {
|
|
// if implementation contains a map, order of map keys will vary.
|
|
// this ensures they return in the same order each time
|
|
for i := 0; i < 10; i++ {
|
|
fs := create()
|
|
// add one and test exists
|
|
err := fs.MakeBucket("bucket1")
|
|
c.Assert(err, check.IsNil)
|
|
err = fs.MakeBucket("bucket2")
|
|
c.Assert(err, check.IsNil)
|
|
buckets, err := fs.ListBuckets()
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(len(buckets), check.Equals, 2)
|
|
c.Assert(buckets[0].Name, check.Equals, "bucket1")
|
|
c.Assert(buckets[1].Name, check.Equals, "bucket2")
|
|
}
|
|
}
|
|
|
|
func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Filesystem) {
|
|
fs := create()
|
|
result, err := fs.ListObjects("bucket", "", "", "", 1000)
|
|
c.Assert(err, check.Not(check.IsNil))
|
|
c.Assert(result.IsTruncated, check.Equals, false)
|
|
c.Assert(len(result.Objects), check.Equals, 0)
|
|
}
|
|
|
|
func testNonExistantObjectInBucket(c *check.C, create func() Filesystem) {
|
|
fs := create()
|
|
err := fs.MakeBucket("bucket")
|
|
c.Assert(err, check.IsNil)
|
|
|
|
var byteBuffer bytes.Buffer
|
|
length, err := fs.GetObject(&byteBuffer, "bucket", "dir1", 0, 0)
|
|
c.Assert(length, check.Equals, int64(0))
|
|
c.Assert(err, check.Not(check.IsNil))
|
|
c.Assert(len(byteBuffer.Bytes()), check.Equals, 0)
|
|
switch err := err.ToGoError().(type) {
|
|
case ObjectNotFound:
|
|
c.Assert(err, check.ErrorMatches, "Object not found: bucket#dir1")
|
|
default:
|
|
c.Assert(err, check.Equals, "fails")
|
|
}
|
|
}
|
|
|
|
func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Filesystem) {
|
|
fs := create()
|
|
err := fs.MakeBucket("bucket")
|
|
c.Assert(err, check.IsNil)
|
|
|
|
_, err = fs.CreateObject("bucket", "dir1/dir2/object", "", int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
var byteBuffer bytes.Buffer
|
|
length, err := fs.GetObject(&byteBuffer, "bucket", "dir1", 0, 0)
|
|
c.Assert(length, check.Equals, int64(0))
|
|
switch err := err.ToGoError().(type) {
|
|
case ObjectNotFound:
|
|
c.Assert(err.Bucket, check.Equals, "bucket")
|
|
c.Assert(err.Object, check.Equals, "dir1")
|
|
default:
|
|
// force a failure with a line number
|
|
c.Assert(err, check.Equals, "ObjectNotFound")
|
|
}
|
|
c.Assert(len(byteBuffer.Bytes()), check.Equals, 0)
|
|
|
|
var byteBuffer2 bytes.Buffer
|
|
length, err = fs.GetObject(&byteBuffer, "bucket", "dir1/", 0, 0)
|
|
c.Assert(length, check.Equals, int64(0))
|
|
switch err := err.ToGoError().(type) {
|
|
case ObjectNotFound:
|
|
c.Assert(err.Bucket, check.Equals, "bucket")
|
|
c.Assert(err.Object, check.Equals, "dir1/")
|
|
default:
|
|
// force a failure with a line number
|
|
c.Assert(err, check.Equals, "ObjectNotFound")
|
|
}
|
|
c.Assert(len(byteBuffer2.Bytes()), check.Equals, 0)
|
|
}
|
|
|
|
func testDefaultContentType(c *check.C, create func() Filesystem) {
|
|
fs := create()
|
|
err := fs.MakeBucket("bucket")
|
|
c.Assert(err, check.IsNil)
|
|
|
|
// test empty
|
|
_, err = fs.CreateObject("bucket", "one", "", int64(len("one")), bytes.NewBufferString("one"), nil)
|
|
metadata, err := fs.GetObjectInfo("bucket", "one")
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(metadata.ContentType, check.Equals, "application/octet-stream")
|
|
}
|
|
|
|
func testContentMD5Set(c *check.C, create func() Filesystem) {
|
|
fs := create()
|
|
err := fs.MakeBucket("bucket")
|
|
c.Assert(err, check.IsNil)
|
|
|
|
// test md5 invalid
|
|
badmd5Sum := "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA"
|
|
calculatedmd5sum, err := fs.CreateObject("bucket", "one", badmd5Sum, int64(len("one")), bytes.NewBufferString("one"), nil)
|
|
c.Assert(err, check.Not(check.IsNil))
|
|
c.Assert(calculatedmd5sum, check.Not(check.Equals), badmd5Sum)
|
|
|
|
goodmd5sum := "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA="
|
|
calculatedmd5sum, err = fs.CreateObject("bucket", "two", goodmd5sum, int64(len("one")), bytes.NewBufferString("one"), nil)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(calculatedmd5sum, check.Equals, goodmd5sum)
|
|
}
|