MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-09 05:34:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

Support bucket versioning (#9377)

Browse Source 
- Implement a new xl.json 2.0.0 format to support,
  this moves the entire marshaling logic to POSIX
  layer, top layer always consumes a common FileInfo
  construct which simplifies the metadata reads.
- Implement list object versions
- Migrate to siphash from crchash for new deployments
  for object placements.

Fixes #2111
This commit is contained in:
Harshavardhana
2020-06-12 20:04:01 -07:00
committed by GitHub
parent 43d6e3ae06
commit 4915433bd2
203 changed files with 13833 additions and 6919 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
cmd/object-api-multipart_test.go
View File

@@ -28,7 +28,7 @@ import (
"github.com/minio/minio/pkg/hash"
)
// Wrapper for calling NewMultipartUpload tests for both XL multiple disks and single node setup.
// Wrapper for calling NewMultipartUpload tests for both Erasure multiple disks and single node setup.
func TestObjectNewMultipartUpload(t *testing.T) {
ExecObjectLayerTest(t, testObjectNewMultipartUpload)
}
@@ -55,7 +55,7 @@ func testObjectNewMultipartUpload(obj ObjectLayer, instanceType string, t TestEr
}
// Create bucket before intiating NewMultipartUpload.
err = obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
err = obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
if err != nil {
// failed to create newbucket, abort.
t.Fatalf("%s : %s", instanceType, err.Error())
@@ -77,7 +77,7 @@ func testObjectNewMultipartUpload(obj ObjectLayer, instanceType string, t TestEr
}
}
// Wrapper for calling AbortMultipartUpload tests for both XL multiple disks and single node setup.
// Wrapper for calling AbortMultipartUpload tests for both Erasure multiple disks and single node setup.
func TestObjectAbortMultipartUpload(t *testing.T) {
ExecObjectLayerTest(t, testObjectAbortMultipartUpload)
}
@@ -89,7 +89,7 @@ func testObjectAbortMultipartUpload(obj ObjectLayer, instanceType string, t Test
object := "minio-object"
opts := ObjectOptions{}
// Create bucket before intiating NewMultipartUpload.
err := obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
if err != nil {
// failed to create newbucket, abort.
t.Fatalf("%s : %s", instanceType, err.Error())
@@ -124,7 +124,7 @@ func testObjectAbortMultipartUpload(obj ObjectLayer, instanceType string, t Test
}
}
// Wrapper for calling isUploadIDExists tests for both XL multiple disks and single node setup.
// Wrapper for calling isUploadIDExists tests for both Erasure multiple disks and single node setup.
func TestObjectAPIIsUploadIDExists(t *testing.T) {
ExecObjectLayerTest(t, testObjectAPIIsUploadIDExists)
}
@@ -135,7 +135,7 @@ func testObjectAPIIsUploadIDExists(obj ObjectLayer, instanceType string, t TestE
object := "minio-object"
// Create bucket before intiating NewMultipartUpload.
err := obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
if err != nil {
// Failed to create newbucket, abort.
t.Fatalf("%s : %s", instanceType, err.Error())
@@ -154,7 +154,7 @@ func testObjectAPIIsUploadIDExists(obj ObjectLayer, instanceType string, t TestE
}
}
// Wrapper for calling PutObjectPart tests for both XL multiple disks and single node setup.
// Wrapper for calling PutObjectPart tests for both Erasure multiple disks and single node setup.
func TestObjectAPIPutObjectPart(t *testing.T) {
ExecObjectLayerTest(t, testObjectAPIPutObjectPart)
}
@@ -166,7 +166,7 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH
object := "minio-object"
opts := ObjectOptions{}
// Create bucket before intiating NewMultipartUpload.
err := obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
if err != nil {
// Failed to create newbucket, abort.
t.Fatalf("%s : %s", instanceType, err.Error())
@@ -178,7 +178,7 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH
t.Fatalf("%s : %s", instanceType, err.Error())
}
// Creating a dummy bucket for tests.
err = obj.MakeBucketWithLocation(context.Background(), "unused-bucket", "", false)
err = obj.MakeBucketWithLocation(context.Background(), "unused-bucket", BucketOptions{})
if err != nil {
// Failed to create newbucket, abort.
t.Fatalf("%s : %s", instanceType, err.Error())
@@ -210,7 +210,7 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH
{"a", "obj", "", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Bucket not found: a")},
// Test case - 5.
// Case with invalid object names.
{bucket, "", "", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Object name invalid: minio-bucket#")},
{bucket, "", "", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Object name invalid: minio-bucket/")},
// Test case - 6.
// Valid object and bucket names but non-existent bucket.
{"abc", "def", "", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Bucket not found: abc")},
@@ -286,7 +286,7 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH
}
}
// Wrapper for calling TestListMultipartUploads tests for both XL multiple disks and single node setup.
// Wrapper for calling TestListMultipartUploads tests for both Erasure multiple disks and single node setup.
func TestListMultipartUploads(t *testing.T) {
ExecObjectLayerTest(t, testListMultipartUploads)
}
@@ -302,7 +302,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
// objectNames[0].
// uploadIds [0].
// Create bucket before initiating NewMultipartUpload.
err := obj.MakeBucketWithLocation(context.Background(), bucketNames[0], "", false)
err := obj.MakeBucketWithLocation(context.Background(), bucketNames[0], BucketOptions{})
if err != nil {
// Failed to create newbucket, abort.
t.Fatalf("%s : %s", instanceType, err.Error())
@@ -320,7 +320,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
// objectNames[0].
// uploadIds [1-3].
// Bucket to test for mutiple upload Id's for a given object.
err = obj.MakeBucketWithLocation(context.Background(), bucketNames[1], "", false)
err = obj.MakeBucketWithLocation(context.Background(), bucketNames[1], BucketOptions{})
if err != nil {
// Failed to create newbucket, abort.
t.Fatalf("%s : %s", instanceType, err.Error())
@@ -341,7 +341,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
// bucketnames[2].
// objectNames[0-2].
// uploadIds [4-9].
err = obj.MakeBucketWithLocation(context.Background(), bucketNames[2], "", false)
err = obj.MakeBucketWithLocation(context.Background(), bucketNames[2], BucketOptions{})
if err != nil {
// Failed to create newbucket, abort.
t.Fatalf("%s : %s", instanceType, err.Error())
@@ -1150,7 +1150,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
}
}
// Wrapper for calling TestListObjectPartsDiskNotFound tests for both XL multiple disks and single node setup.
// Wrapper for calling TestListObjectPartsDiskNotFound tests for both Erasure multiple disks and single node setup.
func TestListObjectPartsDiskNotFound(t *testing.T) {
ExecObjectLayerDiskAlteredTest(t, testListObjectPartsDiskNotFound)
}
@@ -1166,7 +1166,7 @@ func testListObjectPartsDiskNotFound(obj ObjectLayer, instanceType string, disks
// objectNames[0].
// uploadIds [0].
// Create bucket before intiating NewMultipartUpload.
err := obj.MakeBucketWithLocation(context.Background(), bucketNames[0], "", false)
err := obj.MakeBucketWithLocation(context.Background(), bucketNames[0], BucketOptions{})
if err != nil {
// Failed to create newbucket, abort.
t.Fatalf("%s : %s", instanceType, err.Error())
@@ -1395,7 +1395,7 @@ func testListObjectPartsDiskNotFound(obj ObjectLayer, instanceType string, disks
}
}
// Wrapper for calling TestListObjectParts tests for both XL multiple disks and single node setup.
// Wrapper for calling TestListObjectParts tests for both Erasure multiple disks and single node setup.
func TestListObjectParts(t *testing.T) {
ExecObjectLayerTest(t, testListObjectParts)
}
@@ -1411,7 +1411,7 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler)
// objectNames[0].
// uploadIds [0].
// Create bucket before intiating NewMultipartUpload.
err := obj.MakeBucketWithLocation(context.Background(), bucketNames[0], "", false)
err := obj.MakeBucketWithLocation(context.Background(), bucketNames[0], BucketOptions{})
if err != nil {
// Failed to create newbucket, abort.
t.Fatalf("%s : %s", instanceType, err.Error())
@@ -1657,7 +1657,7 @@ func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t T
// objectNames[0].
// uploadIds [0].
// Create bucket before intiating NewMultipartUpload.
err = obj.MakeBucketWithLocation(context.Background(), bucketNames[0], "", false)
err = obj.MakeBucketWithLocation(context.Background(), bucketNames[0], BucketOptions{})
if err != nil {
// Failed to create newbucket, abort.
t.Fatalf("%s : %s", instanceType, err)
@@ -1796,7 +1796,6 @@ func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t T
// the case above successfully completes CompleteMultipartUpload, the remaining Parts will be flushed.
// Expecting to fail with Invalid UploadID.
{bucketNames[0], objectNames[0], uploadIDs[0], inputParts[4].parts, "", InvalidUploadID{UploadID: uploadIDs[0]}, false},
// Expecting to fail due to bad
}
for _, testCase := range testCases {
@@ -1829,16 +1828,16 @@ func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t T
// Benchmarks for ObjectLayer.PutObjectPart().
// The intent is to benchmark PutObjectPart for various sizes ranging from few bytes to 100MB.
// Also each of these Benchmarks are run both XL and FS backends.
// Also each of these Benchmarks are run both Erasure and FS backends.
// BenchmarkPutObjectPart5MbFS - Benchmark FS.PutObjectPart() for object size of 5MB.
func BenchmarkPutObjectPart5MbFS(b *testing.B) {
benchmarkPutObjectPart(b, "FS", 5*humanize.MiByte)
}
// BenchmarkPutObjectPart5MbXL - Benchmark XL.PutObjectPart() for object size of 5MB.
func BenchmarkPutObjectPart5MbXL(b *testing.B) {
benchmarkPutObjectPart(b, "XL", 5*humanize.MiByte)
// BenchmarkPutObjectPart5MbErasure - Benchmark Erasure.PutObjectPart() for object size of 5MB.
func BenchmarkPutObjectPart5MbErasure(b *testing.B) {
benchmarkPutObjectPart(b, "Erasure", 5*humanize.MiByte)
}
// BenchmarkPutObjectPart10MbFS - Benchmark FS.PutObjectPart() for object size of 10MB.
@@ -1846,9 +1845,9 @@ func BenchmarkPutObjectPart10MbFS(b *testing.B) {
benchmarkPutObjectPart(b, "FS", 10*humanize.MiByte)
}
// BenchmarkPutObjectPart10MbXL - Benchmark XL.PutObjectPart() for object size of 10MB.
func BenchmarkPutObjectPart10MbXL(b *testing.B) {
benchmarkPutObjectPart(b, "XL", 10*humanize.MiByte)
// BenchmarkPutObjectPart10MbErasure - Benchmark Erasure.PutObjectPart() for object size of 10MB.
func BenchmarkPutObjectPart10MbErasure(b *testing.B) {
benchmarkPutObjectPart(b, "Erasure", 10*humanize.MiByte)
}
// BenchmarkPutObjectPart25MbFS - Benchmark FS.PutObjectPart() for object size of 25MB.
@@ -1857,9 +1856,9 @@ func BenchmarkPutObjectPart25MbFS(b *testing.B) {
}
// BenchmarkPutObjectPart25MbXL - Benchmark XL.PutObjectPart() for object size of 25MB.
func BenchmarkPutObjectPart25MbXL(b *testing.B) {
benchmarkPutObjectPart(b, "XL", 25*humanize.MiByte)
// BenchmarkPutObjectPart25MbErasure - Benchmark Erasure.PutObjectPart() for object size of 25MB.
func BenchmarkPutObjectPart25MbErasure(b *testing.B) {
benchmarkPutObjectPart(b, "Erasure", 25*humanize.MiByte)
}
// BenchmarkPutObjectPart50MbFS - Benchmark FS.PutObjectPart() for object size of 50MB.
@@ -1867,7 +1866,7 @@ func BenchmarkPutObjectPart50MbFS(b *testing.B) {
benchmarkPutObjectPart(b, "FS", 50*humanize.MiByte)
}
// BenchmarkPutObjectPart50MbXL - Benchmark XL.PutObjectPart() for object size of 50MB.
func BenchmarkPutObjectPart50MbXL(b *testing.B) {
benchmarkPutObjectPart(b, "XL", 50*humanize.MiByte)
// BenchmarkPutObjectPart50MbErasure - Benchmark Erasure.PutObjectPart() for object size of 50MB.
func BenchmarkPutObjectPart50MbErasure(b *testing.B) {
benchmarkPutObjectPart(b, "Erasure", 50*humanize.MiByte)
}