Support bucket versioning (#9377)

- Implement a new xl.json 2.0.0 format to support,
  this moves the entire marshaling logic to POSIX
  layer, top layer always consumes a common FileInfo
  construct which simplifies the metadata reads.
- Implement list object versions
- Migrate to siphash from crchash for new deployments
  for object placements.

Fixes #2111

cmd/object_api_suite_test.go

@@ -70,20 +70,20 @@ func (r *testOneByteReadNoEOF) Read(p []byte) (n int, err error) {
 
 type ObjectLayerAPISuite struct{}
 
-// Wrapper for calling testMakeBucket for both XL and FS.
+// Wrapper for calling testMakeBucket for both Erasure and FS.
 func (s *ObjectLayerAPISuite) TestMakeBucket(t *testing.T) {
 	ExecObjectLayerTest(t, testMakeBucket)
 }
 
 // Tests validate bucket creation.
 func testMakeBucket(obj ObjectLayer, instanceType string, t TestErrHandler) {
-	err := obj.MakeBucketWithLocation(context.Background(), "bucket-unknown", "", false)
+	err := obj.MakeBucketWithLocation(context.Background(), "bucket-unknown", BucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
 }
 
-// Wrapper for calling testMultipartObjectCreation for both XL and FS.
+// Wrapper for calling testMultipartObjectCreation for both Erasure and FS.
 func (s *ObjectLayerAPISuite) TestMultipartObjectCreation(t *testing.T) {
 	ExecObjectLayerTest(t, testMultipartObjectCreation)
 }
@@ -91,7 +91,7 @@ func (s *ObjectLayerAPISuite) TestMultipartObjectCreation(t *testing.T) {
 // Tests validate creation of part files during Multipart operation.
 func testMultipartObjectCreation(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	var opts ObjectOptions
-	err := obj.MakeBucketWithLocation(context.Background(), "bucket", "", false)
+	err := obj.MakeBucketWithLocation(context.Background(), "bucket", BucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -127,7 +127,7 @@ func testMultipartObjectCreation(obj ObjectLayer, instanceType string, t TestErr
 	}
 }
 
-// Wrapper for calling testMultipartObjectAbort for both XL and FS.
+// Wrapper for calling testMultipartObjectAbort for both Erasure and FS.
 func (s *ObjectLayerAPISuite) TestMultipartObjectAbort(t *testing.T) {
 	ExecObjectLayerTest(t, testMultipartObjectAbort)
 }
@@ -135,7 +135,7 @@ func (s *ObjectLayerAPISuite) TestMultipartObjectAbort(t *testing.T) {
 // Tests validate abortion of Multipart operation.
 func testMultipartObjectAbort(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	var opts ObjectOptions
-	err := obj.MakeBucketWithLocation(context.Background(), "bucket", "", false)
+	err := obj.MakeBucketWithLocation(context.Background(), "bucket", BucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -172,7 +172,7 @@ func testMultipartObjectAbort(obj ObjectLayer, instanceType string, t TestErrHan
 	}
 }
 
-// Wrapper for calling testMultipleObjectCreation for both XL and FS.
+// Wrapper for calling testMultipleObjectCreation for both Erasure and FS.
 func (s *ObjectLayerAPISuite) TestMultipleObjectCreation(t *testing.T) {
 	ExecObjectLayerTest(t, testMultipleObjectCreation)
 }
@@ -181,7 +181,7 @@ func (s *ObjectLayerAPISuite) TestMultipleObjectCreation(t *testing.T) {
 func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	objects := make(map[string][]byte)
 	var opts ObjectOptions
-	err := obj.MakeBucketWithLocation(context.Background(), "bucket", "", false)
+	err := obj.MakeBucketWithLocation(context.Background(), "bucket", BucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -229,14 +229,14 @@ func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrH
 	}
 }
 
-// Wrapper for calling TestPaging for both XL and FS.
+// Wrapper for calling TestPaging for both Erasure and FS.
 func (s *ObjectLayerAPISuite) TestPaging(t *testing.T) {
 	ExecObjectLayerTest(t, testPaging)
 }
 
 // Tests validate creation of objects and the order of listing using various filters for ListObjects operation.
 func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) {
-	obj.MakeBucketWithLocation(context.Background(), "bucket", "", false)
+	obj.MakeBucketWithLocation(context.Background(), "bucket", BucketOptions{})
 	result, err := obj.ListObjects(context.Background(), "bucket", "", "", "", 0)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
@@ -433,14 +433,14 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 }
 
-// Wrapper for calling testObjectOverwriteWorks for both XL and FS.
+// Wrapper for calling testObjectOverwriteWorks for both Erasure and FS.
 func (s *ObjectLayerAPISuite) TestObjectOverwriteWorks(t *testing.T) {
 	ExecObjectLayerTest(t, testObjectOverwriteWorks)
 }
 
 // Tests validate overwriting of an existing object.
 func testObjectOverwriteWorks(obj ObjectLayer, instanceType string, t TestErrHandler) {
-	err := obj.MakeBucketWithLocation(context.Background(), "bucket", "", false)
+	err := obj.MakeBucketWithLocation(context.Background(), "bucket", BucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -470,7 +470,7 @@ func testObjectOverwriteWorks(obj ObjectLayer, instanceType string, t TestErrHan
 	}
 }
 
-// Wrapper for calling testNonExistantBucketOperations for both XL and FS.
+// Wrapper for calling testNonExistantBucketOperations for both Erasure and FS.
 func (s *ObjectLayerAPISuite) TestNonExistantBucketOperations(t *testing.T) {
 	ExecObjectLayerTest(t, testNonExistantBucketOperations)
 }
@@ -487,18 +487,18 @@ func testNonExistantBucketOperations(obj ObjectLayer, instanceType string, t Tes
 	}
 }
 
-// Wrapper for calling testBucketRecreateFails for both XL and FS.
+// Wrapper for calling testBucketRecreateFails for both Erasure and FS.
 func (s *ObjectLayerAPISuite) TestBucketRecreateFails(t *testing.T) {
 	ExecObjectLayerTest(t, testBucketRecreateFails)
 }
 
 // Tests validate that recreation of the bucket fails.
 func testBucketRecreateFails(obj ObjectLayer, instanceType string, t TestErrHandler) {
-	err := obj.MakeBucketWithLocation(context.Background(), "string", "", false)
+	err := obj.MakeBucketWithLocation(context.Background(), "string", BucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
-	err = obj.MakeBucketWithLocation(context.Background(), "string", "", false)
+	err = obj.MakeBucketWithLocation(context.Background(), "string", BucketOptions{})
 	if err == nil {
 		t.Fatalf("%s: Expected error but found nil.", instanceType)
 	}
@@ -508,7 +508,7 @@ func testBucketRecreateFails(obj ObjectLayer, instanceType string, t TestErrHand
 	}
 }
 
-// Wrapper for calling testPutObject for both XL and FS.
+// Wrapper for calling testPutObject for both Erasure and FS.
 func (s *ObjectLayerAPISuite) TestPutObject(t *testing.T) {
 	ExecObjectLayerTest(t, testPutObject)
 }
@@ -519,7 +519,7 @@ func testPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	length := int64(len(content))
 	readerEOF := newTestReaderEOF(content)
 	readerNoEOF := newTestReaderNoEOF(content)
-	err := obj.MakeBucketWithLocation(context.Background(), "bucket", "", false)
+	err := obj.MakeBucketWithLocation(context.Background(), "bucket", BucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -552,14 +552,14 @@ func testPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 }
 
-// Wrapper for calling testPutObjectInSubdir for both XL and FS.
+// Wrapper for calling testPutObjectInSubdir for both Erasure and FS.
 func (s *ObjectLayerAPISuite) TestPutObjectInSubdir(t *testing.T) {
 	ExecObjectLayerTest(t, testPutObjectInSubdir)
 }
 
 // Tests validate PutObject with subdirectory prefix.
 func testPutObjectInSubdir(obj ObjectLayer, instanceType string, t TestErrHandler) {
-	err := obj.MakeBucketWithLocation(context.Background(), "bucket", "", false)
+	err := obj.MakeBucketWithLocation(context.Background(), "bucket", BucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -584,7 +584,7 @@ func testPutObjectInSubdir(obj ObjectLayer, instanceType string, t TestErrHandle
 	}
 }
 
-// Wrapper for calling testListBuckets for both XL and FS.
+// Wrapper for calling testListBuckets for both Erasure and FS.
 func (s *ObjectLayerAPISuite) TestListBuckets(t *testing.T) {
 	ExecObjectLayerTest(t, testListBuckets)
 }
@@ -601,7 +601,7 @@ func testListBuckets(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 
 	// add one and test exists.
-	err = obj.MakeBucketWithLocation(context.Background(), "bucket1", "", false)
+	err = obj.MakeBucketWithLocation(context.Background(), "bucket1", BucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -615,7 +615,7 @@ func testListBuckets(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 
 	// add two and test exists.
-	err = obj.MakeBucketWithLocation(context.Background(), "bucket2", "", false)
+	err = obj.MakeBucketWithLocation(context.Background(), "bucket2", BucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -629,7 +629,7 @@ func testListBuckets(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 
 	// add three and test exists + prefix.
-	err = obj.MakeBucketWithLocation(context.Background(), "bucket22", "", false)
+	err = obj.MakeBucketWithLocation(context.Background(), "bucket22", BucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -643,7 +643,7 @@ func testListBuckets(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 }
 
-// Wrapper for calling testListBucketsOrder for both XL and FS.
+// Wrapper for calling testListBucketsOrder for both Erasure and FS.
 func (s *ObjectLayerAPISuite) TestListBucketsOrder(t *testing.T) {
 	ExecObjectLayerTest(t, testListBucketsOrder)
 }
@@ -653,11 +653,11 @@ func testListBucketsOrder(obj ObjectLayer, instanceType string, t TestErrHandler
 	// if implementation contains a map, order of map keys will vary.
 	// this ensures they return in the same order each time.
 	// add one and test exists.
-	err := obj.MakeBucketWithLocation(context.Background(), "bucket1", "", false)
+	err := obj.MakeBucketWithLocation(context.Background(), "bucket1", BucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
-	err = obj.MakeBucketWithLocation(context.Background(), "bucket2", "", false)
+	err = obj.MakeBucketWithLocation(context.Background(), "bucket2", BucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -677,7 +677,7 @@ func testListBucketsOrder(obj ObjectLayer, instanceType string, t TestErrHandler
 	}
 }
 
-// Wrapper for calling testListObjectsTestsForNonExistantBucket for both XL and FS.
+// Wrapper for calling testListObjectsTestsForNonExistantBucket for both Erasure and FS.
 func (s *ObjectLayerAPISuite) TestListObjectsTestsForNonExistantBucket(t *testing.T) {
 	ExecObjectLayerTest(t, testListObjectsTestsForNonExistantBucket)
 }
@@ -699,14 +699,14 @@ func testListObjectsTestsForNonExistantBucket(obj ObjectLayer, instanceType stri
 	}
 }
 
-// Wrapper for calling testNonExistantObjectInBucket for both XL and FS.
+// Wrapper for calling testNonExistantObjectInBucket for both Erasure and FS.
 func (s *ObjectLayerAPISuite) TestNonExistantObjectInBucket(t *testing.T) {
 	ExecObjectLayerTest(t, testNonExistantObjectInBucket)
 }
 
 // Tests validate that GetObject fails on a non-existent bucket as expected.
 func testNonExistantObjectInBucket(obj ObjectLayer, instanceType string, t TestErrHandler) {
-	err := obj.MakeBucketWithLocation(context.Background(), "bucket", "", false)
+	err := obj.MakeBucketWithLocation(context.Background(), "bucket", BucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -726,7 +726,7 @@ func testNonExistantObjectInBucket(obj ObjectLayer, instanceType string, t TestE
 	}
 }
 
-// Wrapper for calling testGetDirectoryReturnsObjectNotFound for both XL and FS.
+// Wrapper for calling testGetDirectoryReturnsObjectNotFound for both Erasure and FS.
 func (s *ObjectLayerAPISuite) TestGetDirectoryReturnsObjectNotFound(t *testing.T) {
 	ExecObjectLayerTest(t, testGetDirectoryReturnsObjectNotFound)
 }
@@ -734,7 +734,7 @@ func (s *ObjectLayerAPISuite) TestGetDirectoryReturnsObjectNotFound(t *testing.T
 // Tests validate that GetObject on an existing directory fails as expected.
 func testGetDirectoryReturnsObjectNotFound(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	bucketName := "bucket"
-	err := obj.MakeBucketWithLocation(context.Background(), bucketName, "", false)
+	err := obj.MakeBucketWithLocation(context.Background(), bucketName, BucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -769,14 +769,14 @@ func testGetDirectoryReturnsObjectNotFound(obj ObjectLayer, instanceType string,
 	}
 }
 
-// Wrapper for calling testContentType for both XL and FS.
+// Wrapper for calling testContentType for both Erasure and FS.
 func (s *ObjectLayerAPISuite) TestContentType(t *testing.T) {
 	ExecObjectLayerTest(t, testContentType)
 }
 
 // Test content-type.
 func testContentType(obj ObjectLayer, instanceType string, t TestErrHandler) {
-	err := obj.MakeBucketWithLocation(context.Background(), "bucket", "", false)
+	err := obj.MakeBucketWithLocation(context.Background(), "bucket", BucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}