Support bucket versioning (#9377)

- Implement a new xl.json 2.0.0 format to support,
  this moves the entire marshaling logic to POSIX
  layer, top layer always consumes a common FileInfo
  construct which simplifies the metadata reads.
- Implement list object versions
- Migrate to siphash from crchash for new deployments
  for object placements.

Fixes #2111

cmd/erasure-metadata_test.go

@@ -0,0 +1,153 @@
+/*
+ * MinIO Cloud Storage, (C) 2015, 2016, 2017 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"strconv"
+	"testing"
+
+	humanize "github.com/dustin/go-humanize"
+)
+
+const ActualSize = 1000
+
+// Test FileInfo.AddObjectPart()
+func TestAddObjectPart(t *testing.T) {
+	testCases := []struct {
+		partNum       int
+		expectedIndex int
+	}{
+		{1, 0},
+		{2, 1},
+		{4, 2},
+		{5, 3},
+		{7, 4},
+		// Insert part.
+		{3, 2},
+		// Replace existing part.
+		{4, 3},
+		// Missing part.
+		{6, -1},
+	}
+
+	// Setup.
+	fi := newFileInfo("test-object", 8, 8)
+	if !fi.IsValid() {
+		t.Fatalf("unable to get xl meta")
+	}
+
+	// Test them.
+	for _, testCase := range testCases {
+		if testCase.expectedIndex > -1 {
+			partNumString := strconv.Itoa(testCase.partNum)
+			fi.AddObjectPart(testCase.partNum, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize)
+		}
+
+		if index := objectPartIndex(fi.Parts, testCase.partNum); index != testCase.expectedIndex {
+			t.Fatalf("%+v: expected = %d, got: %d", testCase, testCase.expectedIndex, index)
+		}
+	}
+}
+
+// Test objectPartIndex(). generates a sample FileInfo data and asserts
+// the output of objectPartIndex() with the expected value.
+func TestObjectPartIndex(t *testing.T) {
+	testCases := []struct {
+		partNum       int
+		expectedIndex int
+	}{
+		{2, 1},
+		{1, 0},
+		{5, 3},
+		{4, 2},
+		{7, 4},
+	}
+
+	// Setup.
+	fi := newFileInfo("test-object", 8, 8)
+	if !fi.IsValid() {
+		t.Fatalf("unable to get xl meta")
+	}
+
+	// Add some parts for testing.
+	for _, testCase := range testCases {
+		partNumString := strconv.Itoa(testCase.partNum)
+		fi.AddObjectPart(testCase.partNum, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize)
+	}
+
+	// Add failure test case.
+	testCases = append(testCases, struct {
+		partNum       int
+		expectedIndex int
+	}{6, -1})
+
+	// Test them.
+	for _, testCase := range testCases {
+		if index := objectPartIndex(fi.Parts, testCase.partNum); index != testCase.expectedIndex {
+			t.Fatalf("%+v: expected = %d, got: %d", testCase, testCase.expectedIndex, index)
+		}
+	}
+}
+
+// Test FileInfo.ObjectToPartOffset().
+func TestObjectToPartOffset(t *testing.T) {
+	// Setup.
+	fi := newFileInfo("test-object", 8, 8)
+	if !fi.IsValid() {
+		t.Fatalf("unable to get xl meta")
+	}
+
+	// Add some parts for testing.
+	// Total size of all parts is 5,242,899 bytes.
+	for _, partNum := range []int{1, 2, 4, 5, 7} {
+		partNumString := strconv.Itoa(partNum)
+		fi.AddObjectPart(partNum, "etag."+partNumString, int64(partNum+humanize.MiByte), ActualSize)
+	}
+
+	testCases := []struct {
+		offset         int64
+		expectedIndex  int
+		expectedOffset int64
+		expectedErr    error
+	}{
+		{0, 0, 0, nil},
+		{1 * humanize.MiByte, 0, 1 * humanize.MiByte, nil},
+		{1 + humanize.MiByte, 1, 0, nil},
+		{2 + humanize.MiByte, 1, 1, nil},
+		// Its valid for zero sized object.
+		{-1, 0, -1, nil},
+		// Max fffset is always (size - 1).
+		{(1 + 2 + 4 + 5 + 7) + (5 * humanize.MiByte) - 1, 4, 1048582, nil},
+		// Error if offset is size.
+		{(1 + 2 + 4 + 5 + 7) + (5 * humanize.MiByte), 0, 0, InvalidRange{}},
+	}
+
+	// Test them.
+	for _, testCase := range testCases {
+		index, offset, err := fi.ObjectToPartOffset(context.Background(), testCase.offset)
+		if err != testCase.expectedErr {
+			t.Fatalf("%+v: expected = %s, got: %s", testCase, testCase.expectedErr, err)
+		}
+		if index != testCase.expectedIndex {
+			t.Fatalf("%+v: index: expected = %d, got: %d", testCase, testCase.expectedIndex, index)
+		}
+		if offset != testCase.expectedOffset {
+			t.Fatalf("%+v: offset: expected = %d, got: %d", testCase, testCase.expectedOffset, offset)
+		}
+	}
+}