Inline small file data in xl.meta file (#11758)

cmd/xl-storage-format-v2_test.go

@@ -0,0 +1,144 @@
+/*
+ * MinIO Cloud Storage, (C) 2021 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"testing"
+	"time"
+)
+
+func TestXLV2FormatData(t *testing.T) {
+	failOnErr := func(err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	data := []byte("some object data")
+	data2 := []byte("some other object data")
+
+	xl := xlMetaV2{}
+	fi := FileInfo{
+		Volume:           "volume",
+		Name:             "object-name",
+		VersionID:        "756100c6-b393-4981-928a-d49bbc164741",
+		IsLatest:         true,
+		Deleted:          false,
+		TransitionStatus: "",
+		DataDir:          "bffea160-ca7f-465f-98bc-9b4f1c3ba1ef",
+		XLV1:             false,
+		ModTime:          time.Now(),
+		Size:             0,
+		Mode:             0,
+		Metadata:         nil,
+		Parts:            nil,
+		Erasure: ErasureInfo{
+			Algorithm:    ReedSolomon.String(),
+			DataBlocks:   4,
+			ParityBlocks: 2,
+			BlockSize:    10000,
+			Index:        1,
+			Distribution: []int{1, 2, 3, 4, 5, 6, 7, 8},
+			Checksums: []ChecksumInfo{{
+				PartNumber: 1,
+				Algorithm:  HighwayHash256S,
+				Hash:       nil,
+			}},
+		},
+		MarkDeleted:                   false,
+		DeleteMarkerReplicationStatus: "",
+		VersionPurgeStatus:            "",
+		Data:                          data,
+		NumVersions:                   1,
+		SuccessorModTime:              time.Time{},
+	}
+
+	failOnErr(xl.AddVersion(fi))
+
+	fi.VersionID = mustGetUUID()
+	fi.DataDir = mustGetUUID()
+	fi.Data = data2
+	failOnErr(xl.AddVersion(fi))
+
+	serialized, err := xl.AppendTo(nil)
+	failOnErr(err)
+	// Roundtrip data
+	var xl2 xlMetaV2
+	failOnErr(xl2.Load(serialized))
+
+	// We should have one data entry
+	list, err := xl2.data.list()
+	failOnErr(err)
+	if len(list) != 2 {
+		t.Fatalf("want 1 entry, got %d", len(list))
+	}
+
+	if !bytes.Equal(xl2.data.find("bffea160-ca7f-465f-98bc-9b4f1c3ba1ef"), data) {
+		t.Fatal("Find data returned", xl2.data.find("bffea160-ca7f-465f-98bc-9b4f1c3ba1ef"))
+	}
+	if !bytes.Equal(xl2.data.find(fi.DataDir), data2) {
+		t.Fatal("Find data returned", xl2.data.find(fi.DataDir))
+	}
+
+	// Remove entry
+	xl2.data.remove(fi.DataDir)
+	failOnErr(xl2.data.validate())
+	if xl2.data.find(fi.DataDir) != nil {
+		t.Fatal("Data was not removed:", xl2.data.find(fi.DataDir))
+	}
+	if xl2.data.entries() != 1 {
+		t.Fatal("want 1 entry, got", xl2.data.entries())
+	}
+	// Re-add
+	xl2.data.replace(fi.DataDir, fi.Data)
+	failOnErr(xl2.data.validate())
+	if xl2.data.entries() != 2 {
+		t.Fatal("want 2 entries, got", xl2.data.entries())
+	}
+
+	// Replace entry
+	xl2.data.replace("bffea160-ca7f-465f-98bc-9b4f1c3ba1ef", data2)
+	failOnErr(xl2.data.validate())
+	if xl2.data.entries() != 2 {
+		t.Fatal("want 2 entries, got", xl2.data.entries())
+	}
+	if !bytes.Equal(xl2.data.find("bffea160-ca7f-465f-98bc-9b4f1c3ba1ef"), data2) {
+		t.Fatal("Find data returned", xl2.data.find("bffea160-ca7f-465f-98bc-9b4f1c3ba1ef"))
+	}
+
+	if !xl2.data.rename("bffea160-ca7f-465f-98bc-9b4f1c3ba1ef", "new-key") {
+		t.Fatal("old key was not found")
+	}
+	failOnErr(xl2.data.validate())
+	if !bytes.Equal(xl2.data.find("new-key"), data2) {
+		t.Fatal("Find data returned", xl2.data.find("bffea160-ca7f-465f-98bc-9b4f1c3ba1ef"))
+	}
+	if xl2.data.entries() != 2 {
+		t.Fatal("want 2 entries, got", xl2.data.entries())
+	}
+	if !bytes.Equal(xl2.data.find(fi.DataDir), data2) {
+		t.Fatal("Find data returned", xl2.data.find(fi.DataDir))
+	}
+
+	// Test trimmed
+	xl2 = xlMetaV2{}
+	failOnErr(xl2.Load(xlMetaV2TrimData(serialized)))
+	if len(xl2.data) != 0 {
+		t.Fatal("data, was not trimmed, bytes left:", len(xl2.data))
+	}
+}