/* * MinIO Cloud Storage, (C) 2015, 2016, 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package cmd import ( "bytes" "encoding/hex" "encoding/json" "reflect" "testing" humanize "github.com/dustin/go-humanize" ) // Tests caclculating disk count. func TestDiskCount(t *testing.T) { testCases := []struct { disks []StorageAPI diskCount int }{ // Test case - 1 { disks: []StorageAPI{&posix{}, &posix{}, &posix{}, &posix{}}, diskCount: 4, }, // Test case - 2 { disks: []StorageAPI{nil, &posix{}, &posix{}, &posix{}}, diskCount: 3, }, } for i, testCase := range testCases { cdiskCount := diskCount(testCase.disks) if cdiskCount != testCase.diskCount { t.Errorf("Test %d: Expected %d, got %d", i+1, testCase.diskCount, cdiskCount) } } } // Test for reduceErrs, reduceErr reduces collection // of errors into a single maximal error with in the list. func TestReduceErrs(t *testing.T) { // List all of all test cases to validate various cases of reduce errors. testCases := []struct { errs []error ignoredErrs []error err error }{ // Validate if have reduced properly. {[]error{ errDiskNotFound, errDiskNotFound, errDiskFull, }, []error{}, errXLReadQuorum}, // Validate if have no consensus. {[]error{ errDiskFull, errDiskNotFound, nil, nil, }, []error{}, errXLReadQuorum}, // Validate if have consensus and errors ignored. {[]error{ errVolumeNotFound, errVolumeNotFound, errVolumeNotFound, errVolumeNotFound, errVolumeNotFound, errDiskNotFound, errDiskNotFound, }, []error{errDiskNotFound}, errVolumeNotFound}, {[]error{}, []error{}, errXLReadQuorum}, {[]error{errFileNotFound, errFileNotFound, errFileNotFound, errFileNotFound, errFileNotFound, nil, nil, nil, nil, nil}, nil, nil}, } // Validates list of all the testcases for returning valid errors. for i, testCase := range testCases { gotErr := reduceReadQuorumErrs(GlobalContext, testCase.errs, testCase.ignoredErrs, 5) if gotErr != testCase.err { t.Errorf("Test %d : expected %s, got %s", i+1, testCase.err, gotErr) } gotNewErr := reduceWriteQuorumErrs(GlobalContext, testCase.errs, testCase.ignoredErrs, 6) if gotNewErr != errXLWriteQuorum { t.Errorf("Test %d : expected %s, got %s", i+1, errXLWriteQuorum, gotErr) } } } // TestHashOrder - test order of ints in array func TestHashOrder(t *testing.T) { testCases := []struct { objectName string hashedOrder []int }{ // cases which should pass the test. // passing in valid object name. {"object", []int{14, 15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}}, {"The Shining Script .pdf", []int{16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}}, {"Cost Benefit Analysis (2009-2010).pptx", []int{15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}}, {"117Gn8rfHL2ACARPAhaFd0AGzic9pUbIA/5OCn5A", []int{3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2}}, {"SHØRT", []int{11, 12, 13, 14, 15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}}, {"There are far too many object names, and far too few bucket names!", []int{15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}}, {"a/b/c/", []int{3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2}}, {"/a/b/c", []int{6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2, 3, 4, 5}}, {string([]byte{0xff, 0xfe, 0xfd}), []int{15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}}, } // Tests hashing order to be consistent. for i, testCase := range testCases { hashedOrder := hashOrder(testCase.objectName, 16) if !reflect.DeepEqual(testCase.hashedOrder, hashedOrder) { t.Errorf("Test case %d: Expected \"%v\" but failed \"%v\"", i+1, testCase.hashedOrder, hashedOrder) } } // Tests hashing order to fail for when order is '-1'. if hashedOrder := hashOrder("This will fail", -1); hashedOrder != nil { t.Errorf("Test: Expect \"nil\" but failed \"%#v\"", hashedOrder) } if hashedOrder := hashOrder("This will fail", 0); hashedOrder != nil { t.Errorf("Test: Expect \"nil\" but failed \"%#v\"", hashedOrder) } } // newTestXLMetaV1 - initializes new xlMetaV1, adds version, allocates a fresh erasure info and metadata. func newTestXLMetaV1() xlMetaV1 { xlMeta := xlMetaV1{} xlMeta.Version = xlMetaVersion xlMeta.Format = xlMetaFormat xlMeta.Minio.Release = "test" xlMeta.Erasure = ErasureInfo{ Algorithm: "klauspost/reedsolomon/vandermonde", DataBlocks: 5, ParityBlocks: 5, BlockSize: 10485760, Index: 10, Distribution: []int{9, 10, 1, 2, 3, 4, 5, 6, 7, 8}, } xlMeta.Stat = statInfo{ Size: int64(20), ModTime: UTCNow(), } // Set meta data. xlMeta.Meta = make(map[string]string) xlMeta.Meta["testKey1"] = "val1" xlMeta.Meta["testKey2"] = "val2" return xlMeta } func (m *xlMetaV1) AddTestObjectCheckSum(partNumber int, algorithm BitrotAlgorithm, hash string) { checksum, err := hex.DecodeString(hash) if err != nil { panic(err) } m.Erasure.Checksums[partNumber-1] = ChecksumInfo{partNumber, algorithm, checksum} } // AddTestObjectPart - add a new object part in order. func (m *xlMetaV1) AddTestObjectPart(partNumber int, partSize int64) { partInfo := ObjectPartInfo{ Number: partNumber, Size: partSize, } // Proceed to include new part info. m.Parts[partNumber-1] = partInfo } // Constructs xlMetaV1{} for given number of parts and converts it into bytes. func getXLMetaBytes(totalParts int) []byte { xlSampleMeta := getSampleXLMeta(totalParts) xlMetaBytes, err := json.Marshal(xlSampleMeta) if err != nil { panic(err) } return xlMetaBytes } // Returns sample xlMetaV1{} for number of parts. func getSampleXLMeta(totalParts int) xlMetaV1 { xlMeta := newTestXLMetaV1() // Number of checksum info == total parts. xlMeta.Erasure.Checksums = make([]ChecksumInfo, totalParts) // total number of parts. xlMeta.Parts = make([]ObjectPartInfo, totalParts) for i := 0; i < totalParts; i++ { // hard coding hash and algo value for the checksum, Since we are benchmarking the parsing of xl.json the magnitude doesn't affect the test, // The magnitude doesn't make a difference, only the size does. xlMeta.AddTestObjectCheckSum(i+1, BLAKE2b512, "a23f5eff248c4372badd9f3b2455a285cd4ca86c3d9a570b091d3fc5cd7ca6d9484bbea3f8c5d8d4f84daae96874419eda578fd736455334afbac2c924b3915a") xlMeta.AddTestObjectPart(i+1, 67108864) } return xlMeta } // Compare the unmarshaled XLMetaV1 with the one obtained from jsoniter parsing. func compareXLMetaV1(t *testing.T, unMarshalXLMeta, jsoniterXLMeta xlMetaV1) { // Start comparing the fields of xlMetaV1 obtained from jsoniter parsing with one parsed using json unmarshaling. if unMarshalXLMeta.Version != jsoniterXLMeta.Version { t.Errorf("Expected the Version to be \"%s\", but got \"%s\".", unMarshalXLMeta.Version, jsoniterXLMeta.Version) } if unMarshalXLMeta.Format != jsoniterXLMeta.Format { t.Errorf("Expected the format to be \"%s\", but got \"%s\".", unMarshalXLMeta.Format, jsoniterXLMeta.Format) } if unMarshalXLMeta.Stat.Size != jsoniterXLMeta.Stat.Size { t.Errorf("Expected the stat size to be %v, but got %v.", unMarshalXLMeta.Stat.Size, jsoniterXLMeta.Stat.Size) } if !unMarshalXLMeta.Stat.ModTime.Equal(jsoniterXLMeta.Stat.ModTime) { t.Errorf("Expected the modTime to be \"%v\", but got \"%v\".", unMarshalXLMeta.Stat.ModTime, jsoniterXLMeta.Stat.ModTime) } if unMarshalXLMeta.Erasure.Algorithm != jsoniterXLMeta.Erasure.Algorithm { t.Errorf("Expected the erasure algorithm to be \"%v\", but got \"%v\".", unMarshalXLMeta.Erasure.Algorithm, jsoniterXLMeta.Erasure.Algorithm) } if unMarshalXLMeta.Erasure.DataBlocks != jsoniterXLMeta.Erasure.DataBlocks { t.Errorf("Expected the erasure data blocks to be %v, but got %v.", unMarshalXLMeta.Erasure.DataBlocks, jsoniterXLMeta.Erasure.DataBlocks) } if unMarshalXLMeta.Erasure.ParityBlocks != jsoniterXLMeta.Erasure.ParityBlocks { t.Errorf("Expected the erasure parity blocks to be %v, but got %v.", unMarshalXLMeta.Erasure.ParityBlocks, jsoniterXLMeta.Erasure.ParityBlocks) } if unMarshalXLMeta.Erasure.BlockSize != jsoniterXLMeta.Erasure.BlockSize { t.Errorf("Expected the erasure block size to be %v, but got %v.", unMarshalXLMeta.Erasure.BlockSize, jsoniterXLMeta.Erasure.BlockSize) } if unMarshalXLMeta.Erasure.Index != jsoniterXLMeta.Erasure.Index { t.Errorf("Expected the erasure index to be %v, but got %v.", unMarshalXLMeta.Erasure.Index, jsoniterXLMeta.Erasure.Index) } if len(unMarshalXLMeta.Erasure.Distribution) != len(jsoniterXLMeta.Erasure.Distribution) { t.Errorf("Expected the size of Erasure Distribution to be %d, but got %d.", len(unMarshalXLMeta.Erasure.Distribution), len(jsoniterXLMeta.Erasure.Distribution)) } else { for i := 0; i < len(unMarshalXLMeta.Erasure.Distribution); i++ { if unMarshalXLMeta.Erasure.Distribution[i] != jsoniterXLMeta.Erasure.Distribution[i] { t.Errorf("Expected the Erasure Distribution to be %d, got %d.", unMarshalXLMeta.Erasure.Distribution[i], jsoniterXLMeta.Erasure.Distribution[i]) } } } if len(unMarshalXLMeta.Erasure.Checksums) != len(jsoniterXLMeta.Erasure.Checksums) { t.Errorf("Expected the size of Erasure Checksums to be %d, but got %d.", len(unMarshalXLMeta.Erasure.Checksums), len(jsoniterXLMeta.Erasure.Checksums)) } else { for i := 0; i < len(unMarshalXLMeta.Erasure.Checksums); i++ { if unMarshalXLMeta.Erasure.Checksums[i].PartNumber != jsoniterXLMeta.Erasure.Checksums[i].PartNumber { t.Errorf("Expected the Erasure Checksum PartNumber to be \"%d\", got \"%d\".", unMarshalXLMeta.Erasure.Checksums[i].PartNumber, jsoniterXLMeta.Erasure.Checksums[i].PartNumber) } if unMarshalXLMeta.Erasure.Checksums[i].Algorithm != jsoniterXLMeta.Erasure.Checksums[i].Algorithm { t.Errorf("Expected the Erasure Checksum Algorithm to be \"%s\", got \"%s\".", unMarshalXLMeta.Erasure.Checksums[i].Algorithm, jsoniterXLMeta.Erasure.Checksums[i].Algorithm) } if !bytes.Equal(unMarshalXLMeta.Erasure.Checksums[i].Hash, jsoniterXLMeta.Erasure.Checksums[i].Hash) { t.Errorf("Expected the Erasure Checksum Hash to be \"%s\", got \"%s\".", unMarshalXLMeta.Erasure.Checksums[i].Hash, jsoniterXLMeta.Erasure.Checksums[i].Hash) } } } if unMarshalXLMeta.Minio.Release != jsoniterXLMeta.Minio.Release { t.Errorf("Expected the Release string to be \"%s\", but got \"%s\".", unMarshalXLMeta.Minio.Release, jsoniterXLMeta.Minio.Release) } if len(unMarshalXLMeta.Parts) != len(jsoniterXLMeta.Parts) { t.Errorf("Expected info of %d parts to be present, but got %d instead.", len(unMarshalXLMeta.Parts), len(jsoniterXLMeta.Parts)) } else { for i := 0; i < len(unMarshalXLMeta.Parts); i++ { if unMarshalXLMeta.Parts[i].Number != jsoniterXLMeta.Parts[i].Number { t.Errorf("Expected the number of part %d to be \"%d\", got \"%d\".", i+1, unMarshalXLMeta.Parts[i].Number, jsoniterXLMeta.Parts[i].Number) } if unMarshalXLMeta.Parts[i].Size != jsoniterXLMeta.Parts[i].Size { t.Errorf("Expected the size of part %d to be %v, got %v.", i+1, unMarshalXLMeta.Parts[i].Size, jsoniterXLMeta.Parts[i].Size) } } } for key, val := range unMarshalXLMeta.Meta { jsoniterVal, exists := jsoniterXLMeta.Meta[key] if !exists { t.Errorf("No meta data entry for Key \"%s\" exists.", key) } if val != jsoniterVal { t.Errorf("Expected the value for Meta data key \"%s\" to be \"%s\", but got \"%s\".", key, val, jsoniterVal) } } } // Tests the correctness of constructing XLMetaV1 using jsoniter lib. // The result will be compared with the result obtained from json.unMarshal of the byte data. func TestGetXLMetaV1Jsoniter1(t *testing.T) { xlMetaJSON := getXLMetaBytes(1) var unMarshalXLMeta xlMetaV1 if err := json.Unmarshal(xlMetaJSON, &unMarshalXLMeta); err != nil { t.Errorf("Unmarshalling failed: %v", err) } jsoniterXLMeta, err := xlMetaV1UnmarshalJSON(GlobalContext, xlMetaJSON) if err != nil { t.Errorf("jsoniter parsing of XLMeta failed: %v", err) } compareXLMetaV1(t, unMarshalXLMeta, jsoniterXLMeta) } // Tests the correctness of constructing XLMetaV1 using jsoniter lib for XLMetaV1 of size 10 parts. // The result will be compared with the result obtained from json.unMarshal of the byte data. func TestGetXLMetaV1Jsoniter10(t *testing.T) { xlMetaJSON := getXLMetaBytes(10) var unMarshalXLMeta xlMetaV1 if err := json.Unmarshal(xlMetaJSON, &unMarshalXLMeta); err != nil { t.Errorf("Unmarshalling failed: %v", err) } jsoniterXLMeta, err := xlMetaV1UnmarshalJSON(GlobalContext, xlMetaJSON) if err != nil { t.Errorf("jsoniter parsing of XLMeta failed: %v", err) } compareXLMetaV1(t, unMarshalXLMeta, jsoniterXLMeta) } // Test the predicted part size from the part index func TestGetPartSizeFromIdx(t *testing.T) { // Create test cases testCases := []struct { totalSize int64 partSize int64 partIndex int expectedSize int64 }{ // Total size is zero {0, 10, 1, 0}, // part size 2MiB, total size 4MiB {4 * humanize.MiByte, 2 * humanize.MiByte, 1, 2 * humanize.MiByte}, {4 * humanize.MiByte, 2 * humanize.MiByte, 2, 2 * humanize.MiByte}, {4 * humanize.MiByte, 2 * humanize.MiByte, 3, 0}, // part size 2MiB, total size 5MiB {5 * humanize.MiByte, 2 * humanize.MiByte, 1, 2 * humanize.MiByte}, {5 * humanize.MiByte, 2 * humanize.MiByte, 2, 2 * humanize.MiByte}, {5 * humanize.MiByte, 2 * humanize.MiByte, 3, 1 * humanize.MiByte}, {5 * humanize.MiByte, 2 * humanize.MiByte, 4, 0}, } for i, testCase := range testCases { s, err := calculatePartSizeFromIdx(GlobalContext, testCase.totalSize, testCase.partSize, testCase.partIndex) if err != nil { t.Errorf("Test %d: Expected to pass but failed. %s", i+1, err) } if err == nil && s != testCase.expectedSize { t.Errorf("Test %d: The calculated part size is incorrect: expected = %d, found = %d\n", i+1, testCase.expectedSize, s) } } testCasesFailure := []struct { totalSize int64 partSize int64 partIndex int err error }{ // partSize is 0, returns error. {10, 0, 1, errPartSizeZero}, // partIndex is 0, returns error. {10, 1, 0, errPartSizeIndex}, // Total size is -1, returns error. {-2, 10, 1, errInvalidArgument}, } for i, testCaseFailure := range testCasesFailure { _, err := calculatePartSizeFromIdx(GlobalContext, testCaseFailure.totalSize, testCaseFailure.partSize, testCaseFailure.partIndex) if err == nil { t.Errorf("Test %d: Expected to failed but passed. %s", i+1, err) } if err != nil && err != testCaseFailure.err { t.Errorf("Test %d: Expected err %s, but got %s", i+1, testCaseFailure.err, err) } } } func TestShuffleDisks(t *testing.T) { nDisks := 16 disks, err := getRandomDisks(nDisks) if err != nil { t.Fatal(err) } objLayer, _, err := initObjectLayer(mustGetZoneEndpoints(disks...)) if err != nil { removeRoots(disks) t.Fatal(err) } defer removeRoots(disks) z := objLayer.(*xlZones) testShuffleDisks(t, z) } // Test shuffleDisks which returns shuffled slice of disks for their actual distribution. func testShuffleDisks(t *testing.T, z *xlZones) { disks := z.zones[0].GetDisks(0)() distribution := []int{16, 14, 12, 10, 8, 6, 4, 2, 1, 3, 5, 7, 9, 11, 13, 15} shuffledDisks := shuffleDisks(disks, distribution) // From the "distribution" above you can notice that: // 1st data block is in the 9th disk (i.e distribution index 8) // 2nd data block is in the 8th disk (i.e distribution index 7) and so on. if shuffledDisks[0] != disks[8] || shuffledDisks[1] != disks[7] || shuffledDisks[2] != disks[9] || shuffledDisks[3] != disks[6] || shuffledDisks[4] != disks[10] || shuffledDisks[5] != disks[5] || shuffledDisks[6] != disks[11] || shuffledDisks[7] != disks[4] || shuffledDisks[8] != disks[12] || shuffledDisks[9] != disks[3] || shuffledDisks[10] != disks[13] || shuffledDisks[11] != disks[2] || shuffledDisks[12] != disks[14] || shuffledDisks[13] != disks[1] || shuffledDisks[14] != disks[15] || shuffledDisks[15] != disks[0] { t.Errorf("shuffleDisks returned incorrect order.") } } // TestEvalDisks tests the behavior of evalDisks func TestEvalDisks(t *testing.T) { nDisks := 16 disks, err := getRandomDisks(nDisks) if err != nil { t.Fatal(err) } objLayer, _, err := initObjectLayer(mustGetZoneEndpoints(disks...)) if err != nil { removeRoots(disks) t.Fatal(err) } defer removeRoots(disks) z := objLayer.(*xlZones) testShuffleDisks(t, z) }