Add naughty disk to simulate disk errors (#2645)

cmd/format-config-v1_test.go

@@ -66,6 +66,75 @@ func genFormatXLInvalidVersion() []*formatConfigV1 {
 	return formatConfigs
 }
 
+// generates a invalid format.json version for XL backend.
+func genFormatXLInvalidFormat() []*formatConfigV1 {
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	// Corrupt version numbers.
+	formatConfigs[0].Format = "lx"
+	formatConfigs[3].Format = "lx"
+	return formatConfigs
+}
+
+// generates a invalid format.json version for XL backend.
+func genFormatXLInvalidXLVersion() []*formatConfigV1 {
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	// Corrupt version numbers.
+	formatConfigs[0].XL.Version = "10"
+	formatConfigs[3].XL.Version = "-1"
+	return formatConfigs
+}
+
+// generates a invalid format.json version for XL backend.
+func genFormatXLInvalidJBODCount() []*formatConfigV1 {
+	jbod := make([]string, 7)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	return formatConfigs
+}
+
 // generates a invalid format.json JBOD for XL backend.
 func genFormatXLInvalidJBOD() []*formatConfigV1 {
 	jbod := make([]string, 8)
@@ -145,18 +214,14 @@ func genFormatXLInvalidDisksOrder() []*formatConfigV1 {
 	return formatConfigs
 }
 
-func TestFormatXLHealFreshDisks(t *testing.T) {
-	// Create an instance of xl backend.
-	obj, fsDirs, err := getXLObjectLayer()
-	if err != nil {
-		t.Fatal(err)
-	}
+func prepareFormatXLHealFreshDisks(obj ObjectLayer) ([]StorageAPI, error) {
 
+	var err error
 	xl := obj.(xlObjects)
 
 	err = obj.MakeBucket("bucket")
 	if err != nil {
-		t.Fatal(err)
+		return []StorageAPI{}, err
 	}
 
 	bucket := "bucket"
@@ -164,34 +229,26 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
 
 	_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil)
 	if err != nil {
-		t.Fatal(err)
+		return []StorageAPI{}, err
 	}
 
-	/* // Now, remove two format files.. Load them and reorder
-	if err = xl.storageDisks[3].DeleteFile(".minio.sys", "format.json"); err != nil {
-		t.Fatal(err)
-	}
-	if err = xl.storageDisks[11].DeleteFile(".minio.sys", "format.json"); err != nil {
-		t.Fatal(err)
-	} */
-
 	// Remove the content of export dir 10 but preserve .minio.sys because it is automatically
 	// created when minio starts
 	for i := 3; i <= 5; i++ {
 		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
-			t.Fatal(err)
+			return []StorageAPI{}, err
 		}
 		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "tmp"); err != nil {
-			t.Fatal(err)
+			return []StorageAPI{}, err
 		}
 		if err = xl.storageDisks[i].DeleteFile(bucket, object+"/xl.json"); err != nil {
-			t.Fatal(err)
+			return []StorageAPI{}, err
 		}
 		if err = xl.storageDisks[i].DeleteFile(bucket, object+"/part.1"); err != nil {
-			t.Fatal(err)
+			return []StorageAPI{}, err
 		}
 		if err = xl.storageDisks[i].DeleteVol(bucket); err != nil {
-			t.Fatal(err)
+			return []StorageAPI{}, err
 		}
 	}
 
@@ -201,14 +258,29 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
 		xl.storageDisks[3], xl.storageDisks[10], xl.storageDisks[12], xl.storageDisks[9],
 		xl.storageDisks[5], xl.storageDisks[11]}
 
+	return permutedStorageDisks, nil
+
+}
+
+func TestFormatXLHealFreshDisks(t *testing.T) {
+	// Create an instance of xl backend.
+	obj, fsDirs, err := getXLObjectLayer()
+	if err != nil {
+		t.Error(err)
+	}
+
+	storageDisks, err := prepareFormatXLHealFreshDisks(obj)
+	if err != nil {
+		t.Fatal(err)
+	}
 	// Start healing disks
-	err = healFormatXLFreshDisks(permutedStorageDisks)
+	err = healFormatXLFreshDisks(storageDisks)
 	if err != nil {
 		t.Fatal("healing corrupted disk failed: ", err)
 	}
 
 	// Load again XL format.json to validate it
-	_, err = loadFormatXL(permutedStorageDisks)
+	_, err = loadFormatXL(storageDisks)
 	if err != nil {
 		t.Fatal("loading healed disk failed: ", err)
 	}
@@ -217,6 +289,39 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
 	removeRoots(fsDirs)
 }
 
+func TestFormatXLHealFreshDisksErrorExpected(t *testing.T) {
+	// Create an instance of xl backend.
+	obj, fsDirs, err := getXLObjectLayer()
+	if err != nil {
+		t.Error(err)
+	}
+
+	storageDisks, err := prepareFormatXLHealFreshDisks(obj)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := 0; i < 16; i++ {
+		d := storageDisks[i].(*posix)
+		storageDisks[i] = &naughtyDisk{disk: d, defaultErr: errDiskNotFound}
+	}
+
+	// Load again XL format.json to validate it
+	_, err = loadFormatXL(storageDisks)
+	if err == nil {
+		t.Fatal("loading format disk error")
+	}
+
+	storageDisks[3] = nil
+	err = healFormatXLFreshDisks(storageDisks)
+	if err != nil {
+		t.Fatal("didn't get nil when one disk is offline")
+	}
+
+	// Clean all
+	removeRoots(fsDirs)
+}
+
 // Simulate XL disks creation, delete some format.json and remove the content of
 // a given disk to test healing a corrupted disk
 func TestFormatXLHealCorruptedDisks(t *testing.T) {
@@ -355,12 +460,18 @@ func TestFormatXLReorderByInspection(t *testing.T) {
 // Wrapper for calling FormatXL tests - currently validates
 //  - valid format
 //  - unrecognized version number
+//  - unrecognized format tag
+//  - unrecognized xl version
+//  - wrong number of JBOD entries
 //  - invalid JBOD
 //  - invalid Disk uuid
 func TestFormatXL(t *testing.T) {
 	formatInputCases := [][]*formatConfigV1{
 		genFormatXLValid(),
 		genFormatXLInvalidVersion(),
+		genFormatXLInvalidFormat(),
+		genFormatXLInvalidXLVersion(),
+		genFormatXLInvalidJBODCount(),
 		genFormatXLInvalidJBOD(),
 		genFormatXLInvalidDisks(),
 		genFormatXLInvalidDisksOrder(),
@@ -389,6 +500,18 @@ func TestFormatXL(t *testing.T) {
 			formatConfigs: formatInputCases[4],
 			shouldPass:    false,
 		},
+		{
+			formatConfigs: formatInputCases[5],
+			shouldPass:    false,
+		},
+		{
+			formatConfigs: formatInputCases[6],
+			shouldPass:    false,
+		},
+		{
+			formatConfigs: formatInputCases[7],
+			shouldPass:    false,
+		},
 	}
 
 	for i, testCase := range testCases {
@@ -443,3 +566,65 @@ func TestSavedUUIDOrder(t *testing.T) {
 		}
 	}
 }
+
+func TestInitFormatXLErrors(t *testing.T) {
+	// Create an instance of xl backend.
+	obj, fsDirs, err := getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl := obj.(xlObjects)
+
+	testStorageDisks := make([]StorageAPI, 16)
+
+	for i := 0; i < 16; i++ {
+		d := xl.storageDisks[i].(*posix)
+		testStorageDisks[i] = &naughtyDisk{disk: d, defaultErr: errDiskNotFound}
+	}
+
+	if err := initFormatXL(testStorageDisks); err != errDiskNotFound {
+		t.Fatal("Got a different error: ", err)
+	}
+
+	for i := 0; i < 15; i++ {
+		d := xl.storageDisks[i].(*posix)
+		testStorageDisks[i] = &naughtyDisk{disk: d, defaultErr: errDiskNotFound, errors: map[int]error{0: nil, 1: nil, 2: nil}}
+	}
+
+	if err := initFormatXL(testStorageDisks); err != errDiskNotFound {
+		t.Fatal("Got a different error: ", err)
+	}
+
+	for i := 0; i < 15; i++ {
+		testStorageDisks[i] = nil
+	}
+
+	if err := initFormatXL(testStorageDisks); err != errDiskNotFound {
+		t.Fatal("Got a different error: ", err)
+	}
+
+	removeRoots(fsDirs)
+}
+
+func TestReduceFormatErrs(t *testing.T) {
+	// No error founds
+	if err := reduceFormatErrs([]error{nil, nil, nil, nil}, 4); err != nil {
+		t.Fatal("Err should be nil, found: ", err)
+	}
+	// Expect corrupted format error
+	if err := reduceFormatErrs([]error{nil, nil, errCorruptedFormat, nil}, 4); err != errCorruptedFormat {
+		t.Fatal("Got a differnt error: ", err)
+	}
+	// Expect unformatted disk
+	if err := reduceFormatErrs([]error{errUnformattedDisk, errUnformattedDisk, errUnformattedDisk, errUnformattedDisk}, 4); err != errUnformattedDisk {
+		t.Fatal("Got a differnt error: ", err)
+	}
+	// Expect some disks unformatted
+	if err := reduceFormatErrs([]error{nil, nil, errUnformattedDisk, errUnformattedDisk}, 4); err != errSomeDiskUnformatted {
+		t.Fatal("Got a differnt error: ", err)
+	}
+	// Expect some disks offline
+	if err := reduceFormatErrs([]error{nil, nil, errDiskNotFound, errUnformattedDisk}, 4); err != errSomeDiskOffline {
+		t.Fatal("Got a differnt error: ", err)
+	}
+}