XL: Make listOnlineDisks and outDatedDisks consistent w/ each other. (#3808)

cmd/xl-v1-healing-common_test.go

@@ -17,6 +17,8 @@
 package cmd
 
 import (
+	"bytes"
+	"path/filepath"
 	"testing"
 	"time"
 )
@@ -81,3 +83,250 @@ func TestCommonTime(t *testing.T) {
 		}
 	}
 }
+
+// partsMetaFromModTimes - returns slice of modTimes given metadata of
+// an object part.
+func partsMetaFromModTimes(modTimes []time.Time, checksums []checkSumInfo) []xlMetaV1 {
+	var partsMetadata []xlMetaV1
+	for _, modTime := range modTimes {
+		partsMetadata = append(partsMetadata, xlMetaV1{
+			Erasure: erasureInfo{
+				Checksum: checksums,
+			},
+			Stat: statInfo{
+				ModTime: modTime,
+			},
+			Parts: []objectPartInfo{
+				{
+					Name: "part.1",
+				},
+			},
+		})
+	}
+	return partsMetadata
+}
+
+// toPosix - fetches *posix object from StorageAPI.
+func toPosix(disk StorageAPI) *posix {
+	retryDisk, ok := disk.(*retryStorage)
+	if !ok {
+		return nil
+	}
+	pDisk, ok := retryDisk.remoteStorage.(*posix)
+	if !ok {
+		return nil
+	}
+	return pDisk
+
+}
+
+// TestListOnlineDisks - checks if listOnlineDisks and outDatedDisks
+// are consistent with each other.
+func TestListOnlineDisks(t *testing.T) {
+	rootPath, err := newTestConfig(globalMinioDefaultRegion)
+	if err != nil {
+		t.Fatalf("Failed to initialize config - %v", err)
+	}
+	defer removeAll(rootPath)
+
+	obj, disks, err := prepareXL()
+	if err != nil {
+		t.Fatalf("Prepare XL backend failed - %v", err)
+	}
+	defer removeRoots(disks)
+
+	type tamperKind int
+	const (
+		noTamper    tamperKind = iota
+		deletePart  tamperKind = iota
+		corruptPart tamperKind = iota
+	)
+	threeNanoSecs := time.Unix(0, 3).UTC()
+	fourNanoSecs := time.Unix(0, 4).UTC()
+	modTimesThreeNone := []time.Time{
+		threeNanoSecs, threeNanoSecs, threeNanoSecs, threeNanoSecs,
+		threeNanoSecs, threeNanoSecs, threeNanoSecs,
+		timeSentinel, timeSentinel, timeSentinel, timeSentinel,
+		timeSentinel, timeSentinel, timeSentinel, timeSentinel,
+		timeSentinel,
+	}
+	modTimesThreeFour := []time.Time{
+		threeNanoSecs, threeNanoSecs, threeNanoSecs, threeNanoSecs,
+		threeNanoSecs, threeNanoSecs, threeNanoSecs, threeNanoSecs,
+		fourNanoSecs, fourNanoSecs, fourNanoSecs, fourNanoSecs,
+		fourNanoSecs, fourNanoSecs, fourNanoSecs, fourNanoSecs,
+	}
+	testCases := []struct {
+		modTimes       []time.Time
+		expectedTime   time.Time
+		errs           []error
+		_tamperBackend tamperKind
+	}{
+		{
+			modTimes:     modTimesThreeFour,
+			expectedTime: fourNanoSecs,
+			errs: []error{
+				nil, nil, nil, nil, nil, nil, nil, nil, nil,
+				nil, nil, nil, nil, nil, nil, nil,
+			},
+			_tamperBackend: noTamper,
+		},
+		{
+			modTimes:     modTimesThreeNone,
+			expectedTime: threeNanoSecs,
+			errs: []error{
+				// Disks that have a valid xl.json.
+				nil, nil, nil, nil, nil, nil, nil,
+				// Majority of disks don't have xl.json.
+				errFileNotFound, errFileNotFound,
+				errFileNotFound, errFileNotFound,
+				errFileNotFound, errDiskAccessDenied,
+				errDiskNotFound, errFileNotFound,
+				errFileNotFound,
+			},
+			_tamperBackend: deletePart,
+		},
+		{
+			modTimes:     modTimesThreeNone,
+			expectedTime: threeNanoSecs,
+			errs: []error{
+				// Disks that have a valid xl.json.
+				nil, nil, nil, nil, nil, nil, nil,
+				// Majority of disks don't have xl.json.
+				errFileNotFound, errFileNotFound,
+				errFileNotFound, errFileNotFound,
+				errFileNotFound, errDiskAccessDenied,
+				errDiskNotFound, errFileNotFound,
+				errFileNotFound,
+			},
+			_tamperBackend: corruptPart,
+		},
+	}
+
+	bucket := "bucket"
+	object := "object"
+	data := bytes.Repeat([]byte("a"), 1024)
+	xlDisks := obj.(*xlObjects).storageDisks
+	for i, test := range testCases {
+		// Prepare bucket/object backend for the tests below.
+
+		// Cleanup from previous test.
+		obj.DeleteObject(bucket, object)
+		obj.DeleteBucket(bucket)
+
+		err = obj.MakeBucket("bucket")
+		if err != nil {
+			t.Fatalf("Failed to make a bucket %v", err)
+		}
+
+		_, err = obj.PutObject(bucket, object, int64(len(data)), bytes.NewReader(data), nil, "")
+		if err != nil {
+			t.Fatalf("Failed to putObject %v", err)
+		}
+
+		// Fetch xl.json from first disk to construct partsMetadata for the tests.
+		xlMeta, err := readXLMeta(xlDisks[0], bucket, object)
+		if err != nil {
+			t.Fatalf("Test %d: Failed to read xl.json %v", i+1, err)
+		}
+
+		tamperedIndex := -1
+		switch test._tamperBackend {
+		case deletePart:
+			for index, err := range test.errs {
+				if err != nil {
+					continue
+				}
+				// Remove a part from a disk
+				// which has a valid xl.json,
+				// and check if that disk
+				// appears in outDatedDisks.
+				tamperedIndex = index
+				dErr := xlDisks[index].DeleteFile(bucket, filepath.Join(object, "part.1"))
+				if dErr != nil {
+					t.Fatalf("Test %d: Failed to delete %s - %v", i+1,
+						filepath.Join(object, "part.1"), dErr)
+				}
+				break
+			}
+		case corruptPart:
+			for index, err := range test.errs {
+				if err != nil {
+					continue
+				}
+				// Corrupt a part from a disk
+				// which has a valid xl.json,
+				// and check if that disk
+				// appears in outDatedDisks.
+				tamperedIndex = index
+				dErr := xlDisks[index].AppendFile(bucket, filepath.Join(object, "part.1"), []byte("corruption"))
+				if dErr != nil {
+					t.Fatalf("Test %d: Failed to append corrupting data at the end of file %s - %v",
+						i+1, filepath.Join(object, "part.1"), dErr)
+				}
+				break
+			}
+
+		}
+
+		partsMetadata := partsMetaFromModTimes(test.modTimes, xlMeta.Erasure.Checksum)
+
+		onlineDisks, modTime := listOnlineDisks(xlDisks, partsMetadata, test.errs)
+		availableDisks, newErrs, err := disksWithAllParts(onlineDisks, partsMetadata, test.errs, bucket, object)
+		test.errs = newErrs
+		outdatedDisks := outDatedDisks(xlDisks, availableDisks, test.errs, partsMetadata, bucket, object)
+		if modTime.Equal(timeSentinel) {
+			t.Fatalf("Test %d: modTime should never be equal to timeSentinel, but found equal",
+				i+1)
+		}
+
+		if test._tamperBackend != noTamper {
+			if tamperedIndex != -1 && outdatedDisks[tamperedIndex] == nil {
+				t.Fatalf("Test %d: disk (%v) with part.1 missing is an outdated disk, but wasn't listed by outDatedDisks",
+					i+1, xlDisks[tamperedIndex])
+			}
+
+		}
+
+		if !modTime.Equal(test.expectedTime) {
+			t.Fatalf("Test %d: Expected modTime to be equal to %v but was found to be %v",
+				i+1, test.expectedTime, modTime)
+		}
+
+		// Check if a disk is considered both online and outdated,
+		// which is a contradiction, except if parts are missing.
+		overlappingDisks := make(map[string]*posix)
+		for _, availableDisk := range availableDisks {
+			if availableDisk == nil {
+				continue
+			}
+			pDisk := toPosix(availableDisk)
+			overlappingDisks[pDisk.diskPath] = pDisk
+		}
+
+		for index, outdatedDisk := range outdatedDisks {
+			// ignore the intentionally tampered disk,
+			// this is expected to appear as outdated
+			// disk, since it doesn't have all the parts.
+			if index == tamperedIndex {
+				continue
+			}
+
+			if outdatedDisk == nil {
+				continue
+			}
+
+			pDisk := toPosix(outdatedDisk)
+			if _, ok := overlappingDisks[pDisk.diskPath]; ok {
+				t.Errorf("Test %d: Outdated disk %v was also detected as an online disk - %v %v",
+					i+1, pDisk, availableDisks, outdatedDisks)
+			}
+
+			// errors other than errFileNotFound doesn't imply that the disk is outdated.
+			if test.errs[index] != nil && test.errs[index] != errFileNotFound && outdatedDisk != nil {
+				t.Errorf("Test %d: error (%v) other than errFileNotFound doesn't imply that the disk (%v) could be outdated",
+					i+1, test.errs[index], pDisk)
+			}
+		}
+	}
+}