add configurable delta for skipping shards (#13967)

This PR is an attempt to make this configurable
as not all situations have same level of tolerable
delta, i.e disks are replaced days apart or even
hours.

There is also a possibility that nodes have drifted
in time, when NTP is not configured on the system.

buildscripts/unaligned-healing.sh

@@ -29,6 +29,7 @@ function start_minio_16drive() {
     export MINIO_ROOT_PASSWORD=minio123
     export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
     unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+    export _MINIO_SHARD_DISKTIME_DELTA="5s" # do not change this as its needed for tests
 
     MC_BUILD_DIR="mc-$RANDOM"
     if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then

cmd/bootstrap-peer-server.go

@@ -86,7 +86,20 @@ func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error {
 		}
 	}
 	if !reflect.DeepEqual(s1.MinioEnv, s2.MinioEnv) {
-		return fmt.Errorf("Expected same MINIO_ environment variables and values")
+		var missing []string
+		var mismatching []string
+		for k, v := range s1.MinioEnv {
+			ev, ok := s2.MinioEnv[k]
+			if !ok {
+				missing = append(missing, k)
+			} else if v != ev {
+				mismatching = append(mismatching, k)
+			}
+		}
+		if len(mismatching) > 0 {
+			return fmt.Errorf(`Expected same MINIO_ environment variables and values across all servers: Missing environment values: %s / Mismatch environment values: %s`, missing, mismatching)
+		}
+		return fmt.Errorf(`Expected same MINIO_ environment variables and values across all servers: Missing environment values: %s`, missing)
 	}
 	return nil
 }
@@ -94,6 +107,7 @@ func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error {
 var skipEnvs = map[string]struct{}{
 	"MINIO_OPTS":         {},
 	"MINIO_CERT_PASSWD":  {},
+	"MINIO_SERVER_DEBUG": {},
 }
 
 func getServerSystemCfg() ServerSystemConfig {

cmd/common-main.go

@@ -67,6 +67,7 @@ import (
 
 // serverDebugLog will enable debug printing
 var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn
+var shardDiskTimeDelta time.Duration
 var defaultAWSCredProvider []credentials.Provider
 
 func init() {
@@ -136,6 +137,12 @@ func init() {
 		},
 	}
 
+	var err error
+	shardDiskTimeDelta, err = time.ParseDuration(env.Get("_MINIO_SHARD_DISKTIME_DELTA", "1m"))
+	if err != nil {
+		shardDiskTimeDelta = 1 * time.Minute
+	}
+
 	// All minio-go API operations shall be performed only once,
 	// another way to look at this is we are turning off retries.
 	minio.MaxRetry = 1

cmd/erasure-healing-common.go

@@ -229,7 +229,6 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 	bucket, object string, scanMode madmin.HealScanMode) ([]StorageAPI, []error) {
 
 	var diskMTime time.Time
-	delta := 5 * time.Second
 	if !latestMeta.DataShardFixed() {
 		diskMTime = pickValidDiskTimeWithQuorum(partsMetadata,
 			latestMeta.Erasure.DataBlocks)
@@ -340,7 +339,7 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 		}
 
 		if !diskMTime.Equal(timeSentinel) && !diskMTime.IsZero() {
-			if !partsMetadata[i].AcceptableDelta(diskMTime, delta) {
+			if !partsMetadata[i].AcceptableDelta(diskMTime, shardDiskTimeDelta) {
 				// not with in acceptable delta, skip.
 				partsMetadata[i] = FileInfo{}
 				continue

cmd/erasure-metadata.go

@@ -354,7 +354,7 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.
 func pickValidDiskTimeWithQuorum(metaArr []FileInfo, quorum int) time.Time {
 	diskMTimes := listObjectDiskMtimes(metaArr)
 
-	diskMTime, diskMaxima := commonTimeAndOccurence(diskMTimes, 5*time.Second)
+	diskMTime, diskMaxima := commonTimeAndOccurence(diskMTimes, shardDiskTimeDelta)
 	if diskMaxima >= quorum {
 		return diskMTime
 	}

cmd/erasure-object.go

@@ -28,7 +28,6 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"time"
 
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7/pkg/tags"
@@ -182,7 +181,6 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 
 	if !fi.DataShardFixed() {
 		diskMTime := pickValidDiskTimeWithQuorum(metaArr, fi.Erasure.DataBlocks)
-		delta := 5 * time.Second
 		if !diskMTime.Equal(timeSentinel) && !diskMTime.IsZero() {
 			for index := range onlineDisks {
 				if onlineDisks[index] == OfflineDisk {
@@ -191,7 +189,7 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 				if !metaArr[index].IsValid() {
 					continue
 				}
-				if !metaArr[index].AcceptableDelta(diskMTime, delta) {
+				if !metaArr[index].AcceptableDelta(diskMTime, shardDiskTimeDelta) {
 					// If disk mTime mismatches it is considered outdated
 					// https://github.com/minio/minio/pull/13803
 					//