From 54ec0a13080d3c20d10fadde0d735c3a03edf913 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 22 Dec 2021 11:43:01 -0800 Subject: [PATCH] add configurable delta for skipping shards (#13967) This PR is an attempt to make this configurable as not all situations have same level of tolerable delta, i.e disks are replaced days apart or even hours. There is also a possibility that nodes have drifted in time, when NTP is not configured on the system. --- buildscripts/unaligned-healing.sh | 1 + cmd/bootstrap-peer-server.go | 20 +++++++++++++++++--- cmd/common-main.go | 7 +++++++ cmd/erasure-healing-common.go | 3 +-- cmd/erasure-metadata.go | 2 +- cmd/erasure-object.go | 4 +--- 6 files changed, 28 insertions(+), 9 deletions(-) diff --git a/buildscripts/unaligned-healing.sh b/buildscripts/unaligned-healing.sh index 5249d0183..ce7ea5faf 100755 --- a/buildscripts/unaligned-healing.sh +++ b/buildscripts/unaligned-healing.sh @@ -29,6 +29,7 @@ function start_minio_16drive() { export MINIO_ROOT_PASSWORD=minio123 export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/" unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects + export _MINIO_SHARD_DISKTIME_DELTA="5s" # do not change this as its needed for tests MC_BUILD_DIR="mc-$RANDOM" if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then diff --git a/cmd/bootstrap-peer-server.go b/cmd/bootstrap-peer-server.go index 32ca5bb3a..13005d061 100644 --- a/cmd/bootstrap-peer-server.go +++ b/cmd/bootstrap-peer-server.go @@ -86,14 +86,28 @@ func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error { } } if !reflect.DeepEqual(s1.MinioEnv, s2.MinioEnv) { - return fmt.Errorf("Expected same MINIO_ environment variables and values") + var missing []string + var mismatching []string + for k, v := range s1.MinioEnv { + ev, ok := s2.MinioEnv[k] + if !ok { + missing = append(missing, k) + } else if v != ev { + mismatching = append(mismatching, k) + } + } + if len(mismatching) > 0 { + return fmt.Errorf(`Expected same MINIO_ environment variables and values across all servers: Missing environment values: %s / Mismatch environment values: %s`, missing, mismatching) + } + return fmt.Errorf(`Expected same MINIO_ environment variables and values across all servers: Missing environment values: %s`, missing) } return nil } var skipEnvs = map[string]struct{}{ - "MINIO_OPTS": {}, - "MINIO_CERT_PASSWD": {}, + "MINIO_OPTS": {}, + "MINIO_CERT_PASSWD": {}, + "MINIO_SERVER_DEBUG": {}, } func getServerSystemCfg() ServerSystemConfig { diff --git a/cmd/common-main.go b/cmd/common-main.go index 770efb202..8a6ffd8fc 100644 --- a/cmd/common-main.go +++ b/cmd/common-main.go @@ -67,6 +67,7 @@ import ( // serverDebugLog will enable debug printing var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn +var shardDiskTimeDelta time.Duration var defaultAWSCredProvider []credentials.Provider func init() { @@ -136,6 +137,12 @@ func init() { }, } + var err error + shardDiskTimeDelta, err = time.ParseDuration(env.Get("_MINIO_SHARD_DISKTIME_DELTA", "1m")) + if err != nil { + shardDiskTimeDelta = 1 * time.Minute + } + // All minio-go API operations shall be performed only once, // another way to look at this is we are turning off retries. minio.MaxRetry = 1 diff --git a/cmd/erasure-healing-common.go b/cmd/erasure-healing-common.go index aac49bc72..d384fa8d7 100644 --- a/cmd/erasure-healing-common.go +++ b/cmd/erasure-healing-common.go @@ -229,7 +229,6 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad bucket, object string, scanMode madmin.HealScanMode) ([]StorageAPI, []error) { var diskMTime time.Time - delta := 5 * time.Second if !latestMeta.DataShardFixed() { diskMTime = pickValidDiskTimeWithQuorum(partsMetadata, latestMeta.Erasure.DataBlocks) @@ -340,7 +339,7 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad } if !diskMTime.Equal(timeSentinel) && !diskMTime.IsZero() { - if !partsMetadata[i].AcceptableDelta(diskMTime, delta) { + if !partsMetadata[i].AcceptableDelta(diskMTime, shardDiskTimeDelta) { // not with in acceptable delta, skip. partsMetadata[i] = FileInfo{} continue diff --git a/cmd/erasure-metadata.go b/cmd/erasure-metadata.go index a3fff78c0..7c1ae2314 100644 --- a/cmd/erasure-metadata.go +++ b/cmd/erasure-metadata.go @@ -354,7 +354,7 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time. func pickValidDiskTimeWithQuorum(metaArr []FileInfo, quorum int) time.Time { diskMTimes := listObjectDiskMtimes(metaArr) - diskMTime, diskMaxima := commonTimeAndOccurence(diskMTimes, 5*time.Second) + diskMTime, diskMaxima := commonTimeAndOccurence(diskMTimes, shardDiskTimeDelta) if diskMaxima >= quorum { return diskMTime } diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index fcde3dbe4..b75721a0e 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -28,7 +28,6 @@ import ( "strconv" "strings" "sync" - "time" "github.com/minio/madmin-go" "github.com/minio/minio-go/v7/pkg/tags" @@ -182,7 +181,6 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri if !fi.DataShardFixed() { diskMTime := pickValidDiskTimeWithQuorum(metaArr, fi.Erasure.DataBlocks) - delta := 5 * time.Second if !diskMTime.Equal(timeSentinel) && !diskMTime.IsZero() { for index := range onlineDisks { if onlineDisks[index] == OfflineDisk { @@ -191,7 +189,7 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri if !metaArr[index].IsValid() { continue } - if !metaArr[index].AcceptableDelta(diskMTime, delta) { + if !metaArr[index].AcceptableDelta(diskMTime, shardDiskTimeDelta) { // If disk mTime mismatches it is considered outdated // https://github.com/minio/minio/pull/13803 //