diff --git a/buildscripts/unaligned-healing.sh b/buildscripts/unaligned-healing.sh index 5249d0183..ce7ea5faf 100755 --- a/buildscripts/unaligned-healing.sh +++ b/buildscripts/unaligned-healing.sh @@ -29,6 +29,7 @@ function start_minio_16drive() { export MINIO_ROOT_PASSWORD=minio123 export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/" unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects + export _MINIO_SHARD_DISKTIME_DELTA="5s" # do not change this as its needed for tests MC_BUILD_DIR="mc-$RANDOM" if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then diff --git a/cmd/bootstrap-peer-server.go b/cmd/bootstrap-peer-server.go index 32ca5bb3a..13005d061 100644 --- a/cmd/bootstrap-peer-server.go +++ b/cmd/bootstrap-peer-server.go @@ -86,14 +86,28 @@ func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error { } } if !reflect.DeepEqual(s1.MinioEnv, s2.MinioEnv) { - return fmt.Errorf("Expected same MINIO_ environment variables and values") + var missing []string + var mismatching []string + for k, v := range s1.MinioEnv { + ev, ok := s2.MinioEnv[k] + if !ok { + missing = append(missing, k) + } else if v != ev { + mismatching = append(mismatching, k) + } + } + if len(mismatching) > 0 { + return fmt.Errorf(`Expected same MINIO_ environment variables and values across all servers: Missing environment values: %s / Mismatch environment values: %s`, missing, mismatching) + } + return fmt.Errorf(`Expected same MINIO_ environment variables and values across all servers: Missing environment values: %s`, missing) } return nil } var skipEnvs = map[string]struct{}{ - "MINIO_OPTS": {}, - "MINIO_CERT_PASSWD": {}, + "MINIO_OPTS": {}, + "MINIO_CERT_PASSWD": {}, + "MINIO_SERVER_DEBUG": {}, } func getServerSystemCfg() ServerSystemConfig { diff --git a/cmd/common-main.go b/cmd/common-main.go index 770efb202..8a6ffd8fc 100644 --- a/cmd/common-main.go +++ b/cmd/common-main.go @@ -67,6 +67,7 @@ import ( // serverDebugLog will enable debug printing var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn +var shardDiskTimeDelta time.Duration var defaultAWSCredProvider []credentials.Provider func init() { @@ -136,6 +137,12 @@ func init() { }, } + var err error + shardDiskTimeDelta, err = time.ParseDuration(env.Get("_MINIO_SHARD_DISKTIME_DELTA", "1m")) + if err != nil { + shardDiskTimeDelta = 1 * time.Minute + } + // All minio-go API operations shall be performed only once, // another way to look at this is we are turning off retries. minio.MaxRetry = 1 diff --git a/cmd/erasure-healing-common.go b/cmd/erasure-healing-common.go index aac49bc72..d384fa8d7 100644 --- a/cmd/erasure-healing-common.go +++ b/cmd/erasure-healing-common.go @@ -229,7 +229,6 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad bucket, object string, scanMode madmin.HealScanMode) ([]StorageAPI, []error) { var diskMTime time.Time - delta := 5 * time.Second if !latestMeta.DataShardFixed() { diskMTime = pickValidDiskTimeWithQuorum(partsMetadata, latestMeta.Erasure.DataBlocks) @@ -340,7 +339,7 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad } if !diskMTime.Equal(timeSentinel) && !diskMTime.IsZero() { - if !partsMetadata[i].AcceptableDelta(diskMTime, delta) { + if !partsMetadata[i].AcceptableDelta(diskMTime, shardDiskTimeDelta) { // not with in acceptable delta, skip. partsMetadata[i] = FileInfo{} continue diff --git a/cmd/erasure-metadata.go b/cmd/erasure-metadata.go index a3fff78c0..7c1ae2314 100644 --- a/cmd/erasure-metadata.go +++ b/cmd/erasure-metadata.go @@ -354,7 +354,7 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time. func pickValidDiskTimeWithQuorum(metaArr []FileInfo, quorum int) time.Time { diskMTimes := listObjectDiskMtimes(metaArr) - diskMTime, diskMaxima := commonTimeAndOccurence(diskMTimes, 5*time.Second) + diskMTime, diskMaxima := commonTimeAndOccurence(diskMTimes, shardDiskTimeDelta) if diskMaxima >= quorum { return diskMTime } diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index fcde3dbe4..b75721a0e 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -28,7 +28,6 @@ import ( "strconv" "strings" "sync" - "time" "github.com/minio/madmin-go" "github.com/minio/minio-go/v7/pkg/tags" @@ -182,7 +181,6 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri if !fi.DataShardFixed() { diskMTime := pickValidDiskTimeWithQuorum(metaArr, fi.Erasure.DataBlocks) - delta := 5 * time.Second if !diskMTime.Equal(timeSentinel) && !diskMTime.IsZero() { for index := range onlineDisks { if onlineDisks[index] == OfflineDisk { @@ -191,7 +189,7 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri if !metaArr[index].IsValid() { continue } - if !metaArr[index].AcceptableDelta(diskMTime, delta) { + if !metaArr[index].AcceptableDelta(diskMTime, shardDiskTimeDelta) { // If disk mTime mismatches it is considered outdated // https://github.com/minio/minio/pull/13803 //