From affee27b05db944078ae00f11142f923b1d11708 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 8 Jul 2021 01:04:37 -0700 Subject: [PATCH] fix: speed up erasure code upgrade checks (#12626) DiskInfo() calls can stagger and wait if run serially timing out 10secs per drive, to avoid this lets check DiskInfo in parallel to avoid delays when nodes get disconnected. --- cmd/erasure-object.go | 33 +++++++++++++++++++++++++-------- go.mod | 1 + 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 7576cf3fc..5a70e6802 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -40,6 +40,7 @@ import ( "github.com/minio/minio/internal/logger" "github.com/minio/minio/internal/sync/errgroup" "github.com/minio/pkg/mimedb" + uatomic "go.uber.org/atomic" ) // list all errors which can be ignored in object operations. @@ -608,19 +609,35 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st // If we have offline disks upgrade the number of erasure codes for this object. parityOrig := parityDrives + + atomicParityDrives := uatomic.NewInt64(0) + // Start with current parityDrives + atomicParityDrives.Store(int64(parityDrives)) + + var wg sync.WaitGroup for _, disk := range storageDisks { - if parityDrives >= len(storageDisks)/2 { - parityDrives = len(storageDisks) / 2 - break - } if disk == nil { - parityDrives++ + atomicParityDrives.Inc() continue } - di, err := disk.DiskInfo(ctx) - if err != nil || di.ID == "" { - parityDrives++ + if !disk.IsOnline() { + atomicParityDrives.Inc() + continue } + wg.Add(1) + go func(disk StorageAPI) { + defer wg.Done() + di, err := disk.DiskInfo(ctx) + if err != nil || di.ID == "" { + atomicParityDrives.Inc() + } + }(disk) + } + wg.Wait() + + parityDrives = int(atomicParityDrives.Load()) + if parityDrives >= len(storageDisks)/2 { + parityDrives = len(storageDisks) / 2 } if parityOrig != parityDrives { opts.UserDefined[minIOErasureUpgraded] = strconv.Itoa(parityOrig) + "->" + strconv.Itoa(parityDrives) diff --git a/go.mod b/go.mod index e56a6dc65..0a62d9342 100644 --- a/go.mod +++ b/go.mod @@ -80,6 +80,7 @@ require ( github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c go.etcd.io/etcd/api/v3 v3.5.0-beta.4 go.etcd.io/etcd/client/v3 v3.5.0-beta.4 + go.uber.org/atomic v1.7.0 go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b golang.org/x/sys v0.0.0-20210510120138-977fb7262007