diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index f9c17b7ce..b56b2bd93 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -1084,8 +1084,6 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http. return } - var bucketExists bool - sizeStr := r.Form.Get(peerRESTSize) durationStr := r.Form.Get(peerRESTDuration) concurrentStr := r.Form.Get(peerRESTConcurrent) @@ -1111,49 +1109,29 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http. duration = time.Second * 10 } - // ignores any errors here. - storageInfo, _ := objectAPI.StorageInfo(ctx) - capacityNeeded := uint64(concurrent * size) - capacity := uint64(GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo)) + sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, objectAPI, concurrent, size, autotune) - if capacity < capacityNeeded { + if !sufficientCapacity { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, AdminError{ - Code: "XMinioSpeedtestInsufficientCapacity", - Message: fmt.Sprintf("not enough usable space available to perform speedtest - expected %s, got %s", - humanize.IBytes(capacityNeeded), humanize.IBytes(capacity)), + Code: "XMinioSpeedtestInsufficientCapacity", + Message: capacityErrMsg, StatusCode: http.StatusInsufficientStorage, }), r.URL) return } - // Verify if we can employ autotune without running out of capacity, - // if we do run out of capacity, make sure to turn-off autotuning - // in such situations. - newConcurrent := concurrent + (concurrent+1)/2 - autoTunedCapacityNeeded := uint64(newConcurrent * size) - if autotune && capacity < autoTunedCapacityNeeded { - // Turn-off auto-tuning if next possible concurrency would reach beyond disk capacity. + if autotune && !canAutotune { autotune = false } - err = objectAPI.MakeBucketWithLocation(ctx, globalObjectPerfBucket, BucketOptions{}) + bucketExists, err := makeObjectPerfBucket(ctx, objectAPI) if err != nil { - if _, ok := err.(BucketExists); !ok { - // Only BucketExists error can be ignored. - writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) - return - } - bucketExists = true + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return } - deleteBucket := func() { - objectAPI.DeleteBucket(context.Background(), globalObjectPerfBucket, DeleteBucketOptions{ - Force: true, - NoRecreate: true, - }) - } if !bucketExists { - defer deleteBucket() + defer deleteObjectPerfBucket(objectAPI) } // Freeze all incoming S3 API calls before running speedtest. @@ -1166,7 +1144,7 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http. defer keepAliveTicker.Stop() enc := json.NewEncoder(w) - ch := speedTest(ctx, speedTestOpts{size, concurrent, duration, autotune, storageClass}) + ch := objectSpeedTest(ctx, speedTestOpts{size, concurrent, duration, autotune, storageClass}) for { select { case <-ctx.Done(): @@ -1189,6 +1167,50 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http. } } +func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer) (bucketExists bool, err error) { + err = objectAPI.MakeBucketWithLocation(ctx, globalObjectPerfBucket, BucketOptions{}) + if err != nil { + if _, ok := err.(BucketExists); !ok { + // Only BucketExists error can be ignored. + return false, err + } + bucketExists = true + } + return bucketExists, nil +} + +func deleteObjectPerfBucket(objectAPI ObjectLayer) { + objectAPI.DeleteBucket(context.Background(), globalObjectPerfBucket, DeleteBucketOptions{ + Force: true, + NoRecreate: true, + }) +} + +func validateObjPerfOptions(ctx context.Context, objectAPI ObjectLayer, concurrent int, size int, autotune bool) (sufficientCapacity bool, canAutotune bool, capacityErrMsg string) { + storageInfo, _ := objectAPI.StorageInfo(ctx) + capacityNeeded := uint64(concurrent * size) + capacity := uint64(GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo)) + + if capacity < capacityNeeded { + return false, false, fmt.Sprintf("not enough usable space available to perform speedtest - expected %s, got %s", + humanize.IBytes(capacityNeeded), humanize.IBytes(capacity)) + } + + // Verify if we can employ autotune without running out of capacity, + // if we do run out of capacity, make sure to turn-off autotuning + // in such situations. + if autotune { + newConcurrent := concurrent + (concurrent+1)/2 + autoTunedCapacityNeeded := uint64(newConcurrent * size) + if capacity < autoTunedCapacityNeeded { + // Turn-off auto-tuning if next possible concurrency would reach beyond disk capacity. + return true, false, "" + } + } + + return true, autotune, "" +} + // NetSpeedtestHandler - reports maximum network throughput func (a adminAPIHandlers) NetSpeedtestHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "NetSpeedtestHandler") @@ -1787,7 +1809,9 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque } deadline := 10 * time.Second // Default deadline is 10secs for health diagnostics. - if query.Get("perfnet") != "" || query.Get("perfdrive") != "" { + if query.Get(string(madmin.HealthDataTypePerfNet)) != "" || + query.Get(string(madmin.HealthDataTypePerfDrive)) != "" || + query.Get(string(madmin.HealthDataTypePerfObj)) != "" { deadline = 1 * time.Hour } if dstr := r.Form.Get("deadline"); dstr != "" { @@ -2032,49 +2056,98 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque } getAndWriteDrivePerfInfo := func() { - if query.Get("perfdrive") == "true" { - localDPI := getDrivePerfInfos(deadlinedCtx, globalLocalNodeName) - anonymizeAddr(&localDPI) - healthInfo.Perf.Drives = append(healthInfo.Perf.Drives, localDPI) - partialWrite(healthInfo) + if query.Get(string(madmin.HealthDataTypePerfDrive)) == "true" { + // Freeze all incoming S3 API calls before running speedtest. + globalNotificationSys.ServiceFreeze(ctx, true) + // unfreeze all incoming S3 API calls after speedtest. + defer globalNotificationSys.ServiceFreeze(ctx, false) - perfCh := globalNotificationSys.GetDrivePerfInfos(deadlinedCtx) - for perfInfo := range perfCh { - anonymizeAddr(&perfInfo) - healthInfo.Perf.Drives = append(healthInfo.Perf.Drives, perfInfo) - partialWrite(healthInfo) + opts := madmin.DriveSpeedTestOpts{ + Serial: false, + BlockSize: 4 * humanize.MiByte, + FileSize: 1 * humanize.GiByte, } + + localDPI := driveSpeedTest(ctx, opts) + healthInfo.Perf.DrivePerf = append(healthInfo.Perf.DrivePerf, localDPI) + + perfCh := globalNotificationSys.DriveSpeedTest(ctx, opts) + for perfInfo := range perfCh { + healthInfo.Perf.DrivePerf = append(healthInfo.Perf.DrivePerf, perfInfo) + } + partialWrite(healthInfo) } } - anonymizeNetPerfInfo := func(npi *madmin.NetPerfInfo) { - anonymizeAddr(npi) - rps := npi.RemotePeers - for idx, peer := range rps { - anonymizeAddr(&peer) - rps[idx] = peer + getAndWriteObjPerfInfo := func() { + if query.Get(string(madmin.HealthDataTypePerfObj)) == "true" { + concurrent := 32 + size := 64 * humanize.MiByte + autotune := true + + sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, objectAPI, concurrent, size, autotune) + + if !sufficientCapacity { + healthInfo.Perf.Error = capacityErrMsg + partialWrite(healthInfo) + return + } + + if !canAutotune { + autotune = false + } + + bucketExists, err := makeObjectPerfBucket(ctx, objectAPI) + if err != nil { + healthInfo.Perf.Error = "Could not make object perf bucket: " + err.Error() + partialWrite(healthInfo) + return + } + + if !bucketExists { + defer deleteObjectPerfBucket(objectAPI) + } + + // Freeze all incoming S3 API calls before running speedtest. + globalNotificationSys.ServiceFreeze(ctx, true) + + // unfreeze all incoming S3 API calls after speedtest. + defer globalNotificationSys.ServiceFreeze(ctx, false) + + opts := speedTestOpts{ + throughputSize: size, + concurrencyStart: concurrent, + duration: 10 * time.Second, + autotune: autotune, + } + + perfCh := objectSpeedTest(ctx, opts) + for perfInfo := range perfCh { + healthInfo.Perf.ObjPerf = append(healthInfo.Perf.ObjPerf, perfInfo) + } + partialWrite(healthInfo) } - npi.RemotePeers = rps } getAndWriteNetPerfInfo := func() { - if globalIsDistErasure && query.Get("perfnet") == "true" { - localNPI := globalNotificationSys.GetNetPerfInfo(deadlinedCtx) - anonymizeNetPerfInfo(&localNPI) - healthInfo.Perf.Net = append(healthInfo.Perf.Net, localNPI) - - partialWrite(healthInfo) - - netInfos := globalNotificationSys.DispatchNetPerfChan(deadlinedCtx) - for netInfo := range netInfos { - anonymizeNetPerfInfo(&netInfo) - healthInfo.Perf.Net = append(healthInfo.Perf.Net, netInfo) - partialWrite(healthInfo) + if query.Get(string(madmin.HealthDataTypePerfObj)) == "true" { + if !globalIsDistErasure { + return } - ppi := globalNotificationSys.GetParallelNetPerfInfo(deadlinedCtx) - anonymizeNetPerfInfo(&ppi) - healthInfo.Perf.NetParallel = ppi + nsLock := objectAPI.NewNSLock(minioMetaBucket, "netperf") + lkctx, err := nsLock.GetLock(ctx, globalOperationTimeout) + if err != nil { + healthInfo.Perf.Error = "Could not acquire lock for netperf: " + err.Error() + } else { + defer nsLock.Unlock(lkctx.Cancel) + + netPerf := globalNotificationSys.Netperf(ctx, time.Second*10) + for _, np := range netPerf { + np.Endpoint = anonAddr(np.Endpoint) + healthInfo.Perf.NetPerf = append(healthInfo.Perf.NetPerf, np) + } + } partialWrite(healthInfo) } } @@ -2108,6 +2181,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque getAndWriteProcInfo() getAndWriteMinioConfig() getAndWriteDrivePerfInfo() + getAndWriteObjPerfInfo() getAndWriteNetPerfInfo() getAndWriteSysErrors() getAndWriteSysServices() @@ -2640,6 +2714,7 @@ func anonymizeHost(hostAnonymizer map[string]string, endpoint Endpoint, poolNum if !found { // In distributed setup, anonymized addr = 'poolNum.serverNum' newHost := fmt.Sprintf("pool%d.server%d", poolNum, srvrNum) + schemePfx := endpoint.Scheme + "://" // Hostname mapIfNotPresent(hostAnonymizer, endpoint.Hostname(), newHost) @@ -2649,6 +2724,7 @@ func anonymizeHost(hostAnonymizer map[string]string, endpoint Endpoint, poolNum // Host + port newHostPort = newHost + ":" + endpoint.Port() mapIfNotPresent(hostAnonymizer, endpoint.Host, newHostPort) + mapIfNotPresent(hostAnonymizer, schemePfx+endpoint.Host, newHostPort) } newHostPortPath := newHostPort @@ -2657,10 +2733,11 @@ func anonymizeHost(hostAnonymizer map[string]string, endpoint Endpoint, poolNum currentHostPortPath := endpoint.Host + endpoint.Path newHostPortPath = newHostPort + endpoint.Path mapIfNotPresent(hostAnonymizer, currentHostPortPath, newHostPortPath) + mapIfNotPresent(hostAnonymizer, schemePfx+currentHostPortPath, newHostPortPath) } // Full url - hostAnonymizer[currentURL] = endpoint.Scheme + "://" + newHostPortPath + hostAnonymizer[currentURL] = schemePfx + newHostPortPath } } diff --git a/cmd/speedtest.go b/cmd/speedtest.go index f548d16cf..6d5d615b1 100644 --- a/cmd/speedtest.go +++ b/cmd/speedtest.go @@ -156,11 +156,6 @@ func objectSpeedTest(ctx context.Context, opts speedTestOpts) chan madmin.SpeedT return ch } -// speedTest - Deprecated. See objectSpeedTest -func speedTest(ctx context.Context, opts speedTestOpts) chan madmin.SpeedTestResult { - return objectSpeedTest(ctx, opts) -} - func driveSpeedTest(ctx context.Context, opts madmin.DriveSpeedTestOpts) madmin.DriveSpeedTestResult { perf := &dperf.DrivePerf{ Serial: opts.Serial, diff --git a/go.mod b/go.mod index 1d079f3b9..ecc68b82d 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,7 @@ require ( github.com/minio/dperf v0.3.5 github.com/minio/highwayhash v1.0.2 github.com/minio/kes v0.19.0 - github.com/minio/madmin-go v1.3.6 + github.com/minio/madmin-go v1.3.11 github.com/minio/minio-go/v7 v7.0.23 github.com/minio/parquet-go v1.1.0 github.com/minio/pkg v1.1.20 @@ -105,6 +105,7 @@ require ( github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect + github.com/briandowns/spinner v1.18.1 // indirect github.com/charmbracelet/bubbles v0.10.3 // indirect github.com/charmbracelet/bubbletea v0.20.0 // indirect github.com/charmbracelet/lipgloss v0.5.0 // indirect @@ -167,10 +168,9 @@ require ( github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect - github.com/minio/argon2 v1.0.0 // indirect github.com/minio/colorjson v1.0.1 // indirect github.com/minio/filepath v1.0.0 // indirect - github.com/minio/mc v0.0.0-20220302011226-f13defa54577 // indirect + github.com/minio/mc v0.0.0-20220407151251-dbc09a8bf054 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect diff --git a/go.sum b/go.sum index 3f3178df0..e2b9ffc09 100644 --- a/go.sum +++ b/go.sum @@ -139,6 +139,8 @@ github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edY github.com/bits-and-blooms/bloom/v3 v3.0.1 h1:Inlf0YXbgehxVjMPmCGv86iMCKMGPPrPSHtBF5yRHwA= github.com/bits-and-blooms/bloom/v3 v3.0.1/go.mod h1:MC8muvBzzPOFsrcdND/A7kU7kMhkqb9KI70JlZCP+C8= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/briandowns/spinner v1.18.1 h1:yhQmQtM1zsqFsouh09Bk/jCjd50pC3EOGsh28gLVvwY= +github.com/briandowns/spinner v1.18.1/go.mod h1:mQak9GHqbspjC/5iUx3qMlIho8xBS/ppAL/hX5SmPJU= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= @@ -657,8 +659,6 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.1.46 h1:uzwpxRtSVxtcIZmz/4Uz6/Rn7G11DvsaslXoy5LxQio= github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= -github.com/minio/argon2 v1.0.0 h1:cLB/fl0EeBqiDYhsIzIPTdLZhCykRrvdx3Eu3E5oqsE= -github.com/minio/argon2 v1.0.0/go.mod h1:XtOGJ7MjwUJDPtCqqrisx5QwVB/jDx+adQHigJVsQHQ= github.com/minio/cli v1.22.0 h1:VTQm7lmXm3quxO917X3p+el1l0Ca5X3S4PM2ruUYO68= github.com/minio/cli v1.22.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY= github.com/minio/colorjson v1.0.1 h1:+hvfP8C1iMB95AT+ZFDRE+Knn9QPd9lg0CRJY9DRpos= @@ -676,10 +676,10 @@ github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/kes v0.19.0 h1:rKzkDXT4ay7FBW34KgXK+y85bie4x4Oiq29ONRuMzh0= github.com/minio/kes v0.19.0/go.mod h1:e9YGKbwFCV7LbqNPMfZBazfNUsFGJ5LG4plSeWL8mmg= -github.com/minio/madmin-go v1.3.6 h1:lgAIfOI1xB+K2JFx5JqxklLS0SE7ya3qZIi1qmwjHNE= -github.com/minio/madmin-go v1.3.6/go.mod h1:vGKGboQgGIWx4DuDUaXixjlIEZOCIp6ivJkQoiVaACc= -github.com/minio/mc v0.0.0-20220302011226-f13defa54577 h1:ZWkSq/Ofoz7fX4syAscCg66Ie6oV5plYFTXAieYRRDg= -github.com/minio/mc v0.0.0-20220302011226-f13defa54577/go.mod h1:lomZcm7bdn9Og7UBKBqu//8YPmWAhhMNlnkIAHmwcS8= +github.com/minio/madmin-go v1.3.11 h1:Cj02kzG2SD1pnZW2n1joe00yqb6NFE40Jt2gp+5mWFQ= +github.com/minio/madmin-go v1.3.11/go.mod h1:ez87VmMtsxP7DRxjKJKD4RDNW+nhO2QF9KSzwxBDQ98= +github.com/minio/mc v0.0.0-20220407151251-dbc09a8bf054 h1:Od5VqIS3Z0U2A3tiyLyfEtBcby3Jx/8BSFj7U+K85E4= +github.com/minio/mc v0.0.0-20220407151251-dbc09a8bf054/go.mod h1:PIQHcb4uOctKyL/y78tVvC0PCSk6q9UavbSkIAB2/FQ= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=