From 5c53620a7299bac5a1f2a1e0d9f452ba562d665a Mon Sep 17 00:00:00 2001 From: Shireesh Anjal <355479+anjalshireesh@users.noreply.github.com> Date: Wed, 13 Apr 2022 01:47:44 +0530 Subject: [PATCH] Include speedtest as part of healthinfo api (#14696) Execute the object, drive and net speedtests as part of the healthinfo (if requested by the client), and include their result in the response. The options for the speedtests have been picked from the default values used by `mc support perf` command. --- cmd/admin-handlers.go | 209 +++++++++++++++++++++++++++++------------- cmd/speedtest.go | 5 - go.mod | 6 +- go.sum | 12 +-- 4 files changed, 152 insertions(+), 80 deletions(-) diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index f9c17b7ce..b56b2bd93 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -1084,8 +1084,6 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http. return } - var bucketExists bool - sizeStr := r.Form.Get(peerRESTSize) durationStr := r.Form.Get(peerRESTDuration) concurrentStr := r.Form.Get(peerRESTConcurrent) @@ -1111,49 +1109,29 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http. duration = time.Second * 10 } - // ignores any errors here. - storageInfo, _ := objectAPI.StorageInfo(ctx) - capacityNeeded := uint64(concurrent * size) - capacity := uint64(GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo)) + sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, objectAPI, concurrent, size, autotune) - if capacity < capacityNeeded { + if !sufficientCapacity { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, AdminError{ - Code: "XMinioSpeedtestInsufficientCapacity", - Message: fmt.Sprintf("not enough usable space available to perform speedtest - expected %s, got %s", - humanize.IBytes(capacityNeeded), humanize.IBytes(capacity)), + Code: "XMinioSpeedtestInsufficientCapacity", + Message: capacityErrMsg, StatusCode: http.StatusInsufficientStorage, }), r.URL) return } - // Verify if we can employ autotune without running out of capacity, - // if we do run out of capacity, make sure to turn-off autotuning - // in such situations. - newConcurrent := concurrent + (concurrent+1)/2 - autoTunedCapacityNeeded := uint64(newConcurrent * size) - if autotune && capacity < autoTunedCapacityNeeded { - // Turn-off auto-tuning if next possible concurrency would reach beyond disk capacity. + if autotune && !canAutotune { autotune = false } - err = objectAPI.MakeBucketWithLocation(ctx, globalObjectPerfBucket, BucketOptions{}) + bucketExists, err := makeObjectPerfBucket(ctx, objectAPI) if err != nil { - if _, ok := err.(BucketExists); !ok { - // Only BucketExists error can be ignored. - writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) - return - } - bucketExists = true + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return } - deleteBucket := func() { - objectAPI.DeleteBucket(context.Background(), globalObjectPerfBucket, DeleteBucketOptions{ - Force: true, - NoRecreate: true, - }) - } if !bucketExists { - defer deleteBucket() + defer deleteObjectPerfBucket(objectAPI) } // Freeze all incoming S3 API calls before running speedtest. @@ -1166,7 +1144,7 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http. defer keepAliveTicker.Stop() enc := json.NewEncoder(w) - ch := speedTest(ctx, speedTestOpts{size, concurrent, duration, autotune, storageClass}) + ch := objectSpeedTest(ctx, speedTestOpts{size, concurrent, duration, autotune, storageClass}) for { select { case <-ctx.Done(): @@ -1189,6 +1167,50 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http. } } +func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer) (bucketExists bool, err error) { + err = objectAPI.MakeBucketWithLocation(ctx, globalObjectPerfBucket, BucketOptions{}) + if err != nil { + if _, ok := err.(BucketExists); !ok { + // Only BucketExists error can be ignored. + return false, err + } + bucketExists = true + } + return bucketExists, nil +} + +func deleteObjectPerfBucket(objectAPI ObjectLayer) { + objectAPI.DeleteBucket(context.Background(), globalObjectPerfBucket, DeleteBucketOptions{ + Force: true, + NoRecreate: true, + }) +} + +func validateObjPerfOptions(ctx context.Context, objectAPI ObjectLayer, concurrent int, size int, autotune bool) (sufficientCapacity bool, canAutotune bool, capacityErrMsg string) { + storageInfo, _ := objectAPI.StorageInfo(ctx) + capacityNeeded := uint64(concurrent * size) + capacity := uint64(GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo)) + + if capacity < capacityNeeded { + return false, false, fmt.Sprintf("not enough usable space available to perform speedtest - expected %s, got %s", + humanize.IBytes(capacityNeeded), humanize.IBytes(capacity)) + } + + // Verify if we can employ autotune without running out of capacity, + // if we do run out of capacity, make sure to turn-off autotuning + // in such situations. + if autotune { + newConcurrent := concurrent + (concurrent+1)/2 + autoTunedCapacityNeeded := uint64(newConcurrent * size) + if capacity < autoTunedCapacityNeeded { + // Turn-off auto-tuning if next possible concurrency would reach beyond disk capacity. + return true, false, "" + } + } + + return true, autotune, "" +} + // NetSpeedtestHandler - reports maximum network throughput func (a adminAPIHandlers) NetSpeedtestHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "NetSpeedtestHandler") @@ -1787,7 +1809,9 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque } deadline := 10 * time.Second // Default deadline is 10secs for health diagnostics. - if query.Get("perfnet") != "" || query.Get("perfdrive") != "" { + if query.Get(string(madmin.HealthDataTypePerfNet)) != "" || + query.Get(string(madmin.HealthDataTypePerfDrive)) != "" || + query.Get(string(madmin.HealthDataTypePerfObj)) != "" { deadline = 1 * time.Hour } if dstr := r.Form.Get("deadline"); dstr != "" { @@ -2032,49 +2056,98 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque } getAndWriteDrivePerfInfo := func() { - if query.Get("perfdrive") == "true" { - localDPI := getDrivePerfInfos(deadlinedCtx, globalLocalNodeName) - anonymizeAddr(&localDPI) - healthInfo.Perf.Drives = append(healthInfo.Perf.Drives, localDPI) - partialWrite(healthInfo) + if query.Get(string(madmin.HealthDataTypePerfDrive)) == "true" { + // Freeze all incoming S3 API calls before running speedtest. + globalNotificationSys.ServiceFreeze(ctx, true) + // unfreeze all incoming S3 API calls after speedtest. + defer globalNotificationSys.ServiceFreeze(ctx, false) - perfCh := globalNotificationSys.GetDrivePerfInfos(deadlinedCtx) - for perfInfo := range perfCh { - anonymizeAddr(&perfInfo) - healthInfo.Perf.Drives = append(healthInfo.Perf.Drives, perfInfo) - partialWrite(healthInfo) + opts := madmin.DriveSpeedTestOpts{ + Serial: false, + BlockSize: 4 * humanize.MiByte, + FileSize: 1 * humanize.GiByte, } + + localDPI := driveSpeedTest(ctx, opts) + healthInfo.Perf.DrivePerf = append(healthInfo.Perf.DrivePerf, localDPI) + + perfCh := globalNotificationSys.DriveSpeedTest(ctx, opts) + for perfInfo := range perfCh { + healthInfo.Perf.DrivePerf = append(healthInfo.Perf.DrivePerf, perfInfo) + } + partialWrite(healthInfo) } } - anonymizeNetPerfInfo := func(npi *madmin.NetPerfInfo) { - anonymizeAddr(npi) - rps := npi.RemotePeers - for idx, peer := range rps { - anonymizeAddr(&peer) - rps[idx] = peer + getAndWriteObjPerfInfo := func() { + if query.Get(string(madmin.HealthDataTypePerfObj)) == "true" { + concurrent := 32 + size := 64 * humanize.MiByte + autotune := true + + sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, objectAPI, concurrent, size, autotune) + + if !sufficientCapacity { + healthInfo.Perf.Error = capacityErrMsg + partialWrite(healthInfo) + return + } + + if !canAutotune { + autotune = false + } + + bucketExists, err := makeObjectPerfBucket(ctx, objectAPI) + if err != nil { + healthInfo.Perf.Error = "Could not make object perf bucket: " + err.Error() + partialWrite(healthInfo) + return + } + + if !bucketExists { + defer deleteObjectPerfBucket(objectAPI) + } + + // Freeze all incoming S3 API calls before running speedtest. + globalNotificationSys.ServiceFreeze(ctx, true) + + // unfreeze all incoming S3 API calls after speedtest. + defer globalNotificationSys.ServiceFreeze(ctx, false) + + opts := speedTestOpts{ + throughputSize: size, + concurrencyStart: concurrent, + duration: 10 * time.Second, + autotune: autotune, + } + + perfCh := objectSpeedTest(ctx, opts) + for perfInfo := range perfCh { + healthInfo.Perf.ObjPerf = append(healthInfo.Perf.ObjPerf, perfInfo) + } + partialWrite(healthInfo) } - npi.RemotePeers = rps } getAndWriteNetPerfInfo := func() { - if globalIsDistErasure && query.Get("perfnet") == "true" { - localNPI := globalNotificationSys.GetNetPerfInfo(deadlinedCtx) - anonymizeNetPerfInfo(&localNPI) - healthInfo.Perf.Net = append(healthInfo.Perf.Net, localNPI) - - partialWrite(healthInfo) - - netInfos := globalNotificationSys.DispatchNetPerfChan(deadlinedCtx) - for netInfo := range netInfos { - anonymizeNetPerfInfo(&netInfo) - healthInfo.Perf.Net = append(healthInfo.Perf.Net, netInfo) - partialWrite(healthInfo) + if query.Get(string(madmin.HealthDataTypePerfObj)) == "true" { + if !globalIsDistErasure { + return } - ppi := globalNotificationSys.GetParallelNetPerfInfo(deadlinedCtx) - anonymizeNetPerfInfo(&ppi) - healthInfo.Perf.NetParallel = ppi + nsLock := objectAPI.NewNSLock(minioMetaBucket, "netperf") + lkctx, err := nsLock.GetLock(ctx, globalOperationTimeout) + if err != nil { + healthInfo.Perf.Error = "Could not acquire lock for netperf: " + err.Error() + } else { + defer nsLock.Unlock(lkctx.Cancel) + + netPerf := globalNotificationSys.Netperf(ctx, time.Second*10) + for _, np := range netPerf { + np.Endpoint = anonAddr(np.Endpoint) + healthInfo.Perf.NetPerf = append(healthInfo.Perf.NetPerf, np) + } + } partialWrite(healthInfo) } } @@ -2108,6 +2181,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque getAndWriteProcInfo() getAndWriteMinioConfig() getAndWriteDrivePerfInfo() + getAndWriteObjPerfInfo() getAndWriteNetPerfInfo() getAndWriteSysErrors() getAndWriteSysServices() @@ -2640,6 +2714,7 @@ func anonymizeHost(hostAnonymizer map[string]string, endpoint Endpoint, poolNum if !found { // In distributed setup, anonymized addr = 'poolNum.serverNum' newHost := fmt.Sprintf("pool%d.server%d", poolNum, srvrNum) + schemePfx := endpoint.Scheme + "://" // Hostname mapIfNotPresent(hostAnonymizer, endpoint.Hostname(), newHost) @@ -2649,6 +2724,7 @@ func anonymizeHost(hostAnonymizer map[string]string, endpoint Endpoint, poolNum // Host + port newHostPort = newHost + ":" + endpoint.Port() mapIfNotPresent(hostAnonymizer, endpoint.Host, newHostPort) + mapIfNotPresent(hostAnonymizer, schemePfx+endpoint.Host, newHostPort) } newHostPortPath := newHostPort @@ -2657,10 +2733,11 @@ func anonymizeHost(hostAnonymizer map[string]string, endpoint Endpoint, poolNum currentHostPortPath := endpoint.Host + endpoint.Path newHostPortPath = newHostPort + endpoint.Path mapIfNotPresent(hostAnonymizer, currentHostPortPath, newHostPortPath) + mapIfNotPresent(hostAnonymizer, schemePfx+currentHostPortPath, newHostPortPath) } // Full url - hostAnonymizer[currentURL] = endpoint.Scheme + "://" + newHostPortPath + hostAnonymizer[currentURL] = schemePfx + newHostPortPath } } diff --git a/cmd/speedtest.go b/cmd/speedtest.go index f548d16cf..6d5d615b1 100644 --- a/cmd/speedtest.go +++ b/cmd/speedtest.go @@ -156,11 +156,6 @@ func objectSpeedTest(ctx context.Context, opts speedTestOpts) chan madmin.SpeedT return ch } -// speedTest - Deprecated. See objectSpeedTest -func speedTest(ctx context.Context, opts speedTestOpts) chan madmin.SpeedTestResult { - return objectSpeedTest(ctx, opts) -} - func driveSpeedTest(ctx context.Context, opts madmin.DriveSpeedTestOpts) madmin.DriveSpeedTestResult { perf := &dperf.DrivePerf{ Serial: opts.Serial, diff --git a/go.mod b/go.mod index 1d079f3b9..ecc68b82d 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,7 @@ require ( github.com/minio/dperf v0.3.5 github.com/minio/highwayhash v1.0.2 github.com/minio/kes v0.19.0 - github.com/minio/madmin-go v1.3.6 + github.com/minio/madmin-go v1.3.11 github.com/minio/minio-go/v7 v7.0.23 github.com/minio/parquet-go v1.1.0 github.com/minio/pkg v1.1.20 @@ -105,6 +105,7 @@ require ( github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect + github.com/briandowns/spinner v1.18.1 // indirect github.com/charmbracelet/bubbles v0.10.3 // indirect github.com/charmbracelet/bubbletea v0.20.0 // indirect github.com/charmbracelet/lipgloss v0.5.0 // indirect @@ -167,10 +168,9 @@ require ( github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect - github.com/minio/argon2 v1.0.0 // indirect github.com/minio/colorjson v1.0.1 // indirect github.com/minio/filepath v1.0.0 // indirect - github.com/minio/mc v0.0.0-20220302011226-f13defa54577 // indirect + github.com/minio/mc v0.0.0-20220407151251-dbc09a8bf054 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect diff --git a/go.sum b/go.sum index 3f3178df0..e2b9ffc09 100644 --- a/go.sum +++ b/go.sum @@ -139,6 +139,8 @@ github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edY github.com/bits-and-blooms/bloom/v3 v3.0.1 h1:Inlf0YXbgehxVjMPmCGv86iMCKMGPPrPSHtBF5yRHwA= github.com/bits-and-blooms/bloom/v3 v3.0.1/go.mod h1:MC8muvBzzPOFsrcdND/A7kU7kMhkqb9KI70JlZCP+C8= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/briandowns/spinner v1.18.1 h1:yhQmQtM1zsqFsouh09Bk/jCjd50pC3EOGsh28gLVvwY= +github.com/briandowns/spinner v1.18.1/go.mod h1:mQak9GHqbspjC/5iUx3qMlIho8xBS/ppAL/hX5SmPJU= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= @@ -657,8 +659,6 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.1.46 h1:uzwpxRtSVxtcIZmz/4Uz6/Rn7G11DvsaslXoy5LxQio= github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= -github.com/minio/argon2 v1.0.0 h1:cLB/fl0EeBqiDYhsIzIPTdLZhCykRrvdx3Eu3E5oqsE= -github.com/minio/argon2 v1.0.0/go.mod h1:XtOGJ7MjwUJDPtCqqrisx5QwVB/jDx+adQHigJVsQHQ= github.com/minio/cli v1.22.0 h1:VTQm7lmXm3quxO917X3p+el1l0Ca5X3S4PM2ruUYO68= github.com/minio/cli v1.22.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY= github.com/minio/colorjson v1.0.1 h1:+hvfP8C1iMB95AT+ZFDRE+Knn9QPd9lg0CRJY9DRpos= @@ -676,10 +676,10 @@ github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/kes v0.19.0 h1:rKzkDXT4ay7FBW34KgXK+y85bie4x4Oiq29ONRuMzh0= github.com/minio/kes v0.19.0/go.mod h1:e9YGKbwFCV7LbqNPMfZBazfNUsFGJ5LG4plSeWL8mmg= -github.com/minio/madmin-go v1.3.6 h1:lgAIfOI1xB+K2JFx5JqxklLS0SE7ya3qZIi1qmwjHNE= -github.com/minio/madmin-go v1.3.6/go.mod h1:vGKGboQgGIWx4DuDUaXixjlIEZOCIp6ivJkQoiVaACc= -github.com/minio/mc v0.0.0-20220302011226-f13defa54577 h1:ZWkSq/Ofoz7fX4syAscCg66Ie6oV5plYFTXAieYRRDg= -github.com/minio/mc v0.0.0-20220302011226-f13defa54577/go.mod h1:lomZcm7bdn9Og7UBKBqu//8YPmWAhhMNlnkIAHmwcS8= +github.com/minio/madmin-go v1.3.11 h1:Cj02kzG2SD1pnZW2n1joe00yqb6NFE40Jt2gp+5mWFQ= +github.com/minio/madmin-go v1.3.11/go.mod h1:ez87VmMtsxP7DRxjKJKD4RDNW+nhO2QF9KSzwxBDQ98= +github.com/minio/mc v0.0.0-20220407151251-dbc09a8bf054 h1:Od5VqIS3Z0U2A3tiyLyfEtBcby3Jx/8BSFj7U+K85E4= +github.com/minio/mc v0.0.0-20220407151251-dbc09a8bf054/go.mod h1:PIQHcb4uOctKyL/y78tVvC0PCSk6q9UavbSkIAB2/FQ= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=