implement configurable timeout for OBD tests (#9324)

cmd/admin-handlers.go

@@ -1173,22 +1173,23 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 	if objectAPI == nil {
 		return
 	}
-	deadlinedCtx, cancel := context.WithDeadline(ctx, time.Now().Add(10*time.Minute))
+
+	vars := mux.Vars(r)
+	pulse := make(chan struct{})
+	obdDone := make(chan struct{})
 	obdInfo := madmin.OBDInfo{}
 
-	setCommonHeaders(w)
-	w.Header().Set(xhttp.ContentType, string(mimeJSON))
-	w.WriteHeader(http.StatusOK)
-
 	enc := json.NewEncoder(w)
-	partialWrite := func() {
+	doPartialWrite := func() {
 		logger.LogIf(ctx, enc.Encode(obdInfo))
 	}
 
+	partialWrite := func() {
+		pulse <- struct{}{}
+	}
+
 	finish := func() {
-		partialWrite()
+		obdDone <- struct{}{}
-		w.(http.Flusher).Flush()
-		cancel()
 	}
 
 	errResp := func(err error) {
@@ -1199,15 +1200,48 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 		finish()
 	}
 
+	deadline := 3600 * time.Second
+	deadlineStr := r.URL.Query().Get("deadline")
+	if deadlineStr != "" {
+		var err error
+		deadline, err = time.ParseDuration(deadlineStr)
+		if err != nil {
+			errResp(err)
+			return
+		}
+	}
+	deadlinedCtx, cancel := context.WithDeadline(ctx, time.Now().Add(deadline))
+
+	setCommonHeaders(w)
+	w.Header().Set(xhttp.ContentType, string(mimeJSON))
+	w.WriteHeader(http.StatusOK)
+
+	ticker := time.NewTicker(30 * time.Second)
+	defer ticker.Stop()
+
+	go func() {
+	loop:
+		for {
+			select {
+			case <-ticker.C:
+				doPartialWrite()
+			case <-pulse:
+				doPartialWrite()
+			case <-obdDone:
+				break loop
+			}
+		}
+		w.(http.Flusher).Flush()
+		cancel()
+	}()
+
 	nsLock := objectAPI.NewNSLock(deadlinedCtx, minioMetaBucket, "obd-in-progress")
-	if err := nsLock.GetLock(newDynamicTimeout(10*time.Minute, 600*time.Second)); err != nil { // returns a locked lock
+	if err := nsLock.GetLock(newDynamicTimeout(deadline, deadline)); err != nil { // returns a locked lock
 		errResp(err)
 		return
 	}
 	defer nsLock.Unlock()
 
-	vars := mux.Vars(r)
-
 	if cpu, ok := vars["syscpu"]; ok && cpu == "true" {
 		cpuInfo := getLocalCPUOBDInfo(deadlinedCtx)
 

cmd/admin-router.go

@@ -178,9 +178,19 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/kms/key/status").HandlerFunc(httpTraceAll(adminAPI.KMSKeyStatusHandler))
 
 		if !globalIsGateway {
-
 			// -- OBD API --
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/obdinfo").HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler)).Queries("perfdrive", "{perfdrive:true|false}", "perfnet", "{perfnet:true|false}", "minioinfo", "{minioinfo:true|false}", "minioconfig", "{minioconfig:true|false}", "syscpu", "{syscpu:true|false}", "sysdiskhw", "{sysdiskhw:true|false}", "sysosinfo", "{sysosinfo:true|false}", "sysmem", "{sysmem:true|false}", "sysprocess", "{sysprocess:true|false}")
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/obdinfo").
+				HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler)).
+				Queries("perfdrive", "{perfdrive:true|false}",
+					"perfnet", "{perfnet:true|false}",
+					"minioinfo", "{minioinfo:true|false}",
+					"minioconfig", "{minioconfig:true|false}",
+					"syscpu", "{syscpu:true|false}",
+					"sysdiskhw", "{sysdiskhw:true|false}",
+					"sysosinfo", "{sysosinfo:true|false}",
+					"sysmem", "{sysmem:true|false}",
+					"sysprocess", "{sysprocess:true|false}",
+				)
 		}
 	}
 

pkg/disk/obd.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"runtime"
 	"time"
 
 	"github.com/dustin/go-humanize"
@@ -52,8 +51,7 @@ type Throughput struct {
 }
 
 // GetOBDInfo about the drive
-func GetOBDInfo(ctx context.Context, drive string, fsPath string) (Latency, Throughput, error) {
+func GetOBDInfo(ctx context.Context, drive, fsPath string) (Latency, Throughput, error) {
-	runtime.LockOSThread()
 
 	// Create a file with O_DIRECT flag, choose default umask and also make sure
 	// we are exclusively writing to a new file using O_EXCL.
@@ -96,8 +94,6 @@ func GetOBDInfo(ctx context.Context, drive string, fsPath string) (Latency, Thro
 		latencies[i] = float64(latencyInSecs)
 	}
 
-	runtime.UnlockOSThread()
-
 	for i := range latencies {
 		throughput := float64(blockSize) / latencies[i]
 		throughputs[i] = throughput

pkg/madmin/obd.go

@@ -221,11 +221,14 @@ var OBDDataTypesList = []OBDDataType{
 
 // ServerOBDInfo - Connect to a minio server and call OBD Info Management API
 // to fetch server's information represented by OBDInfo structure
-func (adm *AdminClient) ServerOBDInfo(ctx context.Context, obdDataTypes []OBDDataType) <-chan OBDInfo {
+func (adm *AdminClient) ServerOBDInfo(ctx context.Context, obdDataTypes []OBDDataType, deadline time.Duration) <-chan OBDInfo {
 	respChan := make(chan OBDInfo)
 	go func() {
 		v := url.Values{}
 
+		v.Set("deadline",
+			deadline.Truncate(1*time.Second).String())
+
 		// start with all set to false
 		for _, d := range OBDDataTypesList {
 			v.Set(string(d), "false")