trace: Add storage & OS tracing (#11889)

pkg/madmin/examples/service-trace.go

@@ -37,14 +37,15 @@ func main() {
 	if err != nil {
 		log.Fatalln(err)
 	}
-	doneCh := make(chan struct{})
-	defer close(doneCh)
 
-	// Start listening on all http trace activity from all servers
-	// in the minio cluster.
-	allTrace := false
-	errTrace := false
-	traceCh := madmClnt.ServiceTrace(context.Background(), allTrace, errTrace, doneCh)
+	// Start listening on all http trace activity from all servers in the minio cluster.
+	traceCh := madmClnt.ServiceTrace(context.Background(), madmin.ServiceTraceOpts{
+		S3:        true,
+		Internal:  true,
+		Storage:   true,
+		OS:        true,
+		Threshold: 0,
+	})
 	for traceInfo := range traceCh {
 		if traceInfo.Err != nil {
 			fmt.Println(traceInfo.Err)

pkg/madmin/service-commands.go

@@ -23,6 +23,7 @@ import (
 	"net/http"
 	"net/url"
 	"strconv"
+	"time"
 
 	trace "github.com/minio/minio/pkg/trace"
 )
@@ -77,16 +78,30 @@ type ServiceTraceInfo struct {
 	Err   error `json:"-"`
 }
 
+// ServiceTraceOpts holds tracing options
+type ServiceTraceOpts struct {
+	S3         bool
+	Internal   bool
+	Storage    bool
+	OS         bool
+	OnlyErrors bool
+	Threshold  time.Duration
+}
+
 // ServiceTrace - listen on http trace notifications.
-func (adm AdminClient) ServiceTrace(ctx context.Context, allTrace, errTrace bool) <-chan ServiceTraceInfo {
+func (adm AdminClient) ServiceTrace(ctx context.Context, opts ServiceTraceOpts) <-chan ServiceTraceInfo {
 	traceInfoCh := make(chan ServiceTraceInfo)
 	// Only success, start a routine to start reading line by line.
 	go func(traceInfoCh chan<- ServiceTraceInfo) {
 		defer close(traceInfoCh)
 		for {
 			urlValues := make(url.Values)
-			urlValues.Set("all", strconv.FormatBool(allTrace))
-			urlValues.Set("err", strconv.FormatBool(errTrace))
+			urlValues.Set("err", strconv.FormatBool(opts.OnlyErrors))
+			urlValues.Set("s3", strconv.FormatBool(opts.S3))
+			urlValues.Set("internal", strconv.FormatBool(opts.Internal))
+			urlValues.Set("storage", strconv.FormatBool(opts.Storage))
+			urlValues.Set("os", strconv.FormatBool(opts.OS))
+			urlValues.Set("threshold", opts.Threshold.String())
 			reqData := requestData{
 				relPath:     adminAPIPrefix + "/trace",
 				queryValues: urlValues,

pkg/trace/trace.go

@@ -21,14 +21,45 @@ import (
 	"time"
 )
 
+// Type indicates the type of the tracing Info
+type Type int
+
+const (
+	// OS tracing (Golang os package calls)
+	OS Type = iota
+	// Storage tracing (MinIO Storage Layer)
+	Storage
+	// HTTP tracing (MinIO S3 & Internode)
+	HTTP
+)
+
 // Info - represents a trace record, additionally
 // also reports errors if any while listening on trace.
 type Info struct {
-	NodeName  string       `json:"nodename"`
-	FuncName  string       `json:"funcname"`
+	TraceType Type `json:"type"`
+
+	NodeName string    `json:"nodename"`
+	FuncName string    `json:"funcname"`
+	Time     time.Time `json:"time"`
+
 	ReqInfo   RequestInfo  `json:"request"`
 	RespInfo  ResponseInfo `json:"response"`
 	CallStats CallStats    `json:"stats"`
+
+	StorageStats StorageStats `json:"storageStats"`
+	OSStats      OSStats      `json:"osStats"`
+}
+
+// StorageStats statistics on MinIO Storage layer calls
+type StorageStats struct {
+	Path     string        `json:"path"`
+	Duration time.Duration `json:"duration"`
+}
+
+// OSStats statistics on operating system specific calls.
+type OSStats struct {
+	Path     string        `json:"path"`
+	Duration time.Duration `json:"duration"`
 }
 
 // CallStats records request stats