Add admin API to send trace notifications to registered (#7128)

Remove current functionality to log trace to file
using MINIO_HTTP_TRACE env, and replace it with
mc admin trace command on mc client.

cmd/admin-handlers.go

@@ -1419,3 +1419,40 @@ func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Re
 	// Send success response
 	writeSuccessResponseHeadersOnly(w)
 }
+
+// TraceHandler - POST /minio/admin/v1/trace
+// ----------
+// The handler sends http trace to the connected HTTP client.
+func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "HTTPTrace")
+	trcAll := r.URL.Query().Get("all") == "true"
+	objectAPI := validateAdminReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+	// Avoid reusing tcp connection if read timeout is hit
+	// This is needed to make r.Context().Done() work as
+	// expected in case of read timeout
+	w.Header().Add("Connection", "close")
+
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+
+	traceCh := globalTrace.Trace(doneCh, trcAll)
+	for {
+		select {
+		case entry := <-traceCh:
+			if _, err := w.Write(entry); err != nil {
+				return
+			}
+			if _, err := w.Write([]byte("\n")); err != nil {
+				return
+			}
+			w.(http.Flusher).Flush()
+		case <-r.Context().Done():
+			return
+		case <-GlobalServiceDoneCh:
+			return
+		}
+	}
+}

cmd/admin-router.go

@@ -116,6 +116,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 	// Top locks
 	adminV1Router.Methods(http.MethodGet).Path("/top/locks").HandlerFunc(httpTraceHdrs(adminAPI.TopLocksHandler))
 
+	// HTTP Trace
+	adminV1Router.Methods(http.MethodGet).Path("/trace").HandlerFunc(adminAPI.TraceHandler)
 	// If none of the routes match, return error.
 	adminV1Router.NotFoundHandler = http.HandlerFunc(httpTraceHdrs(notFoundHandlerJSON))
 }

cmd/common-main.go

@@ -211,13 +211,6 @@ func handleCommonEnvVars() {
 		globalIsBrowserEnabled = bool(browserFlag)
 	}
 
-	traceFile := os.Getenv("MINIO_HTTP_TRACE")
-	if traceFile != "" {
-		var err error
-		globalHTTPTraceFile, err = os.OpenFile(traceFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0660)
-		logger.FatalIf(err, "error opening file %s", traceFile)
-	}
-
 	etcdEndpointsEnv, ok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
 	if ok {
 		etcdEndpoints := strings.Split(etcdEndpointsEnv, ",")

cmd/gateway-main.go

@@ -158,6 +158,9 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		registerSTSRouter(router)
 	}
 
+	// initialize globalTrace system
+	globalTrace = NewTraceSys(context.Background(), globalEndpoints)
+
 	enableConfigOps := globalEtcdClient != nil && gatewayName == "nas"
 	enableIAMOps := globalEtcdClient != nil
 

cmd/globals.go

@@ -159,8 +159,9 @@ var (
 	globalHTTPServerErrorCh = make(chan error)
 	globalOSSignalCh        = make(chan os.Signal, 1)
 
-	// File to log HTTP request/response headers and body.
+	// global Trace system to send HTTP request/response logs to
-	globalHTTPTraceFile *os.File
+	// registered listeners
+	globalTrace *HTTPTraceSys
 
 	globalEndpoints EndpointList
 

cmd/handler-utils.go

@@ -30,7 +30,6 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/handlers"
-	httptracer "github.com/minio/minio/pkg/handlers"
 )
 
 // Parses location constraint from the incoming reader.
@@ -326,18 +325,26 @@ func extractPostPolicyFormValues(ctx context.Context, form *multipart.Form) (fil
 
 // Log headers and body.
 func httpTraceAll(f http.HandlerFunc) http.HandlerFunc {
-	if globalHTTPTraceFile == nil {
+	return func(w http.ResponseWriter, r *http.Request) {
-		return f
+		if !globalTrace.HasTraceListeners() {
+			f.ServeHTTP(w, r)
+			return
+		}
+		trace := Trace(f, true, w, r)
+		globalTrace.Publish(trace)
 	}
-	return httptracer.TraceReqHandlerFunc(f, globalHTTPTraceFile, true)
 }
 
 // Log only the headers.
 func httpTraceHdrs(f http.HandlerFunc) http.HandlerFunc {
-	if globalHTTPTraceFile == nil {
+	return func(w http.ResponseWriter, r *http.Request) {
-		return f
+		if !globalTrace.HasTraceListeners() {
+			f.ServeHTTP(w, r)
+			return
+		}
+		trace := Trace(f, false, w, r)
+		globalTrace.Publish(trace)
 	}
-	return httptracer.TraceReqHandlerFunc(f, globalHTTPTraceFile, false)
 }
 
 // Returns "/bucketName/objectName" for path-style or virtual-host-style requests.

cmd/http-tracer.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package handlers
+package cmd
 
 import (
 	"bytes"
@@ -24,8 +24,12 @@ import (
 	"net/http"
 	"reflect"
 	"runtime"
+	"strconv"
 	"strings"
 	"time"
+
+	xnet "github.com/minio/minio/pkg/net"
+	trace "github.com/minio/minio/pkg/trace"
 )
 
 // recordRequest - records the first recLen bytes
@@ -109,92 +113,64 @@ func (r *recordResponseWriter) Flush() {
 	r.ResponseWriter.(http.Flusher).Flush()
 }
 
-// Return response headers.
-func (r *recordResponseWriter) Headers() []byte {
-	return r.headers.Bytes()
-}
-
 // Return response body.
 func (r *recordResponseWriter) Body() []byte {
 	return r.body.Bytes()
 }
 
-// TraceReqHandlerFunc logs request/response headers and body.
+// Trace gets trace of http request
-func TraceReqHandlerFunc(f http.HandlerFunc, output io.Writer, logBody bool) http.HandlerFunc {
+func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Request) trace.Info {
+
 	name := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
 	name = strings.TrimPrefix(name, "github.com/minio/minio/cmd.")
+	name = strings.TrimSuffix(name, "Handler-fm")
+
 	bodyPlaceHolder := []byte("<BODY>")
+	var reqBodyRecorder *recordRequest
 
-	return func(w http.ResponseWriter, r *http.Request) {
+	t := trace.Info{FuncName: name}
-		const timeFormat = "2006-01-02 15:04:05 -0700"
+	reqBodyRecorder = &recordRequest{Reader: r.Body, logBody: logBody}
-		var reqBodyRecorder *recordRequest
+	r.Body = ioutil.NopCloser(reqBodyRecorder)
 
-		// Generate short random request ID
+	host, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
-		reqID := fmt.Sprintf("%f", float64(time.Now().UnixNano())/1e10)
+	if err == nil {
-
+		t.NodeName = host.Name
-		reqBodyRecorder = &recordRequest{Reader: r.Body, logBody: logBody}
-		r.Body = ioutil.NopCloser(reqBodyRecorder)
-
-		// Setup a http response body recorder
-		respBodyRecorder := &recordResponseWriter{ResponseWriter: w, logBody: logBody}
-
-		b := bytes.NewBuffer(nil)
-		fmt.Fprintf(b, "[REQUEST %s] [%s] [%s]\n", name, reqID, time.Now().Format(timeFormat))
-
-		f(respBodyRecorder, r)
-
-		// Build request log and write it to log file
-		fmt.Fprintf(b, "%s %s", r.Method, r.URL.Path)
-		if r.URL.RawQuery != "" {
-			fmt.Fprintf(b, "?%s", r.URL.RawQuery)
-		}
-		fmt.Fprintf(b, "\n")
-
-		fmt.Fprintf(b, "Host: %s\n", r.Host)
-		for k, v := range r.Header {
-			fmt.Fprintf(b, "%s: %s\n", k, v[0])
-		}
-		fmt.Fprintf(b, "\n")
-		if logBody {
-			bodyContents := reqBodyRecorder.Data()
-			if bodyContents != nil {
-				// If body logging is disabled then we print <BODY> as a placeholder
-				// for the actual body.
-				b.Write(bodyContents)
-				fmt.Fprintf(b, "\n")
-			}
-		} else {
-			b.Write(bodyPlaceHolder)
-			fmt.Fprintf(b, "\n")
-		}
-
-		fmt.Fprintf(b, "\n")
-
-		// Build response log and write it to log file
-		fmt.Fprintf(b, "[RESPONSE] [%s] [%s]\n", reqID, time.Now().Format(timeFormat))
-
-		b.Write(respBodyRecorder.Headers())
-		fmt.Fprintf(b, "\n")
-
-		// recordResponseWriter{} is configured to record only
-		// responses with http code != 200 &  != 206, we don't
-		// have to check for logBody value here.
-		bodyContents := respBodyRecorder.Body()
-		if bodyContents != nil {
-			b.Write(bodyContents)
-			fmt.Fprintf(b, "\n")
-		} else {
-			if !logBody {
-				// If there was no error response and body logging is disabled
-				// then we print <BODY> as a placeholder for the actual body.
-				b.Write(bodyPlaceHolder)
-				fmt.Fprintf(b, "\n")
-			}
-		}
-
-		fmt.Fprintf(b, "\n")
-
-		// Write the contents in one shot so that logs don't get interspersed.
-		output.Write(b.Bytes())
 	}
+	rq := trace.RequestInfo{Time: time.Now().UTC(), Method: r.Method, Path: r.URL.Path, RawQuery: r.URL.RawQuery}
+	rq.Headers = cloneHeader(r.Header)
+	rq.Headers.Set("Content-Length", strconv.Itoa(int(r.ContentLength)))
+	rq.Headers.Set("Host", r.Host)
+	for _, enc := range r.TransferEncoding {
+		rq.Headers.Add("Transfer-Encoding", enc)
+	}
+	if logBody {
+		// If body logging is disabled then we print <BODY> as a placeholder
+		// for the actual body.
+		rq.Body = reqBodyRecorder.Data()
+
+	} else {
+		rq.Body = bodyPlaceHolder
+	}
+	// Setup a http response body recorder
+	respBodyRecorder := &recordResponseWriter{ResponseWriter: w, logBody: logBody}
+	f(respBodyRecorder, r)
+
+	rs := trace.ResponseInfo{Time: time.Now().UTC()}
+	rs.Headers = cloneHeader(respBodyRecorder.Header())
+	rs.StatusCode = respBodyRecorder.statusCode
+	if rs.StatusCode == 0 {
+		rs.StatusCode = http.StatusOK
+	}
+	bodyContents := respBodyRecorder.Body()
+	if bodyContents != nil {
+		rs.Body = bodyContents
+	}
+	if !logBody {
+		// If there was no error response and body logging is disabled
+		// then we print <BODY> as a placeholder for the actual body.
+		rs.Body = bodyPlaceHolder
+	}
+	t.ReqInfo = rq
+	t.RespInfo = rs
+	return t
 }

cmd/httptrace.go

@@ -0,0 +1,115 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"strings"
+	"sync"
+
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/pubsub"
+	"github.com/minio/minio/pkg/trace"
+)
+
+//HTTPTraceSys holds global trace state
+type HTTPTraceSys struct {
+	peers  []*peerRESTClient
+	pubsub *pubsub.PubSub
+}
+
+// NewTraceSys - creates new HTTPTraceSys with all nodes subscribed to
+// the trace pub sub system
+func NewTraceSys(ctx context.Context, endpoints EndpointList) *HTTPTraceSys {
+	remoteHosts := getRemoteHosts(endpoints)
+	remoteClients, err := getRestClients(remoteHosts)
+	if err != nil {
+		logger.FatalIf(err, "Unable to start httptrace sub system")
+	}
+
+	ps := pubsub.New()
+	return &HTTPTraceSys{
+		remoteClients, ps,
+	}
+}
+
+// HasTraceListeners returns true if trace listeners are registered
+// for this node or peers
+func (sys *HTTPTraceSys) HasTraceListeners() bool {
+	return sys != nil && sys.pubsub.HasSubscribers()
+}
+
+// Publish - publishes trace message to the http trace pubsub system
+func (sys *HTTPTraceSys) Publish(traceMsg trace.Info) {
+	sys.pubsub.Publish(traceMsg)
+}
+
+// Trace writes http trace to writer
+func (sys *HTTPTraceSys) Trace(doneCh chan struct{}, trcAll bool) chan []byte {
+	traceCh := make(chan []byte)
+	go func() {
+		defer close(traceCh)
+
+		var wg = &sync.WaitGroup{}
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+
+			buf := &bytes.Buffer{}
+			ch := sys.pubsub.Subscribe()
+			defer sys.pubsub.Unsubscribe(ch)
+			for {
+				select {
+				case entry := <-ch:
+					trcInfo := entry.(trace.Info)
+					path := strings.TrimPrefix(trcInfo.ReqInfo.Path, "/")
+					// omit inter-node traffic if trcAll is false
+					if !trcAll && strings.HasPrefix(path, minioReservedBucket) {
+						continue
+					}
+					buf.Reset()
+					enc := json.NewEncoder(buf)
+					enc.SetEscapeHTML(false)
+					if err := enc.Encode(trcInfo); err != nil {
+						continue
+					}
+					traceCh <- buf.Bytes()
+				case <-doneCh:
+					return
+				}
+			}
+		}()
+
+		for _, peer := range sys.peers {
+			wg.Add(1)
+			go func(peer *peerRESTClient) {
+				defer wg.Done()
+				ch, err := peer.Trace(doneCh, trcAll)
+				if err != nil {
+					return
+				}
+				for entry := range ch {
+					traceCh <- entry
+				}
+			}(peer)
+		}
+		wg.Wait()
+	}()
+	return traceCh
+}

cmd/peer-rest-client-target.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2018, 2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

cmd/peer-rest-client.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"crypto/tls"
@@ -24,6 +25,7 @@ import (
 	"io"
 	"net/url"
 	"strconv"
+	"time"
 
 	"github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
@@ -52,9 +54,16 @@ func (client *peerRESTClient) reConnect() error {
 // permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
 // after verifying format.json
 func (client *peerRESTClient) call(method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
+	return client.callWithContext(context.Background(), method, values, body, length)
+}
+
+// Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
+// permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
+// after verifying format.json
+func (client *peerRESTClient) callWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
 	if !client.connected {
 		err := client.reConnect()
-		logger.LogIf(context.Background(), err)
+		logger.LogIf(ctx, err)
 		if err != nil {
 			return nil, err
 		}
@@ -64,7 +73,7 @@ func (client *peerRESTClient) call(method string, values url.Values, body io.Rea
 		values = make(url.Values)
 	}
 
-	respBody, err = client.restClient.Call(method, values, body, length)
+	respBody, err = client.restClient.CallWithContext(ctx, method, values, body, length)
 	if err == nil {
 		return respBody, nil
 	}
@@ -413,6 +422,54 @@ func (client *peerRESTClient) SignalService(sig serviceSignal) error {
 	return nil
 }
 
+// Trace - send http trace request to peer nodes
+func (client *peerRESTClient) Trace(doneCh chan struct{}, trcAll bool) (chan []byte, error) {
+	ch := make(chan []byte)
+	go func() {
+		cleanupFn := func(cancel context.CancelFunc, ch chan []byte, respBody io.ReadCloser) {
+			close(ch)
+			if cancel != nil {
+				cancel()
+			}
+			http.DrainBody(respBody)
+		}
+		for {
+			values := make(url.Values)
+			values.Set(peerRESTTraceAll, strconv.FormatBool(trcAll))
+			// get cancellation context to properly unsubscribe peers
+			ctx, cancel := context.WithCancel(context.Background())
+			respBody, err := client.callWithContext(ctx, peerRESTMethodTrace, values, nil, -1)
+			if err != nil {
+				//retry
+				time.Sleep(5 * time.Second)
+				select {
+				case <-doneCh:
+					cleanupFn(cancel, ch, respBody)
+					return
+				default:
+				}
+				continue
+			}
+			bio := bufio.NewScanner(respBody)
+			go func() {
+				<-doneCh
+				cancel()
+			}()
+			// Unmarshal each line, returns marshaled values.
+			for bio.Scan() {
+				ch <- bio.Bytes()
+			}
+			select {
+			case <-doneCh:
+				cleanupFn(cancel, ch, respBody)
+				return
+			default:
+			}
+		}
+	}()
+	return ch, nil
+}
+
 func getRemoteHosts(endpoints EndpointList) []*xnet.Host {
 	var remoteHosts []*xnet.Host
 	for _, hostStr := range GetRemotePeers(endpoints) {

cmd/peer-rest-common.go

@@ -41,6 +41,7 @@ const (
 	peerRESTMethodReloadFormat             = "reloadformat"
 	peerRESTMethodTargetExists             = "targetexists"
 	peerRESTMethodSendEvent                = "sendevent"
+	peerRESTMethodTrace                    = "trace"
 )
 
 const (
@@ -51,4 +52,5 @@ const (
 	peerRESTSignal   = "signal"
 	peerRESTProfiler = "profiler"
 	peerRESTDryRun   = "dry-run"
+	peerRESTTraceAll = "all"
 )

cmd/peer-rest-server.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"encoding/gob"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
@@ -32,6 +33,7 @@ import (
 	"github.com/minio/minio/pkg/event"
 	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/policy"
+	trace "github.com/minio/minio/pkg/trace"
 )
 
 // To abstract a node over network.
@@ -666,6 +668,47 @@ func (s *peerRESTServer) SignalServiceHandler(w http.ResponseWriter, r *http.Req
 	}
 }
 
+// TraceHandler sends http trace messages back to peer rest client
+func (s *peerRESTServer) TraceHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+	trcAll := r.URL.Query().Get(peerRESTTraceAll) == "true"
+
+	w.Header().Set("Connection", "close")
+	w.WriteHeader(http.StatusOK)
+	w.(http.Flusher).Flush()
+	ch := globalTrace.pubsub.Subscribe()
+	defer globalTrace.pubsub.Unsubscribe(ch)
+
+	enc := json.NewEncoder(w)
+	enc.SetEscapeHTML(false)
+	for {
+		select {
+		case entry := <-ch:
+			trcInfo := entry.(trace.Info)
+			path := strings.TrimPrefix(trcInfo.ReqInfo.Path, "/")
+			// omit inter-node traffic if trcAll is false
+			if !trcAll && strings.HasPrefix(path, minioReservedBucket) {
+				continue
+			}
+
+			if err := enc.Encode(trcInfo); err != nil {
+				return
+			}
+
+			if _, err := w.Write([]byte("\n")); err != nil {
+				return
+			}
+			w.(http.Flusher).Flush()
+		case <-r.Context().Done():
+			return
+
+		}
+	}
+}
+
 func (s *peerRESTServer) writeErrorResponse(w http.ResponseWriter, err error) {
 	w.WriteHeader(http.StatusForbidden)
 	w.Write([]byte(err.Error()))
@@ -711,5 +754,7 @@ func registerPeerRESTHandlers(router *mux.Router) {
 
 	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodReloadFormat).HandlerFunc(httpTraceHdrs(server.ReloadFormatHandler)).Queries(restQueries(peerRESTDryRun)...)
 
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodTrace).HandlerFunc(server.TraceHandler)
+
 	router.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
 }

cmd/rest/client.go

@@ -52,13 +52,13 @@ type Client struct {
 	newAuthToken        func() string
 }
 
-// Call - make a REST call.
+// CallWithContext - make a REST call with context.
-func (c *Client) Call(method string, values url.Values, body io.Reader, length int64) (reply io.ReadCloser, err error) {
+func (c *Client) CallWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (reply io.ReadCloser, err error) {
 	req, err := http.NewRequest(http.MethodPost, c.url.String()+"/"+method+"?"+values.Encode(), body)
 	if err != nil {
 		return nil, &NetworkError{err}
 	}
-
+	req = req.WithContext(ctx)
 	req.Header.Set("Authorization", "Bearer "+c.newAuthToken())
 	req.Header.Set("X-Minio-Time", time.Now().UTC().Format(time.RFC3339))
 	if length > 0 {
@@ -84,6 +84,12 @@ func (c *Client) Call(method string, values url.Values, body io.Reader, length i
 	return resp.Body, nil
 }
 
+// Call - make a REST call.
+func (c *Client) Call(method string, values url.Values, body io.Reader, length int64) (reply io.ReadCloser, err error) {
+	ctx := context.Background()
+	return c.CallWithContext(ctx, method, values, body, length)
+}
+
 // Close closes all idle connections of the underlying http client
 func (c *Client) Close() {
 	if c.httpIdleConnsCloser != nil {

cmd/server-main.go

@@ -290,6 +290,9 @@ func serverMain(ctx *cli.Context) {
 	// Init global heal state
 	initAllHealState(globalIsXL)
 
+	// initialize globalTrace system
+	globalTrace = NewTraceSys(context.Background(), globalEndpoints)
+
 	// Configure server.
 	var handler http.Handler
 	handler, err = configureServerHandler(globalEndpoints)

cmd/signals.go

@@ -74,7 +74,6 @@ func handleSignals() {
 
 			exit(err == nil && oerr == nil)
 		case osSignal := <-globalOSSignalCh:
-			stopHTTPTrace()
 			logger.Info("Exiting on signal: %s", strings.ToUpper(osSignal.String()))
 			exit(stopProcess())
 		case signal := <-globalServiceSignalCh:
@@ -83,14 +82,12 @@ func handleSignals() {
 				// Ignore this at the moment.
 			case serviceRestart:
 				logger.Info("Restarting on service signal")
-				stopHTTPTrace()
 				stop := stopProcess()
 				rerr := restartProcess()
 				logger.LogIf(context.Background(), rerr)
 				exit(stop && rerr == nil)
 			case serviceStop:
 				logger.Info("Stopping on service signal")
-				stopHTTPTrace()
 				exit(stopProcess())
 			}
 		}

cmd/utils.go

@@ -59,16 +59,6 @@ func IsErr(err error, errs ...error) bool {
 	return false
 }
 
-// Close Http tracing file.
-func stopHTTPTrace() {
-	if globalHTTPTraceFile != nil {
-		reqInfo := (&logger.ReqInfo{}).AppendTags("traceFile", globalHTTPTraceFile.Name())
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
-		logger.LogIf(ctx, globalHTTPTraceFile.Close())
-		globalHTTPTraceFile = nil
-	}
-}
-
 // make a copy of http.Header
 func cloneHeader(h http.Header) http.Header {
 	h2 := make(http.Header, len(h))

cmd/utils_test.go

@@ -54,19 +54,6 @@ func TestCloneHeader(t *testing.T) {
 	}
 }
 
-// Tests closing http tracing file.
-func TestStopHTTPTrace(t *testing.T) {
-	var err error
-	globalHTTPTraceFile, err = ioutil.TempFile("", "")
-	if err != nil {
-		defer os.Remove(globalHTTPTraceFile.Name())
-		stopHTTPTrace()
-		if globalHTTPTraceFile != nil {
-			t.Errorf("globalHTTPTraceFile is not nil, it is expected to be nil")
-		}
-	}
-}
-
 // Tests maximum object size.
 func TestMaxObjectSize(t *testing.T) {
 	sizes := []struct {

pkg/handlers/http-tracer_test.go

@@ -1,154 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2017 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package handlers
-
-import (
-	"bytes"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"regexp"
-	"testing"
-)
-
-func wsTestSuccessHandler(w http.ResponseWriter, r *http.Request) {
-	// A very simple health check.
-	w.WriteHeader(http.StatusOK)
-	w.Header().Set("Content-Type", "application/json")
-
-	ioutil.ReadAll(r.Body)
-
-	// In the future we could report back on the status of our DB, or our cache
-	// (e.g. Redis) by performing a simple PING, and include them in the response.
-	io.WriteString(w, `{"success": true}`)
-}
-
-func wsTest404Handler(w http.ResponseWriter, r *http.Request) {
-	w.WriteHeader(http.StatusNotFound)
-}
-
-func TestTraceHTTPHandler(t *testing.T) {
-
-	logOutput := bytes.NewBuffer([]byte(""))
-
-	testCases := []struct {
-		method            string
-		path              string
-		sentData          string
-		headers           map[string]string
-		handler           http.HandlerFunc
-		expectedStatus    int
-		expectedLogRegexp string
-	}{
-
-		{
-			method:         "PUT",
-			path:           "/test-log",
-			sentData:       "sending data",
-			headers:        map[string]string{"Test-Header": "TestHeaderValue"},
-			handler:        TraceReqHandlerFunc(http.HandlerFunc(wsTestSuccessHandler), logOutput, true),
-			expectedStatus: http.StatusOK,
-			expectedLogRegexp: `\[REQUEST github.com/minio/minio/pkg/handlers.wsTestSuccessHandler\] \[[^\]]*\] \[[^\]]*\]
-PUT /test-log
-Host:\ 
-Test-Header: TestHeaderValue
-
-sending data
-
-\[RESPONSE\] \[[^\]]*\] \[[^\]]*\]
-200 OK
-
-{"success": true}
-
-`,
-		},
-		{
-			method:         "POST",
-			path:           "/test-log",
-			handler:        TraceReqHandlerFunc(http.HandlerFunc(wsTestSuccessHandler), logOutput, false),
-			headers:        map[string]string{"Test-Header": "TestHeaderValue"},
-			expectedStatus: http.StatusOK,
-			expectedLogRegexp: `\[REQUEST github.com/minio/minio/pkg/handlers.wsTestSuccessHandler\] \[[^\]]*\] \[[^\]]*\]
-POST /test-log
-Host:\ 
-Test-Header: TestHeaderValue
-
-<BODY>
-
-\[RESPONSE\] \[[^\]]*\] \[[^\]]*\]
-200 OK
-
-<BODY>
-
-`,
-		},
-		{
-			method:         "POST",
-			path:           "/test-log",
-			handler:        TraceReqHandlerFunc(http.HandlerFunc(wsTest404Handler), logOutput, false),
-			headers:        map[string]string{"Test-Header": "TestHeaderValue"},
-			expectedStatus: http.StatusNotFound,
-			expectedLogRegexp: `\[REQUEST github.com/minio/minio/pkg/handlers.wsTest404Handler\] \[[^\]]*\] \[[^\]]*\]
-POST /test-log
-Host:\ 
-Test-Header: TestHeaderValue
-
-<BODY>
-
-\[RESPONSE\] \[[^\]]*\] \[[^\]]*\]
-404 Not Found
-
-<BODY>
-
-`,
-		},
-	}
-
-	for i, testCase := range testCases {
-		logOutput.Reset()
-
-		req, err := http.NewRequest(testCase.method, testCase.path, bytes.NewBuffer([]byte(testCase.sentData)))
-		if err != nil {
-			t.Fatalf("Test %d: %v\n", i+1, err)
-		}
-
-		for k, v := range testCase.headers {
-			req.Header.Set(k, v)
-		}
-
-		rr := httptest.NewRecorder()
-
-		handler := testCase.handler
-		handler.ServeHTTP(rr, req)
-
-		// Check the status code is what we expect.
-		if status := rr.Code; status != testCase.expectedStatus {
-			t.Errorf("Test %d: handler returned wrong status code: got %v want %v", i+1,
-				status, testCase.expectedStatus)
-		}
-
-		matched, err := regexp.MatchString(testCase.expectedLogRegexp, logOutput.String())
-		if err != nil {
-			t.Fatalf("Test %d: Incorrect regexp: %v", i+1, err)
-		}
-
-		if !matched {
-			t.Fatalf("Test %d: Unexpected log content, found: `%s`", i+1, logOutput.String())
-		}
-	}
-}

pkg/madmin/README.md

@@ -45,7 +45,7 @@ func main() {
 |:------------------------------------------|:--------------------------------------------|:-------------------|:----------------------------------|:------------------------|:--------------------------------------|:--------------------------------------------------|
 | [`ServiceStatus`](#ServiceStatus)         | [`ServerInfo`](#ServerInfo)                 | [`Heal`](#Heal)    | [`GetConfig`](#GetConfig)         | [`TopLocks`](#TopLocks) | [`AddUser`](#AddUser)                 |                                                   |
 | [`ServiceSendAction`](#ServiceSendAction) | [`ServerCPULoadInfo`](#ServerCPULoadInfo)   |                    | [`SetConfig`](#SetConfig)         |                         | [`SetUserPolicy`](#SetUserPolicy)     | [`StartProfiling`](#StartProfiling)               |
-|                                           | [`ServerMemUsageInfo`](#ServerMemUsageInfo) |                    | [`GetConfigKeys`](#GetConfigKeys) |                         | [`ListUsers`](#ListUsers)             | [`DownloadProfilingData`](#DownloadProfilingData) |
+| [`Trace`](#Trace)                                          | [`ServerMemUsageInfo`](#ServerMemUsageInfo) |                    | [`GetConfigKeys`](#GetConfigKeys) |                         | [`ListUsers`](#ListUsers)             | [`DownloadProfilingData`](#DownloadProfilingData) |
 |                                           |                                             |                    | [`SetConfigKeys`](#SetConfigKeys) |                         | [`AddCannedPolicy`](#AddCannedPolicy) |                                                   |
 
 
@@ -537,3 +537,22 @@ __Example__
 
     log.Println("Profiling data successfully downloaded.")
 ```
+
+<a name="Trace"></a>
+### Trace(allTrace bool,doneCh <-chan struct{}) <-chan TraceInfo
+Enable HTTP request tracing on all nodes in a MinIO cluster
+
+__Example__
+
+``` go
+    doneCh := make(chan struct{})
+    defer close(doneCh)
+    // listen to all trace including internal API calls
+    allTrace := true
+    // Start listening on all trace activity.
+    traceCh := madmClnt.Trace(allTrace,doneCh)
+    for traceInfo := range traceCh {
+        fmt.Println(traceInfo.String())
+    }
+    log.Println("Success")
+```

pkg/madmin/api-trace.go

@@ -0,0 +1,92 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package madmin
+
+import (
+	"bufio"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+
+	trace "github.com/minio/minio/pkg/trace"
+)
+
+// TraceInfo holds http trace
+type TraceInfo struct {
+	Trace trace.Info
+	Err   error `json:"-"`
+}
+
+// Trace - listen on http trace notifications.
+func (adm AdminClient) Trace(allTrace bool, doneCh <-chan struct{}) <-chan TraceInfo {
+	traceInfoCh := make(chan TraceInfo, 1)
+	// Only success, start a routine to start reading line by line.
+	go func(traceInfoCh chan<- TraceInfo) {
+		defer close(traceInfoCh)
+		for {
+			urlValues := make(url.Values)
+			urlValues.Set("all", strconv.FormatBool(allTrace))
+			reqData := requestData{
+				relPath:     "/v1/trace",
+				queryValues: urlValues,
+			}
+			// Execute GET to call trace handler
+			resp, err := adm.executeMethod("GET", reqData)
+			if err != nil {
+				closeResponse(resp)
+				return
+			}
+
+			if resp.StatusCode != http.StatusOK {
+				traceInfoCh <- TraceInfo{Err: httpRespToErrorResponse(resp)}
+				return
+			}
+
+			// Initialize a new bufio scanner, to read line by line.
+			bio := bufio.NewScanner(resp.Body)
+
+			// Close the response body.
+			defer resp.Body.Close()
+
+			// Unmarshal each line, returns marshaled values.
+			for bio.Scan() {
+				var traceRec trace.Info
+				if err = json.Unmarshal(bio.Bytes(), &traceRec); err != nil {
+					continue
+				}
+				select {
+				case <-doneCh:
+					return
+				case traceInfoCh <- TraceInfo{Trace: traceRec}:
+				}
+			}
+			// Look for any underlying errors.
+			if err = bio.Err(); err != nil {
+				// For an unexpected connection drop from server, we close the body
+				// and re-connect.
+				if err == io.ErrUnexpectedEOF {
+					resp.Body.Close()
+				}
+			}
+		}
+	}(traceInfoCh)
+
+	// Returns the trace info channel, for caller to start reading from.
+	return traceInfoCh
+}

pkg/madmin/examples/trace.go

@@ -0,0 +1,50 @@
+// +build ignore
+
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package main
+
+import (
+	"log"
+
+	"github.com/minio/minio/pkg/madmin"
+)
+
+func main() {
+	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
+	// dummy values, please replace them with original values.
+
+	// API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise.
+	// New returns an MinIO Admin client object.
+	madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+
+	// Start listening on all http trace activity from all servers
+	// in the minio cluster.
+	traceCh := madmClnt.Trace(false, doneCh)
+	for traceInfo := range traceCh {
+		if traceInfo.Err != nil {
+			fmt.Println(traceInfo.Err)
+		}
+		fmt.Println(traceInfo)
+	}
+}

pkg/pubsub/pubsub.go

@@ -0,0 +1,83 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package pubsub
+
+import (
+	"sync"
+)
+
+// PubSub holds publishers and subscribers
+type PubSub struct {
+	subs  []chan interface{}
+	pub   chan interface{}
+	mutex sync.Mutex
+}
+
+// process item to subscribers.
+func (ps *PubSub) process() {
+	for item := range ps.pub {
+		ps.mutex.Lock()
+		for _, sub := range ps.subs {
+			go func(s chan interface{}) {
+				s <- item
+			}(sub)
+		}
+		ps.mutex.Unlock()
+	}
+}
+
+// Publish message to pubsub system
+func (ps *PubSub) Publish(item interface{}) {
+	ps.pub <- item
+}
+
+// Subscribe - Adds a subscriber to pubsub system
+func (ps *PubSub) Subscribe() chan interface{} {
+	ps.mutex.Lock()
+	defer ps.mutex.Unlock()
+	ch := make(chan interface{})
+	ps.subs = append(ps.subs, ch)
+	return ch
+}
+
+// Unsubscribe removes current subscriber
+func (ps *PubSub) Unsubscribe(ch chan interface{}) {
+	ps.mutex.Lock()
+	defer ps.mutex.Unlock()
+
+	for i, sub := range ps.subs {
+		if sub == ch {
+			close(ch)
+			ps.subs = append(ps.subs[:i], ps.subs[i+1:]...)
+		}
+	}
+}
+
+// HasSubscribers returns true if pubsub system has subscribers
+func (ps *PubSub) HasSubscribers() bool {
+	ps.mutex.Lock()
+	defer ps.mutex.Unlock()
+	return len(ps.subs) > 0
+}
+
+// New inits a PubSub system
+func New() *PubSub {
+	ps := &PubSub{}
+	ps.pub = make(chan interface{})
+	go ps.process()
+	return ps
+}

pkg/pubsub/pubsub_test.go

@@ -0,0 +1,66 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package pubsub
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestSubscribe(t *testing.T) {
+	ps := New()
+	ps.Subscribe()
+	ps.Subscribe()
+	if len(ps.subs) != 2 {
+		t.Errorf("expected 2 subscribers")
+	}
+}
+
+func TestUnsubscribe(t *testing.T) {
+	ps := New()
+	c1 := ps.Subscribe()
+	ps.Subscribe()
+	ps.Unsubscribe(c1)
+	if len(ps.subs) != 1 {
+		t.Errorf("expected 1 subscriber")
+	}
+}
+
+func TestPubSub(t *testing.T) {
+	ps := New()
+	c1 := ps.Subscribe()
+	val := "hello"
+	ps.Publish(val)
+	msg := <-c1
+	if msg != "hello" {
+		t.Errorf(fmt.Sprintf("expected %s , found %s", val, msg))
+	}
+}
+
+func TestMultiPubSub(t *testing.T) {
+	ps := New()
+	c1 := ps.Subscribe()
+	c2 := ps.Subscribe()
+	val := "hello"
+	ps.Publish(val)
+
+	msg1 := <-c1
+	msg2 := <-c2
+	if msg1 != "hello" && msg2 != "hello" {
+		t.Errorf(fmt.Sprintf("expected both subscribers to have%s , found %s and  %s", val, msg1, msg2))
+	}
+}

pkg/trace/trace.go

@@ -0,0 +1,49 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package trace
+
+import (
+	"net/http"
+	"time"
+)
+
+// Info - represents a trace record, additionally
+// also reports errors if any while listening on trace.
+type Info struct {
+	NodeName string       `json:"nodename"`
+	FuncName string       `json:"funcname"`
+	ReqInfo  RequestInfo  `json:"request"`
+	RespInfo ResponseInfo `json:"response"`
+}
+
+// RequestInfo represents trace of http request
+type RequestInfo struct {
+	Time     time.Time   `json:"time"`
+	Method   string      `json:"method"`
+	Path     string      `json:"path,omitempty"`
+	RawQuery string      `json:"rawquery,omitempty"`
+	Headers  http.Header `json:"headers,omitempty"`
+	Body     []byte      `json:"body,omitempty"`
+}
+
+// ResponseInfo represents trace of http request
+type ResponseInfo struct {
+	Time       time.Time   `json:"time"`
+	Headers    http.Header `json:"headers,omitempty"`
+	Body       []byte      `json:"body,omitempty"`
+	StatusCode int         `json:"statuscode,omitempty"`
+}