Add admin API to send trace notifications to registered (#7128)

Remove current functionality to log trace to file
using MINIO_HTTP_TRACE env, and replace it with
mc admin trace command on mc client.

cmd/peer-rest-client.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"crypto/tls"
@@ -24,6 +25,7 @@ import (
 	"io"
 	"net/url"
 	"strconv"
+	"time"
 
 	"github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
@@ -52,9 +54,16 @@ func (client *peerRESTClient) reConnect() error {
 // permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
 // after verifying format.json
 func (client *peerRESTClient) call(method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
+	return client.callWithContext(context.Background(), method, values, body, length)
+}
+
+// Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
+// permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
+// after verifying format.json
+func (client *peerRESTClient) callWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
 	if !client.connected {
 		err := client.reConnect()
-		logger.LogIf(context.Background(), err)
+		logger.LogIf(ctx, err)
 		if err != nil {
 			return nil, err
 		}
@@ -64,7 +73,7 @@ func (client *peerRESTClient) call(method string, values url.Values, body io.Rea
 		values = make(url.Values)
 	}
 
-	respBody, err = client.restClient.Call(method, values, body, length)
+	respBody, err = client.restClient.CallWithContext(ctx, method, values, body, length)
 	if err == nil {
 		return respBody, nil
 	}
@@ -413,6 +422,54 @@ func (client *peerRESTClient) SignalService(sig serviceSignal) error {
 	return nil
 }
 
+// Trace - send http trace request to peer nodes
+func (client *peerRESTClient) Trace(doneCh chan struct{}, trcAll bool) (chan []byte, error) {
+	ch := make(chan []byte)
+	go func() {
+		cleanupFn := func(cancel context.CancelFunc, ch chan []byte, respBody io.ReadCloser) {
+			close(ch)
+			if cancel != nil {
+				cancel()
+			}
+			http.DrainBody(respBody)
+		}
+		for {
+			values := make(url.Values)
+			values.Set(peerRESTTraceAll, strconv.FormatBool(trcAll))
+			// get cancellation context to properly unsubscribe peers
+			ctx, cancel := context.WithCancel(context.Background())
+			respBody, err := client.callWithContext(ctx, peerRESTMethodTrace, values, nil, -1)
+			if err != nil {
+				//retry
+				time.Sleep(5 * time.Second)
+				select {
+				case <-doneCh:
+					cleanupFn(cancel, ch, respBody)
+					return
+				default:
+				}
+				continue
+			}
+			bio := bufio.NewScanner(respBody)
+			go func() {
+				<-doneCh
+				cancel()
+			}()
+			// Unmarshal each line, returns marshaled values.
+			for bio.Scan() {
+				ch <- bio.Bytes()
+			}
+			select {
+			case <-doneCh:
+				cleanupFn(cancel, ch, respBody)
+				return
+			default:
+			}
+		}
+	}()
+	return ch, nil
+}
+
 func getRemoteHosts(endpoints EndpointList) []*xnet.Host {
 	var remoteHosts []*xnet.Host
 	for _, hostStr := range GetRemotePeers(endpoints) {