Add Profiler Admin API (#6463)

Two handlers are added to admin API to enable profiling and disable
profiling of a server in a standalone mode, or all nodes in the
distributed mode.

/minio/admin/profiling/start/{cpu,block,mem}:
  - Start profiling and return starting JSON results, e.g. one
    node is offline.

/minio/admin/profiling/download:
  - Stop the on-going profiling task
  - Stream a zip file which contains all profiling files that can
    be later inspected by go tool pprof

cmd/admin-handlers.go

@@ -17,13 +17,16 @@
 package cmd
 
 import (
+	"archive/zip"
 	"bytes"
 	"context"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"net/http"
+	"os"
 	"strconv"
 	"strings"
 	"sync"
@@ -275,6 +278,121 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	writeSuccessResponseJSON(w, jsonBytes)
 }
 
+// StartProfilingResult contains the status of the starting
+// profiling action in a given server
+type StartProfilingResult struct {
+	NodeName string `json:"nodeName"`
+	Success  bool   `json:"success"`
+	Error    string `json:"error"`
+}
+
+// StartProfilingHandler - POST /minio/admin/v1/profiling/start/{profiler}
+// ----------
+// Enable profiling information
+func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.Request) {
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	profiler := vars["profiler"]
+
+	startProfilingResult := make([]StartProfilingResult, len(globalAdminPeers))
+
+	// Call StartProfiling function on all nodes and save results
+	wg := sync.WaitGroup{}
+	for i, peer := range globalAdminPeers {
+		wg.Add(1)
+		go func(idx int, peer adminPeer) {
+			defer wg.Done()
+			result := StartProfilingResult{NodeName: peer.addr}
+			if err := peer.cmdRunner.StartProfiling(profiler); err != nil {
+				result.Error = err.Error()
+				return
+			}
+			result.Success = true
+			startProfilingResult[idx] = result
+		}(i, peer)
+	}
+	wg.Wait()
+
+	// Create JSON result and send it to the client
+	startProfilingResultInBytes, err := json.Marshal(startProfilingResult)
+	if err != nil {
+		writeCustomErrorResponseJSON(w, http.StatusInternalServerError, err.Error(), r.URL)
+		return
+	}
+	writeSuccessResponseJSON(w, []byte(startProfilingResultInBytes))
+}
+
+// dummyFileInfo represents a dummy representation of a profile data file
+// present only in memory, it helps to generate the zip stream.
+type dummyFileInfo struct {
+	name    string
+	size    int64
+	mode    os.FileMode
+	modTime time.Time
+	isDir   bool
+	sys     interface{}
+}
+
+func (f dummyFileInfo) Name() string       { return f.name }
+func (f dummyFileInfo) Size() int64        { return f.size }
+func (f dummyFileInfo) Mode() os.FileMode  { return f.mode }
+func (f dummyFileInfo) ModTime() time.Time { return f.modTime }
+func (f dummyFileInfo) IsDir() bool        { return f.isDir }
+func (f dummyFileInfo) Sys() interface{}   { return f.sys }
+
+// DownloadProfilingHandler - POST /minio/admin/v1/profiling/download
+// ----------
+// Download profiling information of all nodes in a zip format
+func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) {
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	// Return 200 OK
+	w.WriteHeader(http.StatusOK)
+
+	// Initialize a zip writer which will provide a zipped content
+	// of profiling data of all nodes
+	zipWriter := zip.NewWriter(w)
+	defer zipWriter.Close()
+
+	for i, peer := range globalAdminPeers {
+		// Get profiling data from a node
+		data, err := peer.cmdRunner.DownloadProfilingData()
+		if err != nil {
+			logger.LogIf(context.Background(), fmt.Errorf("Unable to download profiling data from node `%s`, reason: %s", peer.addr, err.Error()))
+			continue
+		}
+
+		// Send profiling data to zip as file
+		header, err := zip.FileInfoHeader(dummyFileInfo{
+			name:    fmt.Sprintf("profiling-%d", i),
+			size:    int64(len(data)),
+			mode:    0600,
+			modTime: time.Now().UTC(),
+			isDir:   false,
+			sys:     nil,
+		})
+		if err != nil {
+			continue
+		}
+		writer, err := zipWriter.CreateHeader(header)
+		if err != nil {
+			continue
+		}
+		if _, err = io.Copy(writer, bytes.NewBuffer(data)); err != nil {
+			return
+		}
+	}
+}
+
 // extractHealInitParams - Validates params for heal init API.
 func extractHealInitParams(r *http.Request) (bucket, objPrefix string,
 	hs madmin.HealOpts, clientToken string, forceStart bool,

cmd/admin-router.go

@@ -60,6 +60,10 @@ func registerAdminRouter(router *mux.Router) {
 	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
 	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
 
+	// Profiling operations
+	adminV1Router.Methods(http.MethodPost).Path("/profiling/start/{profiler}").HandlerFunc(httpTraceAll(adminAPI.StartProfilingHandler))
+	adminV1Router.Methods(http.MethodGet).Path("/profiling/download").HandlerFunc(httpTraceAll(adminAPI.DownloadProfilingHandler))
+
 	/// Config operations
 
 	// Update credentials

cmd/admin-rpc-client.go

@@ -68,6 +68,22 @@ func (rpcClient *AdminRPCClient) GetConfig() ([]byte, error) {
 	return reply, err
 }
 
+// StartProfiling - starts profiling in the remote server.
+func (rpcClient *AdminRPCClient) StartProfiling(profiler string) error {
+	args := StartProfilingArgs{Profiler: profiler}
+	reply := VoidReply{}
+	return rpcClient.Call(adminServiceName+".StartProfiling", &args, &reply)
+}
+
+// DownloadProfilingData - returns profiling data of the remote server.
+func (rpcClient *AdminRPCClient) DownloadProfilingData() ([]byte, error) {
+	args := AuthArgs{}
+	var reply []byte
+
+	err := rpcClient.Call(adminServiceName+".DownloadProfilingData", &args, &reply)
+	return reply, err
+}
+
 // NewAdminRPCClient - returns new admin RPC client.
 func NewAdminRPCClient(host *xnet.Host) (*AdminRPCClient, error) {
 	scheme := "http"
@@ -112,6 +128,8 @@ type adminCmdRunner interface {
 	ReInitFormat(dryRun bool) error
 	ServerInfo() (ServerInfoData, error)
 	GetConfig() ([]byte, error)
+	StartProfiling(string) error
+	DownloadProfilingData() ([]byte, error)
 }
 
 // adminPeer - represents an entity that implements admin API RPCs.

cmd/admin-rpc-server.go

@@ -52,6 +52,23 @@ func (receiver *adminRPCReceiver) ServerInfo(args *AuthArgs, reply *ServerInfoDa
 	return err
 }
 
+// StartProfilingArgs - holds the RPC argument for StartingProfiling RPC call
+type StartProfilingArgs struct {
+	AuthArgs
+	Profiler string
+}
+
+// StartProfiling - starts profiling of this server
+func (receiver *adminRPCReceiver) StartProfiling(args *StartProfilingArgs, reply *VoidReply) error {
+	return receiver.local.StartProfiling(args.Profiler)
+}
+
+// DownloadProfilingData - stops and returns profiling data of this server
+func (receiver *adminRPCReceiver) DownloadProfilingData(args *AuthArgs, reply *[]byte) (err error) {
+	*reply, err = receiver.local.DownloadProfilingData()
+	return
+}
+
 // GetConfig - returns the config.json of this server.
 func (receiver *adminRPCReceiver) GetConfig(args *AuthArgs, reply *[]byte) (err error) {
 	*reply, err = receiver.local.GetConfig()

cmd/api-errors.go

@@ -1727,7 +1727,10 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 
 // getAPIError provides API Error for input API error code.
 func getAPIError(code APIErrorCode) APIError {
-	return errorCodeResponse[code]
+	if apiErr, ok := errorCodeResponse[code]; ok {
+		return apiErr
+	}
+	return errorCodeResponse[ErrInternalError]
 }
 
 // getErrorResponse gets in standard error and resource value and

cmd/common-main.go

@@ -98,7 +98,9 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 func handleCommonEnvVars() {
 	// Start profiler if env is set.
 	if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" {
-		globalProfiler = startProfiler(profiler)
+		var err error
+		globalProfiler, err = startProfiler(profiler, "")
+		logger.FatalIf(err, "Unable to setup a profiler")
 	}
 
 	accessKey := os.Getenv("MINIO_ACCESS_KEY")

cmd/local-admin-client.go

@@ -19,7 +19,11 @@ package cmd
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"os"
+
+	"io/ioutil"
 )
 
 // localAdminClient - represents admin operation to be executed locally.
@@ -80,3 +84,40 @@ func (lc localAdminClient) GetConfig() ([]byte, error) {
 
 	return json.Marshal(globalServerConfig)
 }
+
+// StartProfiling - starts profiling on the local server.
+func (lc localAdminClient) StartProfiling(profiler string) error {
+	if globalProfiler != nil {
+		globalProfiler.Stop()
+	}
+	prof, err := startProfiler(profiler, "")
+	if err != nil {
+		return err
+	}
+	globalProfiler = prof
+	return nil
+}
+
+// DownloadProfilingData - stops and returns profiling data of the local server.
+func (lc localAdminClient) DownloadProfilingData() ([]byte, error) {
+	if globalProfiler == nil {
+		return nil, errors.New("profiler not enabled")
+	}
+
+	profilerPath := globalProfiler.Path()
+
+	// Stop the profiler
+	globalProfiler.Stop()
+
+	profilerFile, err := os.Open(profilerPath)
+	if err != nil {
+		return nil, err
+	}
+
+	data, err := ioutil.ReadAll(profilerFile)
+	if err != nil {
+		return nil, err
+	}
+
+	return data, nil
+}

cmd/utils.go

@@ -23,6 +23,7 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"encoding/xml"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -30,6 +31,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path/filepath"
 	"reflect"
 	"strings"
 	"sync"
@@ -182,26 +184,66 @@ func contains(slice interface{}, elem interface{}) bool {
 	return false
 }
 
+// profilerWrapper is created becauses pkg/profiler doesn't
+// provide any API to calculate the profiler file path in the
+// disk since the name of this latter is randomly generated.
+type profilerWrapper struct {
+	stopFn func()
+	pathFn func() string
+}
+
+func (p profilerWrapper) Stop() {
+	p.stopFn()
+}
+
+func (p profilerWrapper) Path() string {
+	return p.pathFn()
+}
+
 // Starts a profiler returns nil if profiler is not enabled, caller needs to handle this.
-func startProfiler(profiler string) interface {
+func startProfiler(profilerType, dirPath string) (interface {
 	Stop()
-} {
-	// Enable profiler if ``_MINIO_PROFILER`` is set. Supported options are [cpu, mem, block].
-	switch profiler {
-	case "cpu":
-		return profile.Start(profile.CPUProfile, profile.NoShutdownHook)
-	case "mem":
-		return profile.Start(profile.MemProfile, profile.NoShutdownHook)
-	case "block":
-		return profile.Start(profile.BlockProfile, profile.NoShutdownHook)
-	default:
-		return nil
+	Path() string
+}, error) {
+
+	var err error
+	if dirPath == "" {
+		dirPath, err = ioutil.TempDir("", "profile")
+		if err != nil {
+			return nil, err
+		}
 	}
+
+	var profiler interface {
+		Stop()
+	}
+
+	// Enable profiler, supported types are [cpu, mem, block].
+	switch profilerType {
+	case "cpu":
+		profiler = profile.Start(profile.CPUProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
+	case "mem":
+		profiler = profile.Start(profile.MemProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
+	case "block":
+		profiler = profile.Start(profile.BlockProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
+	default:
+		return nil, errors.New("profiler type unknown")
+	}
+
+	return &profilerWrapper{
+		stopFn: profiler.Stop,
+		pathFn: func() string {
+			return filepath.Join(dirPath, profilerType+".pprof")
+		},
+	}, nil
 }
 
 // Global profiler to be used by service go-routine.
 var globalProfiler interface {
+	// Stop the profiler
 	Stop()
+	// Return the path of the profiling file
+	Path() string
 }
 
 // dump the request into a string in JSON format.

cmd/utils_test.go

@@ -228,8 +228,9 @@ func TestURL2BucketObjectName(t *testing.T) {
 
 // Add tests for starting and stopping different profilers.
 func TestStartProfiler(t *testing.T) {
-	if startProfiler("") != nil {
-		t.Fatal("Expected nil, but non-nil value returned for invalid profiler.")
+	_, err := startProfiler("", "")
+	if err == nil {
+		t.Fatal("Expected a non nil error, but nil error returned for invalid profiler.")
 	}
 }