Add ServerDrivesPerfInfo() admin API (#6969)

This is part of implementation for mc admin health command. The
ServerDrivesPerfInfo() admin API returns read and write speed
information for all the drives (local and remote) in a given Minio
server deployment.

Part of minio/mc#2606

cmd/admin-handlers.go

@@ -35,6 +35,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
@@ -284,6 +285,57 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	writeSuccessResponseJSON(w, jsonBytes)
 }
 
+// ServerDrivesPerfInfo holds informantion about address, performance
+// of all drives on one server. It also reports any errors if encountered
+// while trying to reach this server.
+type ServerDrivesPerfInfo struct {
+	Addr  string             `json:"addr"`
+	Error string             `json:"error,omitempty"`
+	Perf  []disk.Performance `json:"perf"`
+}
+
+// PerfInfoHandler - GET /minio/admin/v1/performance?perfType={perfType}
+// ----------
+// Get all performance information based on input type
+// Supported types = drive
+func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PerfInfo")
+
+	// Authenticate request
+	// Setting the region as empty so as the mc server info command is irrespective to the region.
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	perfType := vars["perfType"]
+
+	if perfType == "drive" {
+		// Get drive performance details from local server's drive(s)
+		dp := localEndpointsPerf(globalEndpoints)
+
+		// Notify all other Minio peers to report drive performance numbers
+		dps := globalNotificationSys.DrivePerfInfo()
+		dps = append(dps, dp)
+
+		// Marshal API response
+		jsonBytes, err := json.Marshal(dps)
+		if err != nil {
+			writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
+			return
+		}
+
+		// Reply with performance information (across nodes in a
+		// distributed setup) as json.
+		writeSuccessResponseJSON(w, jsonBytes)
+	} else {
+		writeErrorResponseJSON(w, ErrNotImplemented, r.URL)
+	}
+	return
+}
+
 // StartProfilingResult contains the status of the starting
 // profiling action in a given server
 type StartProfilingResult struct {

cmd/admin-router.go

@@ -60,6 +60,11 @@ func registerAdminRouter(router *mux.Router, enableIAM bool) {
 		adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
 		adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
 		adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+
+		/// Health operations
+
+		// Performance command - return performance details based on input type
+		adminV1Router.Methods(http.MethodGet).Path("/performance").HandlerFunc(httpTraceAll(adminAPI.PerfInfoHandler)).Queries("perfType", "{perfType:.*}")
 	}
 
 	// Profiling operations

cmd/endpoint.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/minio/pkg/mountinfo"
 )
 
@@ -197,6 +198,34 @@ func (endpoints EndpointList) GetString(i int) string {
 	return endpoints[i].String()
 }
 
+// localEndpointsPerf - returns ServerDrivesPerfInfo for only the
+// local endpoints from given list of endpoints
+func localEndpointsPerf(endpoints EndpointList) ServerDrivesPerfInfo {
+	var dps []disk.Performance
+	var addr string
+	for _, endpoint := range endpoints {
+		// Only proceed for local endpoints
+		if endpoint.IsLocal {
+			addr = GetLocalPeer(endpoints)
+			if _, err := os.Stat(endpoint.Path); err != nil {
+				// Since this drive is not available, add relevant details and proceed
+				dps = append(dps, disk.Performance{Path: endpoint.Path, Error: err.Error()})
+				continue
+			}
+			tempObj := mustGetUUID()
+			fsPath := pathJoin(endpoint.Path, minioMetaTmpBucket, tempObj)
+			dp := disk.GetPerformance(fsPath)
+			dp.Path = endpoint.Path
+			dps = append(dps, dp)
+		}
+	}
+
+	return ServerDrivesPerfInfo{
+		Addr: addr,
+		Perf: dps,
+	}
+}
+
 // NewEndpointList - returns new endpoint list based on input args.
 func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
 	var endpointType EndpointType

cmd/notification.go

@@ -512,6 +512,31 @@ func (sys *NotificationSys) Send(args eventArgs) []event.TargetIDErr {
 	return sys.send(args.BucketName, args.ToEvent(), targetIDs...)
 }
 
+// DrivePerfInfo - Drive speed (read and write) information
+func (sys *NotificationSys) DrivePerfInfo() []ServerDrivesPerfInfo {
+	reply := make([]ServerDrivesPerfInfo, len(sys.peerRPCClientMap))
+	var wg sync.WaitGroup
+	var i int
+	for addr, client := range sys.peerRPCClientMap {
+		wg.Add(1)
+		go func(addr xnet.Host, client *PeerRPCClient, idx int) {
+			defer wg.Done()
+			di, err := client.DrivePerfInfo()
+			if err != nil {
+				reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr.String())
+				ctx := logger.SetReqInfo(context.Background(), reqInfo)
+				logger.LogIf(ctx, err)
+				di.Addr = addr.String()
+				di.Error = err.Error()
+			}
+			reply[idx] = di
+		}(addr, client, i)
+		i++
+	}
+	wg.Wait()
+	return reply
+}
+
 // NewNotificationSys - creates new notification system object.
 func NewNotificationSys(config *serverConfig, endpoints EndpointList) *NotificationSys {
 	targetList := getNotificationTargets(config)

cmd/peer-rpc-client.go

@@ -141,6 +141,15 @@ func (rpcClient *PeerRPCClient) LoadCredentials() error {
 	return rpcClient.Call(peerServiceName+".LoadCredentials", &args, &reply)
 }
 
+// DrivePerfInfo - returns drive performance info for remote server.
+func (rpcClient *PeerRPCClient) DrivePerfInfo() (ServerDrivesPerfInfo, error) {
+	args := AuthArgs{}
+	var reply ServerDrivesPerfInfo
+
+	err := rpcClient.Call(peerServiceName+".DrivePerfInfo", &args, &reply)
+	return reply, err
+}
+
 // NewPeerRPCClient - returns new peer RPC client.
 func NewPeerRPCClient(host *xnet.Host) (*PeerRPCClient, error) {
 	scheme := "http"

cmd/peer-rpc-server.go

@@ -238,6 +238,17 @@ func (receiver *peerRPCReceiver) LoadCredentials(args *AuthArgs, reply *VoidRepl
 	return globalConfigSys.Load(newObjectLayerFn())
 }
 
+// DrivePerfInfo - handles drive performance RPC call
+func (receiver *peerRPCReceiver) DrivePerfInfo(args *AuthArgs, reply *ServerDrivesPerfInfo) error {
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
+	*reply = localEndpointsPerf(globalEndpoints)
+	return nil
+}
+
 // NewPeerRPCServer - returns new peer RPC server.
 func NewPeerRPCServer() (*xrpc.Server, error) {
 	rpcServer := xrpc.NewServer()

pkg/disk/disk.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,6 +16,26 @@
 
 package disk
 
+import (
+	"bytes"
+	"crypto/rand"
+	"errors"
+	"os"
+	"path"
+	"strconv"
+	"time"
+
+	humanize "github.com/dustin/go-humanize"
+)
+
+// file size for performance read and write checks
+const randBufSize = 1 * humanize.KiByte
+const randParts = 1024
+const fileSize = randParts * randBufSize
+
+// Total count of read / write iteration for performance measurement
+const iterations = 10
+
 // Info stat fs struct is container which holds following values
 // Total - total size of the volume / disk
 // Free - free size of the volume / disk
@@ -32,3 +52,87 @@ type Info struct {
 	// Usage is calculated per tenant.
 	Usage uint64
 }
+
+// Performance holds informantion about read and write speed of a disk
+type Performance struct {
+	Path       string  `json:"path"`
+	Error      string  `json:"error,omitempty"`
+	WriteSpeed float64 `json:"writeSpeed"`
+	ReadSpeed  float64 `json:"readSpeed"`
+}
+
+// GetPerformance returns given disk's read and write performance
+func GetPerformance(path string) Performance {
+	perf := Performance{}
+	write, read, err := doPerfMeasure(path)
+	if err != nil {
+		perf.Error = err.Error()
+		return perf
+	}
+	perf.WriteSpeed = write
+	perf.ReadSpeed = read
+	return perf
+}
+
+// Calculate the write and read performance - write and read 10 tmp (1 MiB)
+// files and find the average time taken (Bytes / Sec)
+func doPerfMeasure(fsPath string) (write, read float64, err error) {
+	var count int
+	var totalWriteElapsed time.Duration
+	var totalReadElapsed time.Duration
+
+	defer os.RemoveAll(fsPath)
+
+	randBuf := make([]byte, randBufSize)
+	rand.Read(randBuf)
+	buf := bytes.Repeat(randBuf, randParts)
+
+	// create the enclosing directory
+	err = os.MkdirAll(fsPath, 0777)
+	if err != nil {
+		return 0, 0, err
+	}
+
+	for count = 1; count <= iterations; count++ {
+		fsTempObjPath := path.Join(fsPath, strconv.Itoa(count))
+
+		// Write performance calculation
+		writeStart := time.Now()
+		n, err := writeFile(fsTempObjPath, buf)
+
+		if err != nil {
+			return 0, 0, err
+		}
+		if n != fileSize {
+			return 0, 0, errors.New("Could not write temporary data to disk")
+		}
+
+		writeElapsed := time.Since(writeStart)
+		totalWriteElapsed += writeElapsed
+
+		// Read performance calculation
+		readStart := time.Now()
+		n, err = readFile(fsTempObjPath, buf)
+
+		if err != nil {
+			return 0, 0, err
+		}
+		if n != fileSize {
+			return 0, 0, errors.New("Could not read temporary data from disk")
+		}
+
+		readElapsed := time.Since(readStart)
+		totalReadElapsed += readElapsed
+	}
+	// Average time spent = total time elapsed / number of writes
+	avgWriteTime := totalWriteElapsed.Seconds() / float64(count)
+	// Write perf = fileSize (in Bytes) / average time spent writing (in seconds)
+	write = fileSize / avgWriteTime
+
+	// Average time spent = total time elapsed / number of writes
+	avgReadTime := totalReadElapsed.Seconds() / float64(count)
+	// read perf = fileSize (in Bytes) / average time spent reading (in seconds)
+	read = fileSize / avgReadTime
+
+	return write, read, nil
+}

pkg/disk/helpers.go

@@ -0,0 +1,52 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package disk
+
+import (
+	"os"
+)
+
+func readFile(path string, buf []byte) (int, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return 0, err
+	}
+	defer f.Close()
+
+	n, err := f.Read(buf)
+	if err != nil {
+		return 0, err
+	}
+
+	return n, nil
+}
+
+func writeFile(path string, data []byte) (int, error) {
+	f, err := os.Create(path)
+	if err != nil {
+		return 0, err
+	}
+	defer f.Close()
+
+	n, err := f.Write(data)
+	if err != nil {
+		return 0, err
+	}
+
+	f.Sync()
+	return n, nil
+}

pkg/madmin/API.md

@@ -39,7 +39,7 @@ func main() {
 | Service operations         | Info operations  | Healing operations                    | Config operations        | IAM operations | Misc                                |
 |:----------------------------|:----------------------------|:--------------------------------------|:--------------------------|:------------------------------------|:------------------------------------|
 | [`ServiceStatus`](#ServiceStatus) | [`ServerInfo`](#ServerInfo) | [`Heal`](#Heal) | [`GetConfig`](#GetConfig) | [`AddUser`](#AddUser) | [`SetAdminCredentials`](#SetAdminCredentials) |
-| [`ServiceSendAction`](#ServiceSendAction) | | | [`SetConfig`](#SetConfig) | [`SetUserPolicy`](#SetUserPolicy) | [`StartProfiling`](#StartProfiling) |
+| [`ServiceSendAction`](#ServiceSendAction) | [`ServerDrivesPerfInfo`](#ServerDrivesPerfInfo) | [`SetConfig`](#SetConfig) | [`SetUserPolicy`](#SetUserPolicy) | [`StartProfiling`](#StartProfiling) |
 | | |            | [`GetConfigKeys`](#GetConfigKeys) | [`ListUsers`](#ListUsers) | [`DownloadProfilingData`](#DownloadProfilingData) |
 | | |            | [`SetConfigKeys`](#SetConfigKeys) | [`AddCannedPolicy`](#AddCannedPolicy) | |
 
@@ -204,6 +204,23 @@ Fetches information for all cluster nodes, such as server properties, storage in
 
  ```
 
+<a name="ServerDrivesPerfInfo"></a>
+### ServerDrivesPerfInfo() ([]ServerDrivesPerfInfo, error)
+
+Fetches drive performance information for all cluster nodes. Returned value is in Bytes/s.
+
+| Param | Type | Description |
+|---|---|---|
+|`di.Addr` | _string_ | Address of the server the following information is retrieved from. |
+|`di.Error` | _string _ | Errors (if any) encountered while reaching this node |
+|`di.DrivesPerf` | _disk.Performance_ | Path of the drive mount on above server and read, write speed. |
+
+| Param | Type | Description |
+|---|---|---|
+|`disk.Performance.Path` | _string_ | Path of drive mount. |
+|`disk.Performance.Error` | _string_ | Error (if any) encountered while accessing this drive. |
+|`disk.Performance.WriteSpeed` | _float64_ | Write speed on above path in Bytes/s. |
+|`disk.Performance.ReadSpeed` | _float64_ | Read speed on above path in Bytes/s. |
 
 ## 6. Heal operations
 

pkg/madmin/examples/drives-perf-info.go

@@ -0,0 +1,44 @@
+// +build ignore
+
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package main
+
+import (
+	"log"
+
+	"github.com/minio/minio/pkg/madmin"
+)
+
+func main() {
+	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are
+	// dummy values, please replace them with original values.
+
+	// API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise.
+	// New returns an Minio Admin client object.
+	madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	st, err := madmClnt.ServerDrivesPerfInfo()
+	if err != nil {
+		log.Fatalln(err)
+	}
+	log.Println(st)
+}

pkg/madmin/info-commands.go

@@ -21,7 +21,10 @@ import (
 	"encoding/json"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"time"
+
+	"github.com/minio/minio/pkg/disk"
 )
 
 // BackendType - represents different backend types.
@@ -147,3 +150,46 @@ func (adm *AdminClient) ServerInfo() ([]ServerInfo, error) {
 
 	return serversInfo, nil
 }
+
+// ServerDrivesPerfInfo holds informantion about address and write speed of
+// all drives in a single server node
+type ServerDrivesPerfInfo struct {
+	Addr  string             `json:"addr"`
+	Error string             `json:"error,omitempty"`
+	Perf  []disk.Performance `json:"perf"`
+}
+
+// ServerDrivesPerfInfo - Returns drive's read and write performance information
+func (adm *AdminClient) ServerDrivesPerfInfo() ([]ServerDrivesPerfInfo, error) {
+	v := url.Values{}
+	v.Set("perfType", string("drive"))
+	resp, err := adm.executeMethod("GET", requestData{
+		relPath:     "/v1/performance",
+		queryValues: v,
+	})
+
+	defer closeResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check response http status code
+	if resp.StatusCode != http.StatusOK {
+		return nil, httpRespToErrorResponse(resp)
+	}
+
+	// Unmarshal the server's json response
+	var info []ServerDrivesPerfInfo
+
+	respBytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	err = json.Unmarshal(respBytes, &info)
+	if err != nil {
+		return nil, err
+	}
+
+	return info, nil
+}