mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
Add ServerDrivesPerfInfo() admin API (#6969)
This is part of implementation for mc admin health command. The ServerDrivesPerfInfo() admin API returns read and write speed information for all the drives (local and remote) in a given Minio server deployment. Part of minio/mc#2606
This commit is contained in:
parent
75cd4201b0
commit
fcb56d864c
@ -35,6 +35,7 @@ import (
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/disk"
|
||||
"github.com/minio/minio/pkg/handlers"
|
||||
"github.com/minio/minio/pkg/iam/policy"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
@ -284,6 +285,57 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
writeSuccessResponseJSON(w, jsonBytes)
|
||||
}
|
||||
|
||||
// ServerDrivesPerfInfo holds informantion about address, performance
|
||||
// of all drives on one server. It also reports any errors if encountered
|
||||
// while trying to reach this server.
|
||||
type ServerDrivesPerfInfo struct {
|
||||
Addr string `json:"addr"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Perf []disk.Performance `json:"perf"`
|
||||
}
|
||||
|
||||
// PerfInfoHandler - GET /minio/admin/v1/performance?perfType={perfType}
|
||||
// ----------
|
||||
// Get all performance information based on input type
|
||||
// Supported types = drive
|
||||
func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "PerfInfo")
|
||||
|
||||
// Authenticate request
|
||||
// Setting the region as empty so as the mc server info command is irrespective to the region.
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(w, adminAPIErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
perfType := vars["perfType"]
|
||||
|
||||
if perfType == "drive" {
|
||||
// Get drive performance details from local server's drive(s)
|
||||
dp := localEndpointsPerf(globalEndpoints)
|
||||
|
||||
// Notify all other Minio peers to report drive performance numbers
|
||||
dps := globalNotificationSys.DrivePerfInfo()
|
||||
dps = append(dps, dp)
|
||||
|
||||
// Marshal API response
|
||||
jsonBytes, err := json.Marshal(dps)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Reply with performance information (across nodes in a
|
||||
// distributed setup) as json.
|
||||
writeSuccessResponseJSON(w, jsonBytes)
|
||||
} else {
|
||||
writeErrorResponseJSON(w, ErrNotImplemented, r.URL)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// StartProfilingResult contains the status of the starting
|
||||
// profiling action in a given server
|
||||
type StartProfilingResult struct {
|
||||
|
@ -60,6 +60,11 @@ func registerAdminRouter(router *mux.Router, enableIAM bool) {
|
||||
adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
|
||||
adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
|
||||
adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
|
||||
|
||||
/// Health operations
|
||||
|
||||
// Performance command - return performance details based on input type
|
||||
adminV1Router.Methods(http.MethodGet).Path("/performance").HandlerFunc(httpTraceAll(adminAPI.PerfInfoHandler)).Queries("perfType", "{perfType:.*}")
|
||||
}
|
||||
|
||||
// Profiling operations
|
||||
|
@ -29,6 +29,7 @@ import (
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/disk"
|
||||
"github.com/minio/minio/pkg/mountinfo"
|
||||
)
|
||||
|
||||
@ -197,6 +198,34 @@ func (endpoints EndpointList) GetString(i int) string {
|
||||
return endpoints[i].String()
|
||||
}
|
||||
|
||||
// localEndpointsPerf - returns ServerDrivesPerfInfo for only the
|
||||
// local endpoints from given list of endpoints
|
||||
func localEndpointsPerf(endpoints EndpointList) ServerDrivesPerfInfo {
|
||||
var dps []disk.Performance
|
||||
var addr string
|
||||
for _, endpoint := range endpoints {
|
||||
// Only proceed for local endpoints
|
||||
if endpoint.IsLocal {
|
||||
addr = GetLocalPeer(endpoints)
|
||||
if _, err := os.Stat(endpoint.Path); err != nil {
|
||||
// Since this drive is not available, add relevant details and proceed
|
||||
dps = append(dps, disk.Performance{Path: endpoint.Path, Error: err.Error()})
|
||||
continue
|
||||
}
|
||||
tempObj := mustGetUUID()
|
||||
fsPath := pathJoin(endpoint.Path, minioMetaTmpBucket, tempObj)
|
||||
dp := disk.GetPerformance(fsPath)
|
||||
dp.Path = endpoint.Path
|
||||
dps = append(dps, dp)
|
||||
}
|
||||
}
|
||||
|
||||
return ServerDrivesPerfInfo{
|
||||
Addr: addr,
|
||||
Perf: dps,
|
||||
}
|
||||
}
|
||||
|
||||
// NewEndpointList - returns new endpoint list based on input args.
|
||||
func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
|
||||
var endpointType EndpointType
|
||||
|
@ -512,6 +512,31 @@ func (sys *NotificationSys) Send(args eventArgs) []event.TargetIDErr {
|
||||
return sys.send(args.BucketName, args.ToEvent(), targetIDs...)
|
||||
}
|
||||
|
||||
// DrivePerfInfo - Drive speed (read and write) information
|
||||
func (sys *NotificationSys) DrivePerfInfo() []ServerDrivesPerfInfo {
|
||||
reply := make([]ServerDrivesPerfInfo, len(sys.peerRPCClientMap))
|
||||
var wg sync.WaitGroup
|
||||
var i int
|
||||
for addr, client := range sys.peerRPCClientMap {
|
||||
wg.Add(1)
|
||||
go func(addr xnet.Host, client *PeerRPCClient, idx int) {
|
||||
defer wg.Done()
|
||||
di, err := client.DrivePerfInfo()
|
||||
if err != nil {
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr.String())
|
||||
ctx := logger.SetReqInfo(context.Background(), reqInfo)
|
||||
logger.LogIf(ctx, err)
|
||||
di.Addr = addr.String()
|
||||
di.Error = err.Error()
|
||||
}
|
||||
reply[idx] = di
|
||||
}(addr, client, i)
|
||||
i++
|
||||
}
|
||||
wg.Wait()
|
||||
return reply
|
||||
}
|
||||
|
||||
// NewNotificationSys - creates new notification system object.
|
||||
func NewNotificationSys(config *serverConfig, endpoints EndpointList) *NotificationSys {
|
||||
targetList := getNotificationTargets(config)
|
||||
|
@ -141,6 +141,15 @@ func (rpcClient *PeerRPCClient) LoadCredentials() error {
|
||||
return rpcClient.Call(peerServiceName+".LoadCredentials", &args, &reply)
|
||||
}
|
||||
|
||||
// DrivePerfInfo - returns drive performance info for remote server.
|
||||
func (rpcClient *PeerRPCClient) DrivePerfInfo() (ServerDrivesPerfInfo, error) {
|
||||
args := AuthArgs{}
|
||||
var reply ServerDrivesPerfInfo
|
||||
|
||||
err := rpcClient.Call(peerServiceName+".DrivePerfInfo", &args, &reply)
|
||||
return reply, err
|
||||
}
|
||||
|
||||
// NewPeerRPCClient - returns new peer RPC client.
|
||||
func NewPeerRPCClient(host *xnet.Host) (*PeerRPCClient, error) {
|
||||
scheme := "http"
|
||||
|
@ -238,6 +238,17 @@ func (receiver *peerRPCReceiver) LoadCredentials(args *AuthArgs, reply *VoidRepl
|
||||
return globalConfigSys.Load(newObjectLayerFn())
|
||||
}
|
||||
|
||||
// DrivePerfInfo - handles drive performance RPC call
|
||||
func (receiver *peerRPCReceiver) DrivePerfInfo(args *AuthArgs, reply *ServerDrivesPerfInfo) error {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
|
||||
*reply = localEndpointsPerf(globalEndpoints)
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewPeerRPCServer - returns new peer RPC server.
|
||||
func NewPeerRPCServer() (*xrpc.Server, error) {
|
||||
rpcServer := xrpc.NewServer()
|
||||
|
106
pkg/disk/disk.go
106
pkg/disk/disk.go
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2015 Minio, Inc.
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -16,6 +16,26 @@
|
||||
|
||||
package disk
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"os"
|
||||
"path"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
)
|
||||
|
||||
// file size for performance read and write checks
|
||||
const randBufSize = 1 * humanize.KiByte
|
||||
const randParts = 1024
|
||||
const fileSize = randParts * randBufSize
|
||||
|
||||
// Total count of read / write iteration for performance measurement
|
||||
const iterations = 10
|
||||
|
||||
// Info stat fs struct is container which holds following values
|
||||
// Total - total size of the volume / disk
|
||||
// Free - free size of the volume / disk
|
||||
@ -32,3 +52,87 @@ type Info struct {
|
||||
// Usage is calculated per tenant.
|
||||
Usage uint64
|
||||
}
|
||||
|
||||
// Performance holds informantion about read and write speed of a disk
|
||||
type Performance struct {
|
||||
Path string `json:"path"`
|
||||
Error string `json:"error,omitempty"`
|
||||
WriteSpeed float64 `json:"writeSpeed"`
|
||||
ReadSpeed float64 `json:"readSpeed"`
|
||||
}
|
||||
|
||||
// GetPerformance returns given disk's read and write performance
|
||||
func GetPerformance(path string) Performance {
|
||||
perf := Performance{}
|
||||
write, read, err := doPerfMeasure(path)
|
||||
if err != nil {
|
||||
perf.Error = err.Error()
|
||||
return perf
|
||||
}
|
||||
perf.WriteSpeed = write
|
||||
perf.ReadSpeed = read
|
||||
return perf
|
||||
}
|
||||
|
||||
// Calculate the write and read performance - write and read 10 tmp (1 MiB)
|
||||
// files and find the average time taken (Bytes / Sec)
|
||||
func doPerfMeasure(fsPath string) (write, read float64, err error) {
|
||||
var count int
|
||||
var totalWriteElapsed time.Duration
|
||||
var totalReadElapsed time.Duration
|
||||
|
||||
defer os.RemoveAll(fsPath)
|
||||
|
||||
randBuf := make([]byte, randBufSize)
|
||||
rand.Read(randBuf)
|
||||
buf := bytes.Repeat(randBuf, randParts)
|
||||
|
||||
// create the enclosing directory
|
||||
err = os.MkdirAll(fsPath, 0777)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
for count = 1; count <= iterations; count++ {
|
||||
fsTempObjPath := path.Join(fsPath, strconv.Itoa(count))
|
||||
|
||||
// Write performance calculation
|
||||
writeStart := time.Now()
|
||||
n, err := writeFile(fsTempObjPath, buf)
|
||||
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
if n != fileSize {
|
||||
return 0, 0, errors.New("Could not write temporary data to disk")
|
||||
}
|
||||
|
||||
writeElapsed := time.Since(writeStart)
|
||||
totalWriteElapsed += writeElapsed
|
||||
|
||||
// Read performance calculation
|
||||
readStart := time.Now()
|
||||
n, err = readFile(fsTempObjPath, buf)
|
||||
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
if n != fileSize {
|
||||
return 0, 0, errors.New("Could not read temporary data from disk")
|
||||
}
|
||||
|
||||
readElapsed := time.Since(readStart)
|
||||
totalReadElapsed += readElapsed
|
||||
}
|
||||
// Average time spent = total time elapsed / number of writes
|
||||
avgWriteTime := totalWriteElapsed.Seconds() / float64(count)
|
||||
// Write perf = fileSize (in Bytes) / average time spent writing (in seconds)
|
||||
write = fileSize / avgWriteTime
|
||||
|
||||
// Average time spent = total time elapsed / number of writes
|
||||
avgReadTime := totalReadElapsed.Seconds() / float64(count)
|
||||
// read perf = fileSize (in Bytes) / average time spent reading (in seconds)
|
||||
read = fileSize / avgReadTime
|
||||
|
||||
return write, read, nil
|
||||
}
|
||||
|
52
pkg/disk/helpers.go
Normal file
52
pkg/disk/helpers.go
Normal file
@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package disk
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
func readFile(path string, buf []byte) (int, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
n, err := f.Read(buf)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func writeFile(path string, data []byte) (int, error) {
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
n, err := f.Write(data)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
f.Sync()
|
||||
return n, nil
|
||||
}
|
@ -39,7 +39,7 @@ func main() {
|
||||
| Service operations | Info operations | Healing operations | Config operations | IAM operations | Misc |
|
||||
|:----------------------------|:----------------------------|:--------------------------------------|:--------------------------|:------------------------------------|:------------------------------------|
|
||||
| [`ServiceStatus`](#ServiceStatus) | [`ServerInfo`](#ServerInfo) | [`Heal`](#Heal) | [`GetConfig`](#GetConfig) | [`AddUser`](#AddUser) | [`SetAdminCredentials`](#SetAdminCredentials) |
|
||||
| [`ServiceSendAction`](#ServiceSendAction) | | | [`SetConfig`](#SetConfig) | [`SetUserPolicy`](#SetUserPolicy) | [`StartProfiling`](#StartProfiling) |
|
||||
| [`ServiceSendAction`](#ServiceSendAction) | [`ServerDrivesPerfInfo`](#ServerDrivesPerfInfo) | [`SetConfig`](#SetConfig) | [`SetUserPolicy`](#SetUserPolicy) | [`StartProfiling`](#StartProfiling) |
|
||||
| | | | [`GetConfigKeys`](#GetConfigKeys) | [`ListUsers`](#ListUsers) | [`DownloadProfilingData`](#DownloadProfilingData) |
|
||||
| | | | [`SetConfigKeys`](#SetConfigKeys) | [`AddCannedPolicy`](#AddCannedPolicy) | |
|
||||
|
||||
@ -204,6 +204,23 @@ Fetches information for all cluster nodes, such as server properties, storage in
|
||||
|
||||
```
|
||||
|
||||
<a name="ServerDrivesPerfInfo"></a>
|
||||
### ServerDrivesPerfInfo() ([]ServerDrivesPerfInfo, error)
|
||||
|
||||
Fetches drive performance information for all cluster nodes. Returned value is in Bytes/s.
|
||||
|
||||
| Param | Type | Description |
|
||||
|---|---|---|
|
||||
|`di.Addr` | _string_ | Address of the server the following information is retrieved from. |
|
||||
|`di.Error` | _string _ | Errors (if any) encountered while reaching this node |
|
||||
|`di.DrivesPerf` | _disk.Performance_ | Path of the drive mount on above server and read, write speed. |
|
||||
|
||||
| Param | Type | Description |
|
||||
|---|---|---|
|
||||
|`disk.Performance.Path` | _string_ | Path of drive mount. |
|
||||
|`disk.Performance.Error` | _string_ | Error (if any) encountered while accessing this drive. |
|
||||
|`disk.Performance.WriteSpeed` | _float64_ | Write speed on above path in Bytes/s. |
|
||||
|`disk.Performance.ReadSpeed` | _float64_ | Read speed on above path in Bytes/s. |
|
||||
|
||||
## 6. Heal operations
|
||||
|
||||
|
44
pkg/madmin/examples/drives-perf-info.go
Normal file
44
pkg/madmin/examples/drives-perf-info.go
Normal file
@ -0,0 +1,44 @@
|
||||
// +build ignore
|
||||
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are
|
||||
// dummy values, please replace them with original values.
|
||||
|
||||
// API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise.
|
||||
// New returns an Minio Admin client object.
|
||||
madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
st, err := madmClnt.ServerDrivesPerfInfo()
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
log.Println(st)
|
||||
}
|
@ -21,7 +21,10 @@ import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/pkg/disk"
|
||||
)
|
||||
|
||||
// BackendType - represents different backend types.
|
||||
@ -147,3 +150,46 @@ func (adm *AdminClient) ServerInfo() ([]ServerInfo, error) {
|
||||
|
||||
return serversInfo, nil
|
||||
}
|
||||
|
||||
// ServerDrivesPerfInfo holds informantion about address and write speed of
|
||||
// all drives in a single server node
|
||||
type ServerDrivesPerfInfo struct {
|
||||
Addr string `json:"addr"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Perf []disk.Performance `json:"perf"`
|
||||
}
|
||||
|
||||
// ServerDrivesPerfInfo - Returns drive's read and write performance information
|
||||
func (adm *AdminClient) ServerDrivesPerfInfo() ([]ServerDrivesPerfInfo, error) {
|
||||
v := url.Values{}
|
||||
v.Set("perfType", string("drive"))
|
||||
resp, err := adm.executeMethod("GET", requestData{
|
||||
relPath: "/v1/performance",
|
||||
queryValues: v,
|
||||
})
|
||||
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check response http status code
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, httpRespToErrorResponse(resp)
|
||||
}
|
||||
|
||||
// Unmarshal the server's json response
|
||||
var info []ServerDrivesPerfInfo
|
||||
|
||||
respBytes, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = json.Unmarshal(respBytes, &info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return info, nil
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user