MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-01-11 15:03:22 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add ability to test drive speeds on a MinIO setup (#7664)

Browse Source 
- Extends existing Admin API to measure disk performance
This commit is contained in:
Krishnan Parthasarathi 2019-09-12 14:52:30 -07:00 committed by kannappanr
parent e7b3f39064
commit 6ba323b009
14 changed files with 178 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
cmd/admin-handlers.go
View File

@ -45,7 +45,6 @@ import (
xhttp "github.com/minio/minio/cmd/http"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/cpu"
"github.com/minio/minio/pkg/disk"
"github.com/minio/minio/pkg/handlers"
iampolicy "github.com/minio/minio/pkg/iam/policy"
"github.com/minio/minio/pkg/madmin"
@ -318,15 +317,6 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
writeSuccessResponseJSON(w, jsonBytes)
}
// ServerDrivesPerfInfo holds information about address, performance
// of all drives on one server. It also reports any errors if encountered
// while trying to reach this server.
type ServerDrivesPerfInfo struct {
Addr string `json:"addr"`
Error string `json:"error,omitempty"`
Perf []disk.Performance `json:"perf"`
}
// ServerCPULoadInfo holds informantion about cpu utilization
// of one minio node. It also reports any errors if encountered
// while trying to reach this server.
@ -406,16 +396,25 @@ func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request
writeSuccessResponseJSON(w, jsonBytes)
case "drive":
info := objectAPI.StorageInfo(ctx)
if !(info.Backend.Type == BackendFS || info.Backend.Type == BackendErasure) {
// Drive Perf is only implemented for Erasure coded backends
if !globalIsXL {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
return
}
var size int64 = madmin.DefaultDrivePerfSize
if sizeStr, found := vars["size"]; found {
var err error
if size, err = strconv.ParseInt(sizeStr, 10, 64); err != nil || size <= 0 {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
return
}
}
// Get drive performance details from local server's drive(s)
dp := getLocalDrivesPerf(globalEndpoints, r)
dp := getLocalDrivesPerf(globalEndpoints, size, r)
// Notify all other MinIO peers to report drive performance numbers
dps := globalNotificationSys.DrivePerfInfo()
dps := globalNotificationSys.DrivePerfInfo(size)
dps = append(dps, dp)
// Marshal API response

2
cmd/admin-router.go
View File

@ -64,7 +64,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
}
// Performance command - return performance details based on input type
adminV1Router.Methods(http.MethodGet).Path("/performance").HandlerFunc(httpTraceAll(adminAPI.PerfInfoHandler)).Queries("perfType", "{perfType:.*}")
adminV1Router.Methods(http.MethodGet).Path("/performance").HandlerFunc(httpTraceAll(adminAPI.PerfInfoHandler)).Queries("perfType", "{perfType:.*}").Queries("size", "{size:.*}")
// Profiling operations
adminV1Router.Methods(http.MethodPost).Path("/profiling/start").HandlerFunc(httpTraceAll(adminAPI.StartProfilingHandler)).

7
cmd/admin-server-info.go
View File

@ -23,6 +23,7 @@ import (
"github.com/minio/minio-go/pkg/set"
"github.com/minio/minio/pkg/cpu"
"github.com/minio/minio/pkg/disk"
"github.com/minio/minio/pkg/madmin"
"github.com/minio/minio/pkg/mem"
)
@ -86,7 +87,7 @@ func getLocalCPULoad(endpoints EndpointList, r *http.Request) ServerCPULoadInfo
// getLocalDrivesPerf - returns ServerDrivesPerfInfo for only the
// local endpoints from given list of endpoints
func getLocalDrivesPerf(endpoints EndpointList, r *http.Request) ServerDrivesPerfInfo {
func getLocalDrivesPerf(endpoints EndpointList, size int64, r *http.Request) madmin.ServerDrivesPerfInfo {
var dps []disk.Performance
for _, endpoint := range endpoints {
// Only proceed for local endpoints
@ -96,7 +97,7 @@ func getLocalDrivesPerf(endpoints EndpointList, r *http.Request) ServerDrivesPer
dps = append(dps, disk.Performance{Path: endpoint.Path, Error: err.Error()})
continue
}
dp := disk.GetPerformance(pathJoin(endpoint.Path, minioMetaTmpBucket, mustGetUUID()))
dp := disk.GetPerformance(pathJoin(endpoint.Path, minioMetaTmpBucket, mustGetUUID()), size)
dp.Path = endpoint.Path
dps = append(dps, dp)
}
@ -105,7 +106,7 @@ func getLocalDrivesPerf(endpoints EndpointList, r *http.Request) ServerDrivesPer
if globalIsDistXL {
addr = GetLocalPeer(endpoints)
}
return ServerDrivesPerfInfo{
return madmin.ServerDrivesPerfInfo{
Addr: addr,
Perf: dps,
}

6
cmd/notification.go
View File

@ -977,8 +977,8 @@ func (sys *NotificationSys) CollectNetPerfInfo(size int64) map[string][]ServerNe
}
// DrivePerfInfo - Drive speed (read and write) information
func (sys *NotificationSys) DrivePerfInfo() []ServerDrivesPerfInfo {
reply := make([]ServerDrivesPerfInfo, len(sys.peerClients))
func (sys *NotificationSys) DrivePerfInfo(size int64) []madmin.ServerDrivesPerfInfo {
reply := make([]madmin.ServerDrivesPerfInfo, len(sys.peerClients))
var wg sync.WaitGroup
for i, client := range sys.peerClients {
if client == nil {
@ -987,7 +987,7 @@ func (sys *NotificationSys) DrivePerfInfo() []ServerDrivesPerfInfo {
wg.Add(1)
go func(client *peerRESTClient, idx int) {
defer wg.Done()
di, err := client.DrivePerfInfo()
di, err := client.DrivePerfInfo(size)
if err != nil {
reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", client.host.String())
ctx := logger.SetReqInfo(context.Background(), reqInfo)

6
cmd/peer-rest-client.go
View File

@ -173,8 +173,10 @@ func (client *peerRESTClient) CPULoadInfo() (info ServerCPULoadInfo, err error)
}
// DrivePerfInfo - fetch Drive performance information for a remote node.
func (client *peerRESTClient) DrivePerfInfo() (info ServerDrivesPerfInfo, err error) {
respBody, err := client.call(peerRESTMethodDrivePerfInfo, nil, nil, -1)
func (client *peerRESTClient) DrivePerfInfo(size int64) (info madmin.ServerDrivesPerfInfo, err error) {
params := make(url.Values)
params.Set(peerRESTDrivePerfSize, strconv.FormatInt(size, 10))
respBody, err := client.call(peerRESTMethodDrivePerfInfo, params, nil, -1)
if err != nil {
return
}

1
cmd/peer-rest-common.go
View File

@ -56,6 +56,7 @@ const (
const (
peerRESTNetPerfSize = "netperfsize"
peerRESTDrivePerfSize = "driveperfsize"
peerRESTBucket = "bucket"
peerRESTUser = "user"
peerRESTGroup = "group"

19
cmd/peer-rest-server.go
View File

@ -431,8 +431,23 @@ func (s *peerRESTServer) DrivePerfInfoHandler(w http.ResponseWriter, r *http.Req
return
}
params := mux.Vars(r)
sizeStr, found := params[peerRESTDrivePerfSize]
if !found {
s.writeErrorResponse(w, errors.New("size is missing"))
return
}
size, err := strconv.ParseInt(sizeStr, 10, 64)
if err != nil || size < 0 {
s.writeErrorResponse(w, errInvalidArgument)
return
}
ctx := newContext(r, w, "DrivePerfInfo")
info := getLocalDrivesPerf(globalEndpoints, r)
info := getLocalDrivesPerf(globalEndpoints, size, r)
defer w.(http.Flusher).Flush()
logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
@ -960,7 +975,7 @@ func registerPeerRESTHandlers(router *mux.Router) {
subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodServerInfo).HandlerFunc(httpTraceHdrs(server.ServerInfoHandler))
subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodCPULoadInfo).HandlerFunc(httpTraceHdrs(server.CPULoadInfoHandler))
subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodMemUsageInfo).HandlerFunc(httpTraceHdrs(server.MemUsageInfoHandler))
subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodDrivePerfInfo).HandlerFunc(httpTraceHdrs(server.DrivePerfInfoHandler))
subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodDrivePerfInfo).HandlerFunc(httpTraceHdrs(server.DrivePerfInfoHandler)).Queries(restQueries(peerRESTDrivePerfSize)...)
subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodDeleteBucket).HandlerFunc(httpTraceHdrs(server.DeleteBucketHandler)).Queries(restQueries(peerRESTBucket)...)
subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodSignalService).HandlerFunc(httpTraceHdrs(server.SignalServiceHandler)).Queries(restQueries(peerRESTSignal)...)
subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodServerUpdate).HandlerFunc(httpTraceHdrs(server.ServerUpdateHandler)).Queries(restQueries(peerRESTUpdateURL, peerRESTSha256Hex, peerRESTLatestRelease)...)

4
go.mod
View File

@ -65,8 +65,8 @@ require (
github.com/tidwall/sjson v1.0.4
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a
go.uber.org/atomic v1.3.2
golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456
google.golang.org/api v0.4.0
gopkg.in/Shopify/sarama.v1 v1.20.0
gopkg.in/ldap.v3 v3.0.3

6
go.sum
View File

@ -667,9 +667,8 @@ golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACk
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0=
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 h1:Gv7RPwsi3eZ2Fgewe3CBsuOebPwO27PoXzRpJPsvSSM=
golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -739,9 +738,8 @@ golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=

95
pkg/disk/disk.go
View File

@ -1,5 +1,5 @@
/*
* MinIO Cloud Storage, (C) 2018 MinIO, Inc.
* MinIO Cloud Storage, (C) 2018-2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -17,25 +17,9 @@
package disk
import (
"bytes"
"crypto/rand"
"errors"
"os"
"path"
"strconv"
"time"
humanize "github.com/dustin/go-humanize"
)
// file size for performance read and write checks
const (
randBufSize = 1 * humanize.KiByte
randParts = 1024
fileSize = randParts * randBufSize
// Total count of read / write iteration for performance measurement
iterations = 10
"github.com/ncw/directio"
)
// Info stat fs struct is container which holds following values
@ -64,9 +48,9 @@ type Performance struct {
}
// GetPerformance returns given disk's read and write performance
func GetPerformance(path string) Performance {
func GetPerformance(path string, size int64) Performance {
perf := Performance{}
write, read, err := doPerfMeasure(path)
write, read, err := doPerfMeasure(path, size)
if err != nil {
perf.Error = err.Error()
return perf
@ -78,63 +62,36 @@ func GetPerformance(path string) Performance {
// Calculate the write and read performance - write and read 10 tmp (1 MiB)
// files and find the average time taken (Bytes / Sec)
func doPerfMeasure(fsPath string) (write, read float64, err error) {
var count int
var totalWriteElapsed time.Duration
var totalReadElapsed time.Duration
func doPerfMeasure(fsPath string, size int64) (writeSpeed, readSpeed float64, err error) {
// Remove the file created for speed test purposes
defer os.RemoveAll(fsPath)
randBuf := make([]byte, randBufSize)
rand.Read(randBuf)
buf := bytes.Repeat(randBuf, randParts)
// create the enclosing directory
err = os.MkdirAll(fsPath, 0777)
// Create a file with O_DIRECT flag
w, err := OpenFileDirectIO(fsPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0666)
if err != nil {
return 0, 0, err
}
for count = 1; count <= iterations; count++ {
fsTempObjPath := path.Join(fsPath, strconv.Itoa(count))
// Fetch aligned buf for direct-io
buf := directio.AlignedBlock(speedTestBlockSize)
// Write performance calculation
writeStart := time.Now()
n, err := writeFile(fsTempObjPath, buf)
if err != nil {
return 0, 0, err
}
if n != fileSize {
return 0, 0, errors.New("Could not write temporary data to disk")
}
writeElapsed := time.Since(writeStart)
totalWriteElapsed += writeElapsed
// Read performance calculation
readStart := time.Now()
n, err = readFile(fsTempObjPath, buf)
if err != nil {
return 0, 0, err
}
if n != fileSize {
return 0, 0, errors.New("Could not read temporary data from disk")
}
readElapsed := time.Since(readStart)
totalReadElapsed += readElapsed
writeSpeed, err = speedTestWrite(w, buf, size)
w.Close()
if err != nil {
return 0, 0, err
}
// Average time spent = total time elapsed / number of writes
avgWriteTime := totalWriteElapsed.Seconds() / float64(count)
// Write perf = fileSize (in Bytes) / average time spent writing (in seconds)
write = fileSize / avgWriteTime
// Average time spent = total time elapsed / number of writes
avgReadTime := totalReadElapsed.Seconds() / float64(count)
// read perf = fileSize (in Bytes) / average time spent reading (in seconds)
read = fileSize / avgReadTime
// Open file to compute read speed
r, err := OpenFileDirectIO(fsPath, os.O_RDONLY, 0666)
if err != nil {
return 0, 0, err
}
defer r.Close()
return write, read, nil
readSpeed, err = speedTestRead(r, buf, size)
if err != nil {
return 0, 0, err
}
return writeSpeed, readSpeed, nil
}

52
pkg/disk/helpers.go
View File

@ -1,52 +0,0 @@
/*
* MinIO Cloud Storage, (C) 2018 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package disk
import (
"os"
)
func readFile(path string, buf []byte) (int, error) {
f, err := os.Open(path)
if err != nil {
return 0, err
}
defer f.Close()
n, err := f.Read(buf)
if err != nil {
return 0, err
}
return n, nil
}
func writeFile(path string, data []byte) (int, error) {
f, err := os.Create(path)
if err != nil {
return 0, err
}
defer f.Close()
n, err := f.Write(data)
if err != nil {
return 0, err
}
f.Sync()
return n, nil
}

97
pkg/disk/speedtest.go Normal file
View File

@ -0,0 +1,97 @@
/*
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package disk
import (
"io"
"math"
"time"
humanize "github.com/dustin/go-humanize"
)
var speedTestBlockSize = 4 * humanize.MiByte
// speedTestWrite computes the write speed by writing
// `speedTestFileSize` bytes of data to `w` in 4MiB direct-aligned
// blocks present in `buf`
func speedTestWrite(w io.Writer, buf []byte, size int64) (float64, error) {
// Write speedTestFileSize of data and record write speed
startTime := time.Now()
remaining := size
for remaining > 0 {
var toWrite int
// there's more remaining to write than the buffer can hold
if int64(len(buf)) < remaining {
toWrite = len(buf)
} else { // buffer can hold all there is to write
toWrite = int(remaining)
}
written, err := w.Write(buf[:toWrite])
if err != nil {
return 0, err
}
remaining = remaining - int64(written)
}
elapsedTime := time.Since(startTime).Seconds()
totalWriteMBs := float64(size) / humanize.MiByte
writeSpeed := totalWriteMBs / elapsedTime
return roundToTwoDecimals(writeSpeed), nil
}
// speedTestRead computes the read speed by reading
// `speedTestFileSize` bytes from the reader `r` using 4MiB size `buf`
func speedTestRead(r io.Reader, buf []byte, size int64) (float64, error) {
// Read speedTestFileSize and record read speed
startTime := time.Now()
remaining := size
for remaining > 0 {
// reads `speedTestBlockSize` on every read
n, err := io.ReadFull(r, buf)
if err == io.ErrUnexpectedEOF || err == nil {
remaining = remaining - int64(n)
continue
}
// Nothing more left to read from the Reader
if err == io.EOF {
break
}
// Error while reading from the underlying Reader
if err != nil {
return 0, err
}
}
if remaining > 0 {
return 0, io.ErrUnexpectedEOF
}
elapsedTime := time.Since(startTime).Seconds()
totalReadMBs := float64(size) / humanize.MiByte
readSpeed := totalReadMBs / elapsedTime
return roundToTwoDecimals(readSpeed), nil
}
func roundToTwoDecimals(num float64) float64 {
return math.Round(num*100) / 100
}

2
pkg/madmin/examples/drives-perf-info.go
View File

@ -36,7 +36,7 @@ func main() {
log.Fatalln(err)
}
st, err := madmClnt.ServerDrivesPerfInfo()
st, err := madmClnt.ServerDrivesPerfInfo(madmin.DefaultDrivePerfSize)
if err != nil {
log.Fatalln(err)
}

8
pkg/madmin/info-commands.go
View File

@ -35,6 +35,8 @@ import (
const (
// DefaultNetPerfSize - default payload size used for network performance.
DefaultNetPerfSize = 100 * humanize.MiByte
// DefaultDrivePerfSize - default file size for testing drive performance
DefaultDrivePerfSize = 100 * humanize.MiByte
)
// BackendType - represents different backend types.
@ -172,12 +174,16 @@ type ServerDrivesPerfInfo struct {
Addr string `json:"addr"`
Error string `json:"error,omitempty"`
Perf []disk.Performance `json:"perf"`
Size int64 `json:"size,omitempty"`
}
// ServerDrivesPerfInfo - Returns drive's read and write performance information
func (adm *AdminClient) ServerDrivesPerfInfo() ([]ServerDrivesPerfInfo, error) {
func (adm *AdminClient) ServerDrivesPerfInfo(size int64) ([]ServerDrivesPerfInfo, error) {
v := url.Values{}
v.Set("perfType", string("drive"))
v.Set("size", strconv.FormatInt(size, 10))
resp, err := adm.executeMethod("GET", requestData{
relPath: "/v1/performance",
queryValues: v,