mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
simplify the reader for speedtest (#13682)
additionally count only success operations, truncated incomplete calls don't need to be counted.
This commit is contained in:
parent
17fd71164c
commit
556ae07857
@ -19,7 +19,6 @@ package cmd
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/rand"
|
|
||||||
"encoding/gob"
|
"encoding/gob"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@ -40,6 +39,7 @@ import (
|
|||||||
"github.com/minio/minio/internal/event"
|
"github.com/minio/minio/internal/event"
|
||||||
"github.com/minio/minio/internal/hash"
|
"github.com/minio/minio/internal/hash"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
|
"github.com/minio/pkg/randreader"
|
||||||
"github.com/tinylib/msgp/msgp"
|
"github.com/tinylib/msgp/msgp"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -1125,31 +1125,8 @@ type SpeedtestResult struct {
|
|||||||
Error string
|
Error string
|
||||||
}
|
}
|
||||||
|
|
||||||
// SpeedtestObject implements "random-read" object reader
|
func newRandomReader(size int) io.Reader {
|
||||||
type SpeedtestObject struct {
|
return io.LimitReader(randreader.New(), int64(size))
|
||||||
buf []byte
|
|
||||||
remaining int
|
|
||||||
totalBytesWritten *uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
func (bo *SpeedtestObject) Read(b []byte) (int, error) {
|
|
||||||
if bo.remaining == 0 {
|
|
||||||
return 0, io.EOF
|
|
||||||
}
|
|
||||||
if len(b) == 0 {
|
|
||||||
return 0, nil
|
|
||||||
}
|
|
||||||
if len(b) > len(bo.buf) {
|
|
||||||
b = b[:len(bo.buf)]
|
|
||||||
}
|
|
||||||
if len(b) > bo.remaining {
|
|
||||||
b = b[:bo.remaining]
|
|
||||||
}
|
|
||||||
copy(b, bo.buf)
|
|
||||||
bo.remaining -= len(b)
|
|
||||||
|
|
||||||
atomic.AddUint64(bo.totalBytesWritten, uint64(len(b)))
|
|
||||||
return len(b), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Runs the speedtest on local MinIO process.
|
// Runs the speedtest on local MinIO process.
|
||||||
@ -1162,10 +1139,6 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
|
|||||||
var retError string
|
var retError string
|
||||||
|
|
||||||
bucket := minioMetaSpeedTestBucket
|
bucket := minioMetaSpeedTestBucket
|
||||||
|
|
||||||
buf := make([]byte, humanize.MiByte)
|
|
||||||
rand.Read(buf)
|
|
||||||
|
|
||||||
objCountPerThread := make([]uint64, concurrent)
|
objCountPerThread := make([]uint64, concurrent)
|
||||||
|
|
||||||
uploadsStopped := false
|
uploadsStopped := false
|
||||||
@ -1187,7 +1160,7 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
|
|||||||
go func(i int) {
|
go func(i int) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
for {
|
for {
|
||||||
hashReader, err := hash.NewReader(&SpeedtestObject{buf, size, &totalBytesWritten},
|
hashReader, err := hash.NewReader(newRandomReader(size),
|
||||||
int64(size), "", "", int64(size))
|
int64(size), "", "", int64(size))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
retError = err.Error()
|
retError = err.Error()
|
||||||
@ -1195,7 +1168,7 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
|
|||||||
break
|
break
|
||||||
}
|
}
|
||||||
reader := NewPutObjReader(hashReader)
|
reader := NewPutObjReader(hashReader)
|
||||||
_, err = objAPI.PutObject(uploadsCtx, bucket, fmt.Sprintf("%s.%d.%d",
|
objInfo, err := objAPI.PutObject(uploadsCtx, bucket, fmt.Sprintf("%s.%d.%d",
|
||||||
objNamePrefix, i, objCountPerThread[i]), reader, ObjectOptions{})
|
objNamePrefix, i, objCountPerThread[i]), reader, ObjectOptions{})
|
||||||
if err != nil && !uploadsStopped {
|
if err != nil && !uploadsStopped {
|
||||||
retError = err.Error()
|
retError = err.Error()
|
||||||
@ -1204,6 +1177,7 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
atomic.AddUint64(&totalBytesWritten, uint64(objInfo.Size))
|
||||||
objCountPerThread[i]++
|
objCountPerThread[i]++
|
||||||
}
|
}
|
||||||
}(i)
|
}(i)
|
||||||
@ -1243,8 +1217,12 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
|
|||||||
}
|
}
|
||||||
n, err := io.Copy(ioutil.Discard, r)
|
n, err := io.Copy(ioutil.Discard, r)
|
||||||
r.Close()
|
r.Close()
|
||||||
|
if err == nil {
|
||||||
atomic.AddUint64(&totalBytesRead, uint64(n))
|
// Only capture success criteria - do not
|
||||||
|
// have to capture failed reads, truncated
|
||||||
|
// reads etc.
|
||||||
|
atomic.AddUint64(&totalBytesRead, uint64(n))
|
||||||
|
}
|
||||||
if err != nil && !downloadsStopped {
|
if err != nil && !downloadsStopped {
|
||||||
retError = err.Error()
|
retError = err.Error()
|
||||||
logger.LogIf(ctx, err)
|
logger.LogIf(ctx, err)
|
||||||
|
2
go.mod
2
go.mod
@ -52,7 +52,7 @@ require (
|
|||||||
github.com/minio/mc v0.0.0-20211115052100-7fd441ec6c5b // indirect
|
github.com/minio/mc v0.0.0-20211115052100-7fd441ec6c5b // indirect
|
||||||
github.com/minio/minio-go/v7 v7.0.16-0.20211108161804-a7a36ee131df
|
github.com/minio/minio-go/v7 v7.0.16-0.20211108161804-a7a36ee131df
|
||||||
github.com/minio/parquet-go v1.1.0
|
github.com/minio/parquet-go v1.1.0
|
||||||
github.com/minio/pkg v1.1.7
|
github.com/minio/pkg v1.1.8
|
||||||
github.com/minio/selfupdate v0.3.1
|
github.com/minio/selfupdate v0.3.1
|
||||||
github.com/minio/sha256-simd v1.0.0
|
github.com/minio/sha256-simd v1.0.0
|
||||||
github.com/minio/simdjson-go v0.2.1
|
github.com/minio/simdjson-go v0.2.1
|
||||||
|
3
go.sum
3
go.sum
@ -1111,8 +1111,9 @@ github.com/minio/pkg v1.0.3/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP
|
|||||||
github.com/minio/pkg v1.0.4/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8=
|
github.com/minio/pkg v1.0.4/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8=
|
||||||
github.com/minio/pkg v1.0.11/go.mod h1:32x/3OmGB0EOi1N+3ggnp+B5VFkSBBB9svPMVfpnf14=
|
github.com/minio/pkg v1.0.11/go.mod h1:32x/3OmGB0EOi1N+3ggnp+B5VFkSBBB9svPMVfpnf14=
|
||||||
github.com/minio/pkg v1.1.3/go.mod h1:32x/3OmGB0EOi1N+3ggnp+B5VFkSBBB9svPMVfpnf14=
|
github.com/minio/pkg v1.1.3/go.mod h1:32x/3OmGB0EOi1N+3ggnp+B5VFkSBBB9svPMVfpnf14=
|
||||||
github.com/minio/pkg v1.1.7 h1:v+2/ol/h1Sl0iJdOFN1Srk4CzksMIDsfugXCZYb5L7Y=
|
|
||||||
github.com/minio/pkg v1.1.7/go.mod h1:32x/3OmGB0EOi1N+3ggnp+B5VFkSBBB9svPMVfpnf14=
|
github.com/minio/pkg v1.1.7/go.mod h1:32x/3OmGB0EOi1N+3ggnp+B5VFkSBBB9svPMVfpnf14=
|
||||||
|
github.com/minio/pkg v1.1.8 h1:m/yKFtUCmLKp6kOO6Wf8C2SRUIIhtkIf6+rGlrB5RVk=
|
||||||
|
github.com/minio/pkg v1.1.8/go.mod h1:32x/3OmGB0EOi1N+3ggnp+B5VFkSBBB9svPMVfpnf14=
|
||||||
github.com/minio/selfupdate v0.3.1 h1:BWEFSNnrZVMUWXbXIgLDNDjbejkmpAmZvy/nCz1HlEs=
|
github.com/minio/selfupdate v0.3.1 h1:BWEFSNnrZVMUWXbXIgLDNDjbejkmpAmZvy/nCz1HlEs=
|
||||||
github.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM=
|
github.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM=
|
||||||
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
|
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
|
||||||
|
Loading…
Reference in New Issue
Block a user