From 1a40c7c27c32166945d828a87dd096a73909fee4 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 22 Jun 2022 16:28:25 -0700 Subject: [PATCH] use signature-v2 for 'object perf' tests to avoid CPU using sha256 (#15151) It is observed in a local 8 drive system the CPU seems to be bottlenecked at ``` (pprof) top Showing nodes accounting for 1385.31s, 88.47% of 1565.88s total Dropped 1304 nodes (cum <= 7.83s) Showing top 10 nodes out of 159 flat flat% sum% cum cum% 724s 46.24% 46.24% 724s 46.24% crypto/sha256.block 219.04s 13.99% 60.22% 226.63s 14.47% syscall.Syscall 158.04s 10.09% 70.32% 158.04s 10.09% runtime.memmove 127.58s 8.15% 78.46% 127.58s 8.15% crypto/md5.block 58.67s 3.75% 82.21% 58.67s 3.75% github.com/minio/highwayhash.updateAVX2 40.07s 2.56% 84.77% 40.07s 2.56% runtime.epollwait 33.76s 2.16% 86.93% 33.76s 2.16% github.com/klauspost/reedsolomon._galMulAVX512Parallel84 8.88s 0.57% 87.49% 11.56s 0.74% runtime.step 7.84s 0.5% 87.99% 7.84s 0.5% runtime.memclrNoHeapPointers 7.43s 0.47% 88.47% 22.18s 1.42% runtime.pcvalue ``` Bonus changes: - re-use transport for bucket replication clients, also site replication clients. - use 32KiB buffer for all read and writes at transport layer seems to help TLS read connections. - Do not have 'MaxConnsPerHost' this is problematic to be used with net/http connection pooling 'MaxIdleConnsPerHost' is enough. --- cmd/bucket-targets.go | 12 +----------- cmd/encryption-v1.go | 2 +- cmd/globals.go | 2 ++ cmd/perf-tests.go | 2 +- cmd/server-main.go | 1 + cmd/site-replication.go | 4 ++-- cmd/utils.go | 21 +++++++++++---------- 7 files changed, 19 insertions(+), 25 deletions(-) diff --git a/cmd/bucket-targets.go b/cmd/bucket-targets.go index 26dc5685b..01a0ea184 100644 --- a/cmd/bucket-targets.go +++ b/cmd/bucket-targets.go @@ -19,7 +19,6 @@ package cmd import ( "context" - "net/http" "sync" "time" @@ -325,25 +324,16 @@ func (sys *BucketTargetSys) set(bucket BucketInfo, meta BucketMetadata) { sys.targetsMap[bucket.Name] = cfg.Targets } -// getRemoteTargetInstanceTransport contains a singleton roundtripper. -var ( - getRemoteTargetInstanceTransport http.RoundTripper - getRemoteTargetInstanceTransportOnce sync.Once -) - // Returns a minio-go Client configured to access remote host described in replication target config. func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*TargetClient, error) { config := tcfg.Credentials creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, "") - getRemoteTargetInstanceTransportOnce.Do(func() { - getRemoteTargetInstanceTransport = NewRemoteTargetHTTPTransport() - }) api, err := minio.New(tcfg.Endpoint, &miniogo.Options{ Creds: creds, Secure: tcfg.Secure, Region: tcfg.Region, - Transport: getRemoteTargetInstanceTransport, + Transport: globalRemoteTargetTransport, }) if err != nil { return nil, err diff --git a/cmd/encryption-v1.go b/cmd/encryption-v1.go index d48c7c7ed..f5ecc7e98 100644 --- a/cmd/encryption-v1.go +++ b/cmd/encryption-v1.go @@ -22,7 +22,6 @@ import ( "context" "crypto/hmac" "crypto/rand" - "crypto/sha256" "crypto/subtle" "encoding/binary" "encoding/hex" @@ -38,6 +37,7 @@ import ( "github.com/minio/minio/internal/crypto" "github.com/minio/minio/internal/etag" "github.com/minio/minio/internal/fips" + "github.com/minio/minio/internal/hash/sha256" xhttp "github.com/minio/minio/internal/http" "github.com/minio/minio/internal/kms" "github.com/minio/minio/internal/logger" diff --git a/cmd/globals.go b/cmd/globals.go index 3a968ad71..5ae76edb4 100644 --- a/cmd/globals.go +++ b/cmd/globals.go @@ -338,6 +338,8 @@ var ( globalProxyTransport http.RoundTripper + globalRemoteTargetTransport http.RoundTripper + globalDNSCache = &dnscache.Resolver{ Timeout: 5 * time.Second, } diff --git a/cmd/perf-tests.go b/cmd/perf-tests.go index bdf527479..4c0ef8b22 100644 --- a/cmd/perf-tests.go +++ b/cmd/perf-tests.go @@ -68,7 +68,7 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura } client, err := minio.New(globalLocalNodeName, &minio.Options{ - Creds: credentials.NewStaticV4(globalActiveCred.AccessKey, globalActiveCred.SecretKey, ""), + Creds: credentials.NewStaticV2(globalActiveCred.AccessKey, globalActiveCred.SecretKey, ""), Secure: globalIsTLS, Transport: globalProxyTransport, Region: region, diff --git a/cmd/server-main.go b/cmd/server-main.go index 5be160cff..3efc62946 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -217,6 +217,7 @@ func serverHandleCmdArgs(ctx *cli.Context) { CurvePreferences: fips.TLSCurveIDs(), ClientSessionCache: tls.NewLRUClientSessionCache(tlsClientSessionCacheSize), }, rest.DefaultTimeout)() + globalRemoteTargetTransport = NewRemoteTargetHTTPTransport()() // On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back // to IPv6 address ie minio will start listening on IPv6 address whereas another diff --git a/cmd/site-replication.go b/cmd/site-replication.go index 5418ca64e..1724da237 100644 --- a/cmd/site-replication.go +++ b/cmd/site-replication.go @@ -2094,7 +2094,7 @@ func getAdminClient(endpoint, accessKey, secretKey string) (*madmin.AdminClient, if err != nil { return nil, err } - client.SetCustomTransport(NewRemoteTargetHTTPTransport()) + client.SetCustomTransport(globalRemoteTargetTransport) return client, nil } @@ -2106,7 +2106,7 @@ func getS3Client(pc madmin.PeerSite) (*minioClient.Client, error) { return minioClient.New(ep.Host, &minioClient.Options{ Creds: credentials.NewStaticV4(pc.AccessKey, pc.SecretKey, ""), Secure: ep.Scheme == "https", - Transport: NewRemoteTargetHTTPTransport(), + Transport: globalRemoteTargetTransport, }) } diff --git a/cmd/utils.go b/cmd/utils.go index 8ef0ef0cf..5443137a3 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -561,9 +561,8 @@ func newCustomHTTPProxyTransport(tlsConfig *tls.Config, dialTimeout time.Duratio Proxy: http.ProxyFromEnvironment, DialContext: xhttp.DialContextWithDNSCache(globalDNSCache, xhttp.NewInternodeDialContext(dialTimeout)), MaxIdleConnsPerHost: 1024, - MaxConnsPerHost: 1024, - WriteBufferSize: 16 << 10, // 16KiB moving up from 4KiB default - ReadBufferSize: 16 << 10, // 16KiB moving up from 4KiB default + WriteBufferSize: 32 << 10, // 32KiB moving up from 4KiB default + ReadBufferSize: 32 << 10, // 32KiB moving up from 4KiB default IdleConnTimeout: 15 * time.Second, ResponseHeaderTimeout: 30 * time.Minute, // Set larger timeouts for proxied requests. TLSHandshakeTimeout: 10 * time.Second, @@ -587,10 +586,10 @@ func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration) fu Proxy: http.ProxyFromEnvironment, DialContext: xhttp.DialContextWithDNSCache(globalDNSCache, xhttp.NewInternodeDialContext(dialTimeout)), MaxIdleConnsPerHost: 1024, - WriteBufferSize: 16 << 10, // 16KiB moving up from 4KiB default - ReadBufferSize: 16 << 10, // 16KiB moving up from 4KiB default + WriteBufferSize: 32 << 10, // 32KiB moving up from 4KiB default + ReadBufferSize: 32 << 10, // 32KiB moving up from 4KiB default IdleConnTimeout: 15 * time.Second, - ResponseHeaderTimeout: 3 * time.Minute, // Set conservative timeouts for MinIO internode. + ResponseHeaderTimeout: 3 * time.Minute, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 10 * time.Second, TLSClientConfig: tlsConfig, @@ -666,7 +665,7 @@ func newGatewayHTTPTransport(timeout time.Duration) *http.Transport { // NewRemoteTargetHTTPTransport returns a new http configuration // used while communicating with the remote replication targets. -func NewRemoteTargetHTTPTransport() *http.Transport { +func NewRemoteTargetHTTPTransport() func() *http.Transport { // For more details about various values used here refer // https://golang.org/pkg/net/http/#Transport documentation tr := &http.Transport{ @@ -676,8 +675,8 @@ func NewRemoteTargetHTTPTransport() *http.Transport { KeepAlive: 30 * time.Second, }).DialContext, MaxIdleConnsPerHost: 1024, - WriteBufferSize: 16 << 10, // 16KiB moving up from 4KiB default - ReadBufferSize: 16 << 10, // 16KiB moving up from 4KiB default + WriteBufferSize: 32 << 10, // 32KiB moving up from 4KiB default + ReadBufferSize: 32 << 10, // 32KiB moving up from 4KiB default IdleConnTimeout: 15 * time.Second, TLSHandshakeTimeout: 5 * time.Second, ExpectContinueTimeout: 5 * time.Second, @@ -690,7 +689,9 @@ func NewRemoteTargetHTTPTransport() *http.Transport { // in raw stream. DisableCompression: true, } - return tr + return func() *http.Transport { + return tr + } } // Load the json (typically from disk file).