MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-10-29 15:55:00 -04:00
Code Issues Packages Projects Releases Wiki Activity

Remove timeout from putobject and listobjects (#9986)

Browse Source 
Use a separate client for these calls that can take a long time.

Add request context to these so they are canceled when the client 
disconnects instead except for ListObject which doesn't have any equivalent.
This commit is contained in:
Klaus Post 2020-07-07 12:19:57 -07:00 committed by GitHub
parent 93e7e4a0e5
commit aa4d1021eb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 31 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
cmd/object-handlers.go
View File

@ -701,6 +701,7 @@ func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta m
// getRemoteInstanceTransport contains a singleton roundtripper.
var getRemoteInstanceTransport http.RoundTripper
var getRemoteInstanceTransportLongTO http.RoundTripper
var getRemoteInstanceTransportOnce sync.Once
// Returns a minio-go Client configured to access remote host described by destDNSRecord
@ -715,11 +716,31 @@ var getRemoteInstanceClient = func(r *http.Request, host string) (*miniogo.Core,
}
getRemoteInstanceTransportOnce.Do(func() {
getRemoteInstanceTransport = NewGatewayHTTPTransport()
getRemoteInstanceTransportLongTO = newGatewayHTTPTransport(time.Hour)
})
core.SetCustomTransport(getRemoteInstanceTransport)
return core, nil
}
// Returns a minio-go Client configured to access remote host described by destDNSRecord
// Applicable only in a federated deployment.
// The transport does not contain any timeout except for dialing.
func getRemoteInstanceClientLongTimeout(r *http.Request, host string) (*miniogo.Core, error) {
cred := getReqAccessCred(r, globalServerRegion)
// In a federated deployment, all the instances share config files
// and hence expected to have same credentials.
core, err := miniogo.NewCore(host, cred.AccessKey, cred.SecretKey, globalIsSSL)
if err != nil {
return nil, err
}
getRemoteInstanceTransportOnce.Do(func() {
getRemoteInstanceTransport = NewGatewayHTTPTransport()
getRemoteInstanceTransportLongTO = newGatewayHTTPTransport(time.Hour)
})
core.SetCustomTransport(getRemoteInstanceTransportLongTO)
return core, nil
}
// Check if the destination bucket is on a remote site, this code only gets executed
// when federation is enabled, ie when globalDNSConfig is non 'nil'.
//
@ -1166,7 +1187,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
}
// Send PutObject request to appropriate instance (in federated deployment)
client, rerr := getRemoteInstanceClient(r, getHostFromSrv(dstRecords))
client, rerr := getRemoteInstanceClientLongTimeout(r, getHostFromSrv(dstRecords))
if rerr != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, rerr), r.URL, guessIsBrowserReq(r))
return
@ -1181,7 +1202,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
ServerSideEncryption: dstOpts.ServerSideEncryption,
UserTags: tag.ToMap(),
}
remoteObjInfo, rerr := client.PutObject(dstBucket, dstObject, srcInfo.Reader,
remoteObjInfo, rerr := client.PutObjectWithContext(ctx, dstBucket, dstObject, srcInfo.Reader,
srcInfo.Size, "", "", opts)
if rerr != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, rerr), r.URL, guessIsBrowserReq(r))
@ -1858,13 +1879,13 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
}
// Send PutObject request to appropriate instance (in federated deployment)
client, rerr := getRemoteInstanceClient(r, getHostFromSrv(dstRecords))
client, rerr := getRemoteInstanceClientLongTimeout(r, getHostFromSrv(dstRecords))
if rerr != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, rerr), r.URL, guessIsBrowserReq(r))
return
}
partInfo, err := client.PutObjectPart(dstBucket, dstObject, uploadID, partID,
partInfo, err := client.PutObjectPartWithContext(ctx, dstBucket, dstObject, uploadID, partID,
srcInfo.Reader, srcInfo.Size, "", "", dstOpts.ServerSideEncryption)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))

7
cmd/utils.go
View File

@ -477,13 +477,16 @@ func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration) fu
// This sets the value for MaxIdleConnsPerHost from 2 (go default)
// to 256.
func NewGatewayHTTPTransport() *http.Transport {
return newGatewayHTTPTransport(1 * time.Minute)
}
func newGatewayHTTPTransport(timeout time.Duration) *http.Transport {
tr := newCustomHTTPTransport(&tls.Config{
RootCAs: globalRootCAs,
}, defaultDialTimeout)()
// Set aggressive timeouts for gateway
tr.ResponseHeaderTimeout = 1 * time.Minute
// Allow more requests to be in flight.
tr.ResponseHeaderTimeout = timeout
tr.MaxConnsPerHost = 256
tr.MaxIdleConnsPerHost = 16
tr.MaxIdleConns = 256

2
cmd/web-handlers.go
View File

@ -406,7 +406,7 @@ func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, r
}
return toJSONError(ctx, err, args.BucketName)
}
core, err := getRemoteInstanceClient(r, getHostFromSrv(sr))
core, err := getRemoteInstanceClientLongTimeout(r, getHostFromSrv(sr))
if err != nil {
return toJSONError(ctx, err, args.BucketName)
}