MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-04-05 04:10:28 -04:00
Code Issues Packages Projects Releases Wiki Activity

use single dynamic timeout for most locked API/heal ops (#10275)

Browse Source 
newDynamicTimeout should be allocated once, in-case
of temporary locks in config and IAM we should
have allocated timeout once before the `for loop`

This PR doesn't fix any issue as such, but provides
enough dynamism for the timeout as per expectation.
This commit is contained in:
Harshavardhana 2020-08-17 11:29:58 -07:00 committed by GitHub
parent bb5976d727
commit e57c742674
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 32 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/admin-handlers.go
View File

@ -1270,10 +1270,9 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
} }
deadlinedCtx, cancel := context.WithTimeout(ctx, deadline) deadlinedCtx, cancel := context.WithTimeout(ctx, deadline)
defer cancel() defer cancel()
nsLock := objectAPI.NewNSLock(deadlinedCtx, minioMetaBucket, "obd-in-progress") nsLock := objectAPI.NewNSLock(ctx, minioMetaBucket, "obd-in-progress")
if err := nsLock.GetLock(newDynamicTimeout(deadline, deadline)); err != nil { // returns a locked lock if err := nsLock.GetLock(newDynamicTimeout(deadline, deadline)); err != nil { // returns a locked lock
errResp(err) errResp(err)
return return

10
cmd/disk-cache-backend.go
View File

@ -417,7 +417,7 @@ func (c *diskCache) Stat(ctx context.Context, bucket, object string) (oi ObjectI
func (c *diskCache) statCachedMeta(ctx context.Context, cacheObjPath string) (meta *cacheMeta, partial bool, numHits int, err error) { func (c *diskCache) statCachedMeta(ctx context.Context, cacheObjPath string) (meta *cacheMeta, partial bool, numHits int, err error) {
cLock := c.NewNSLockFn(ctx, cacheObjPath) cLock := c.NewNSLockFn(ctx, cacheObjPath)
if err = cLock.GetRLock(globalObjectTimeout); err != nil { if err = cLock.GetRLock(globalOperationTimeout); err != nil {
return return
} }
@ -499,7 +499,7 @@ func (c *diskCache) statCache(ctx context.Context, cacheObjPath string) (meta *c
func (c *diskCache) SaveMetadata(ctx context.Context, bucket, object string, meta map[string]string, actualSize int64, rs *HTTPRangeSpec, rsFileName string, incHitsOnly bool) error { func (c *diskCache) SaveMetadata(ctx context.Context, bucket, object string, meta map[string]string, actualSize int64, rs *HTTPRangeSpec, rsFileName string, incHitsOnly bool) error {
cachedPath := getCacheSHADir(c.dir, bucket, object) cachedPath := getCacheSHADir(c.dir, bucket, object)
cLock := c.NewNSLockFn(ctx, cachedPath) cLock := c.NewNSLockFn(ctx, cachedPath)
if err := cLock.GetLock(globalObjectTimeout); err != nil { if err := cLock.GetLock(globalOperationTimeout); err != nil {
return err return err
} }
defer cLock.Unlock() defer cLock.Unlock()
@ -665,7 +665,7 @@ func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Read
} }
cachePath := getCacheSHADir(c.dir, bucket, object) cachePath := getCacheSHADir(c.dir, bucket, object)
cLock := c.NewNSLockFn(ctx, cachePath) cLock := c.NewNSLockFn(ctx, cachePath)
if err := cLock.GetLock(globalObjectTimeout); err != nil { if err := cLock.GetLock(globalOperationTimeout); err != nil {
return err return err
} }
defer cLock.Unlock() defer cLock.Unlock()
@ -871,7 +871,7 @@ func (c *diskCache) bitrotReadFromCache(ctx context.Context, filePath string, of
func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, numHits int, err error) { func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, numHits int, err error) {
cacheObjPath := getCacheSHADir(c.dir, bucket, object) cacheObjPath := getCacheSHADir(c.dir, bucket, object)
cLock := c.NewNSLockFn(ctx, cacheObjPath) cLock := c.NewNSLockFn(ctx, cacheObjPath)
if err := cLock.GetRLock(globalObjectTimeout); err != nil { if err := cLock.GetRLock(globalOperationTimeout); err != nil {
return nil, numHits, err return nil, numHits, err
} }
@ -935,7 +935,7 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
// Deletes the cached object // Deletes the cached object
func (c *diskCache) delete(ctx context.Context, cacheObjPath string) (err error) { func (c *diskCache) delete(ctx context.Context, cacheObjPath string) (err error) {
cLock := c.NewNSLockFn(ctx, cacheObjPath) cLock := c.NewNSLockFn(ctx, cacheObjPath)
if err := cLock.GetLock(globalObjectTimeout); err != nil { if err := cLock.GetLock(globalOperationTimeout); err != nil {
return err return err
} }
defer cLock.Unlock() defer cLock.Unlock()

20
cmd/erasure-zones.go
View File

@ -458,12 +458,12 @@ func (z *erasureZones) GetObjectNInfo(ctx context.Context, bucket, object string
lock := z.NewNSLock(ctx, bucket, object) lock := z.NewNSLock(ctx, bucket, object)
switch lockType { switch lockType {
case writeLock: case writeLock:
if err = lock.GetLock(globalObjectTimeout); err != nil { if err = lock.GetLock(globalOperationTimeout); err != nil {
return nil, err return nil, err
} }
nsUnlocker = lock.Unlock nsUnlocker = lock.Unlock
case readLock: case readLock:
if err = lock.GetRLock(globalObjectTimeout); err != nil { if err = lock.GetRLock(globalOperationTimeout); err != nil {
return nil, err return nil, err
} }
nsUnlocker = lock.RUnlock nsUnlocker = lock.RUnlock
@ -492,7 +492,7 @@ func (z *erasureZones) GetObjectNInfo(ctx context.Context, bucket, object string
func (z *erasureZones) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error { func (z *erasureZones) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error {
// Lock the object before reading. // Lock the object before reading.
lk := z.NewNSLock(ctx, bucket, object) lk := z.NewNSLock(ctx, bucket, object)
if err := lk.GetRLock(globalObjectTimeout); err != nil { if err := lk.GetRLock(globalOperationTimeout); err != nil {
return err return err
} }
defer lk.RUnlock() defer lk.RUnlock()
@ -516,7 +516,7 @@ func (z *erasureZones) GetObject(ctx context.Context, bucket, object string, sta
func (z *erasureZones) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) { func (z *erasureZones) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
// Lock the object before reading. // Lock the object before reading.
lk := z.NewNSLock(ctx, bucket, object) lk := z.NewNSLock(ctx, bucket, object)
if err := lk.GetRLock(globalObjectTimeout); err != nil { if err := lk.GetRLock(globalOperationTimeout); err != nil {
return ObjectInfo{}, err return ObjectInfo{}, err
} }
defer lk.RUnlock() defer lk.RUnlock()
@ -544,7 +544,7 @@ func (z *erasureZones) GetObjectInfo(ctx context.Context, bucket, object string,
func (z *erasureZones) PutObject(ctx context.Context, bucket string, object string, data *PutObjReader, opts ObjectOptions) (ObjectInfo, error) { func (z *erasureZones) PutObject(ctx context.Context, bucket string, object string, data *PutObjReader, opts ObjectOptions) (ObjectInfo, error) {
// Lock the object. // Lock the object.
lk := z.NewNSLock(ctx, bucket, object) lk := z.NewNSLock(ctx, bucket, object)
if err := lk.GetLock(globalObjectTimeout); err != nil { if err := lk.GetLock(globalOperationTimeout); err != nil {
return ObjectInfo{}, err return ObjectInfo{}, err
} }
defer lk.Unlock() defer lk.Unlock()
@ -625,7 +625,7 @@ func (z *erasureZones) CopyObject(ctx context.Context, srcBucket, srcObject, dst
cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject)) cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
if !cpSrcDstSame { if !cpSrcDstSame {
lk := z.NewNSLock(ctx, dstBucket, dstObject) lk := z.NewNSLock(ctx, dstBucket, dstObject)
if err := lk.GetLock(globalObjectTimeout); err != nil { if err := lk.GetLock(globalOperationTimeout); err != nil {
return objInfo, err return objInfo, err
} }
defer lk.Unlock() defer lk.Unlock()
@ -1732,7 +1732,7 @@ func (z *erasureZones) ListBuckets(ctx context.Context) (buckets []BucketInfo, e
func (z *erasureZones) ReloadFormat(ctx context.Context, dryRun bool) error { func (z *erasureZones) ReloadFormat(ctx context.Context, dryRun bool) error {
// Acquire lock on format.json // Acquire lock on format.json
formatLock := z.NewNSLock(ctx, minioMetaBucket, formatConfigFile) formatLock := z.NewNSLock(ctx, minioMetaBucket, formatConfigFile)
if err := formatLock.GetRLock(globalHealingTimeout); err != nil { if err := formatLock.GetRLock(globalOperationTimeout); err != nil {
return err return err
} }
defer formatLock.RUnlock() defer formatLock.RUnlock()
@ -1748,7 +1748,7 @@ func (z *erasureZones) ReloadFormat(ctx context.Context, dryRun bool) error {
func (z *erasureZones) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) { func (z *erasureZones) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) {
// Acquire lock on format.json // Acquire lock on format.json
formatLock := z.NewNSLock(ctx, minioMetaBucket, formatConfigFile) formatLock := z.NewNSLock(ctx, minioMetaBucket, formatConfigFile)
if err := formatLock.GetLock(globalHealingTimeout); err != nil { if err := formatLock.GetLock(globalOperationTimeout); err != nil {
return madmin.HealResultItem{}, err return madmin.HealResultItem{}, err
} }
defer formatLock.Unlock() defer formatLock.Unlock()
@ -1951,14 +1951,14 @@ func (z *erasureZones) HealObject(ctx context.Context, bucket, object, versionID
lk := z.NewNSLock(ctx, bucket, object) lk := z.NewNSLock(ctx, bucket, object)
if bucket == minioMetaBucket { if bucket == minioMetaBucket {
// For .minio.sys bucket heals we should hold write locks. // For .minio.sys bucket heals we should hold write locks.
if err := lk.GetLock(globalHealingTimeout); err != nil { if err := lk.GetLock(globalOperationTimeout); err != nil {
return madmin.HealResultItem{}, err return madmin.HealResultItem{}, err
} }
defer lk.Unlock() defer lk.Unlock()
} else { } else {
// Lock the object before healing. Use read lock since healing // Lock the object before healing. Use read lock since healing
// will only regenerate parts & xl.meta of outdated disks. // will only regenerate parts & xl.meta of outdated disks.
if err := lk.GetRLock(globalHealingTimeout); err != nil { if err := lk.GetRLock(globalOperationTimeout); err != nil {
return madmin.HealResultItem{}, err return madmin.HealResultItem{}, err
} }
defer lk.RUnlock() defer lk.RUnlock()

2
cmd/fs-v1-multipart.go
View File

@ -707,7 +707,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
// Hold write lock on the object. // Hold write lock on the object.
destLock := fs.NewNSLock(ctx, bucket, object) destLock := fs.NewNSLock(ctx, bucket, object)
if err = destLock.GetLock(globalObjectTimeout); err != nil { if err = destLock.GetLock(globalOperationTimeout); err != nil {
return oi, err return oi, err
} }
defer destLock.Unlock() defer destLock.Unlock()

14
cmd/fs-v1.go
View File

@ -601,7 +601,7 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
if !cpSrcDstSame { if !cpSrcDstSame {
objectDWLock := fs.NewNSLock(ctx, dstBucket, dstObject) objectDWLock := fs.NewNSLock(ctx, dstBucket, dstObject)
if err := objectDWLock.GetLock(globalObjectTimeout); err != nil { if err := objectDWLock.GetLock(globalOperationTimeout); err != nil {
return oi, err return oi, err
} }
defer objectDWLock.Unlock() defer objectDWLock.Unlock()
@ -691,12 +691,12 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
lock := fs.NewNSLock(ctx, bucket, object) lock := fs.NewNSLock(ctx, bucket, object)
switch lockType { switch lockType {
case writeLock: case writeLock:
if err = lock.GetLock(globalObjectTimeout); err != nil { if err = lock.GetLock(globalOperationTimeout); err != nil {
return nil, err return nil, err
} }
nsUnlocker = lock.Unlock nsUnlocker = lock.Unlock
case readLock: case readLock:
if err = lock.GetRLock(globalObjectTimeout); err != nil { if err = lock.GetRLock(globalOperationTimeout); err != nil {
return nil, err return nil, err
} }
nsUnlocker = lock.RUnlock nsUnlocker = lock.RUnlock
@ -782,7 +782,7 @@ func (fs *FSObjects) GetObject(ctx context.Context, bucket, object string, offse
// Lock the object before reading. // Lock the object before reading.
lk := fs.NewNSLock(ctx, bucket, object) lk := fs.NewNSLock(ctx, bucket, object)
if err := lk.GetRLock(globalObjectTimeout); err != nil { if err := lk.GetRLock(globalOperationTimeout); err != nil {
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
return err return err
} }
@ -1006,7 +1006,7 @@ func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (
func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) { func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) {
// Lock the object before reading. // Lock the object before reading.
lk := fs.NewNSLock(ctx, bucket, object) lk := fs.NewNSLock(ctx, bucket, object)
if err := lk.GetRLock(globalObjectTimeout); err != nil { if err := lk.GetRLock(globalOperationTimeout); err != nil {
return oi, err return oi, err
} }
defer lk.RUnlock() defer lk.RUnlock()
@ -1044,7 +1044,7 @@ func (fs *FSObjects) GetObjectInfo(ctx context.Context, bucket, object string, o
oi, err := fs.getObjectInfoWithLock(ctx, bucket, object) oi, err := fs.getObjectInfoWithLock(ctx, bucket, object)
if err == errCorruptedFormat || err == io.EOF { if err == errCorruptedFormat || err == io.EOF {
lk := fs.NewNSLock(ctx, bucket, object) lk := fs.NewNSLock(ctx, bucket, object)
if err = lk.GetLock(globalObjectTimeout); err != nil { if err = lk.GetLock(globalOperationTimeout); err != nil {
return oi, toObjectErr(err, bucket, object) return oi, toObjectErr(err, bucket, object)
} }
@ -1095,7 +1095,7 @@ func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string
// Lock the object. // Lock the object.
lk := fs.NewNSLock(ctx, bucket, object) lk := fs.NewNSLock(ctx, bucket, object)
if err := lk.GetLock(globalObjectTimeout); err != nil { if err := lk.GetLock(globalOperationTimeout); err != nil {
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
return objInfo, err return objInfo, err
} }

4
cmd/globals.go
View File

@ -208,9 +208,7 @@ var (
globalDomainNames []string // Root domains for virtual host style requests globalDomainNames []string // Root domains for virtual host style requests
globalDomainIPs set.StringSet // Root domain IP address(s) for a distributed MinIO deployment globalDomainIPs set.StringSet // Root domain IP address(s) for a distributed MinIO deployment
globalObjectTimeout = newDynamicTimeout( /*1*/ 10*time.Minute /*10*/, 600*time.Second) // timeout for Object API related ops globalOperationTimeout = newDynamicTimeout(10*time.Minute, 5*time.Minute) // default timeout for general ops
globalOperationTimeout = newDynamicTimeout(10*time.Minute /*30*/, 600*time.Second) // default timeout for general ops
globalHealingTimeout = newDynamicTimeout(30*time.Minute /*1*/, 30*time.Minute) // timeout for healing related ops
globalBucketObjectLockSys *BucketObjectLockSys globalBucketObjectLockSys *BucketObjectLockSys
globalBucketQuotaSys *BucketQuotaSys globalBucketQuotaSys *BucketQuotaSys

5
cmd/iam.go
View File

@ -449,10 +449,13 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
rquorum := InsufficientReadQuorum{} rquorum := InsufficientReadQuorum{}
wquorum := InsufficientWriteQuorum{} wquorum := InsufficientWriteQuorum{}
// allocate dynamic timeout once before the loop
iamLockTimeout := newDynamicTimeout(3*time.Second, 5*time.Second)
for range retry.NewTimerWithJitter(retryCtx, time.Second, 5*time.Second, retry.MaxJitter) { for range retry.NewTimerWithJitter(retryCtx, time.Second, 5*time.Second, retry.MaxJitter) {
// let one of the server acquire the lock, if not let them timeout. // let one of the server acquire the lock, if not let them timeout.
// which shall be retried again by this loop. // which shall be retried again by this loop.
if err := txnLk.GetLock(newDynamicTimeout(3*time.Second, 5*time.Second)); err != nil { if err := txnLk.GetLock(iamLockTimeout); err != nil {
logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. trying to acquire lock") logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. trying to acquire lock")
continue continue
} }

5
cmd/server-main.go
View File

@ -229,7 +229,8 @@ func initSafeMode(ctx context.Context, newObject ObjectLayer) (err error) {
initAutoHeal(ctx, newObject) initAutoHeal(ctx, newObject)
} }
timeout := newDynamicTimeout(3*time.Second, 3*time.Second) // allocate dynamic timeout once before the loop
configLockTimeout := newDynamicTimeout(3*time.Second, 5*time.Second)
// **** WARNING **** // **** WARNING ****
// Migrating to encrypted backend should happen before initialization of any // Migrating to encrypted backend should happen before initialization of any
@ -246,7 +247,7 @@ func initSafeMode(ctx context.Context, newObject ObjectLayer) (err error) {
for range retry.NewTimer(retryCtx) { for range retry.NewTimer(retryCtx) {
// let one of the server acquire the lock, if not let them timeout. // let one of the server acquire the lock, if not let them timeout.
// which shall be retried again by this loop. // which shall be retried again by this loop.
if err = txnLk.GetLock(timeout); err != nil { if err = txnLk.GetLock(configLockTimeout); err != nil {
logger.Info("Waiting for all MinIO sub-systems to be initialized.. trying to acquire lock") logger.Info("Waiting for all MinIO sub-systems to be initialized.. trying to acquire lock")
continue continue
} }