Add support for timeouts for locks (#4377)

cmd/namespace-lock.go

@@ -21,7 +21,10 @@ import (
 	pathutil "path"
 	"sync"
 
+	"fmt"
 	"github.com/minio/dsync"
+	"github.com/minio/lsync"
+	"time"
 )
 
 // Global name space lock.
@@ -30,11 +33,19 @@ var globalNSMutex *nsLockMap
 // Global lock servers
 var globalLockServers []*lockServer
 
-// RWLocker - locker interface extends sync.Locker
-// to introduce RLock, RUnlock.
+// RWLocker - locker interface to introduce GetRLock, RUnlock.
 type RWLocker interface {
-	sync.Locker
-	RLock()
+	GetLock(timeout *dynamicTimeout) (timedOutErr error)
+	Unlock()
+	GetRLock(timeout *dynamicTimeout) (timedOutErr error)
+	RUnlock()
+}
+
+// RWLockerSync - internal locker interface.
+type RWLockerSync interface {
+	GetLock(timeout time.Duration) bool
+	Unlock()
+	GetRLock(timeout time.Duration) bool
 	RUnlock()
 }
 
@@ -101,7 +112,7 @@ type nsParam struct {
 
 // nsLock - provides primitives for locking critical namespace regions.
 type nsLock struct {
-	RWLocker
+	RWLockerSync
 	ref uint
 }
 
@@ -119,7 +130,7 @@ type nsLockMap struct {
 }
 
 // Lock the namespace resource.
-func (n *nsLockMap) lock(volume, path string, lockSource, opsID string, readLock bool) {
+func (n *nsLockMap) lock(volume, path string, lockSource, opsID string, readLock bool, timeout time.Duration) (locked bool) {
 	var nsLk *nsLock
 	n.lockMapMutex.Lock()
 
@@ -127,11 +138,11 @@ func (n *nsLockMap) lock(volume, path string, lockSource, opsID string, readLock
 	nsLk, found := n.lockMap[param]
 	if !found {
 		nsLk = &nsLock{
-			RWLocker: func() RWLocker {
+			RWLockerSync: func() RWLockerSync {
 				if n.isDistXL {
 					return dsync.NewDRWMutex(pathJoin(volume, path))
 				}
-				return &sync.RWMutex{}
+				return &lsync.LRWMutex{}
 			}(),
 			ref: 0,
 		}
@@ -144,17 +155,45 @@ func (n *nsLockMap) lock(volume, path string, lockSource, opsID string, readLock
 	// unblocks. The lock for accessing `globalNSMutex` is held inside
 	// the function itself.
 	if err := n.statusNoneToBlocked(param, lockSource, opsID, readLock); err != nil {
-		errorIf(err, "Failed to set lock state to blocked")
+		errorIf(err, fmt.Sprintf("Failed to set lock state to blocked (param = %v; opsID = %s)", param, opsID))
 	}
 
 	// Unlock map before Locking NS which might block.
 	n.lockMapMutex.Unlock()
 
-	// Locking here can block.
+	// Locking here will block (until timeout).
 	if readLock {
-		nsLk.RLock()
+		locked = nsLk.GetRLock(timeout)
 	} else {
-		nsLk.Lock()
+		locked = nsLk.GetLock(timeout)
+	}
+
+	if !locked { // We failed to get the lock
+		n.lockMapMutex.Lock()
+		defer n.lockMapMutex.Unlock()
+		// Changing the status of the operation from blocked to none
+		if err := n.statusBlockedToNone(param, lockSource, opsID, readLock); err != nil {
+			errorIf(err, fmt.Sprintf("Failed to clear the lock state (param = %v; opsID = %s)", param, opsID))
+		}
+
+		nsLk.ref-- // Decrement ref count since we failed to get the lock
+		// delete the lock state entry for given operation ID.
+		err := n.deleteLockInfoEntryForOps(param, opsID)
+		if err != nil {
+			errorIf(err, fmt.Sprintf("Failed to delete lock info entry (param = %v; opsID = %s)", param, opsID))
+		}
+		if nsLk.ref == 0 {
+			// Remove from the map if there are no more references.
+			delete(n.lockMap, param)
+
+			// delete the lock state entry for given
+			// <volume, path> pair.
+			err := n.deleteLockInfoEntryForVolumePath(param)
+			if err != nil {
+				errorIf(err, fmt.Sprintf("Failed to delete lock info entry (param = %v)", param))
+			}
+		}
+		return
 	}
 
 	// Changing the status of the operation from blocked to
@@ -163,6 +202,7 @@ func (n *nsLockMap) lock(volume, path string, lockSource, opsID string, readLock
 	if err := n.statusBlockedToRunning(param, lockSource, opsID, readLock); err != nil {
 		errorIf(err, "Failed to set the lock state to running")
 	}
+	return
 }
 
 // Unlock the namespace resource.
@@ -208,11 +248,11 @@ func (n *nsLockMap) unlock(volume, path, opsID string, readLock bool) {
 
 // Lock - locks the given resource for writes, using a previously
 // allocated name space lock or initializing a new one.
-func (n *nsLockMap) Lock(volume, path, opsID string) {
+func (n *nsLockMap) Lock(volume, path, opsID string, timeout time.Duration) (locked bool) {
 	readLock := false // This is a write lock.
 
 	lockSource := getSource() // Useful for debugging
-	n.lock(volume, path, lockSource, opsID, readLock)
+	return n.lock(volume, path, lockSource, opsID, readLock, timeout)
 }
 
 // Unlock - unlocks any previously acquired write locks.
@@ -222,11 +262,11 @@ func (n *nsLockMap) Unlock(volume, path, opsID string) {
 }
 
 // RLock - locks any previously acquired read locks.
-func (n *nsLockMap) RLock(volume, path, opsID string) {
+func (n *nsLockMap) RLock(volume, path, opsID string, timeout time.Duration) (locked bool) {
 	readLock := true
 
 	lockSource := getSource() // Useful for debugging
-	n.lock(volume, path, lockSource, opsID, readLock)
+	return n.lock(volume, path, lockSource, opsID, readLock, timeout)
 }
 
 // RUnlock - unlocks any previously acquired read locks.
@@ -282,11 +322,17 @@ func (n *nsLockMap) NewNSLock(volume, path string) RWLocker {
 	return &lockInstance{n, volume, path, getOpsID()}
 }
 
-// Lock - block until write lock is taken.
-func (li *lockInstance) Lock() {
+// Lock - block until write lock is taken or timeout has occurred.
+func (li *lockInstance) GetLock(timeout *dynamicTimeout) (timedOutErr error) {
 	lockSource := getSource()
+	start := UTCNow()
 	readLock := false
-	li.ns.lock(li.volume, li.path, lockSource, li.opsID, readLock)
+	if !li.ns.lock(li.volume, li.path, lockSource, li.opsID, readLock, timeout.Timeout()) {
+		timeout.LogFailure()
+		return OperationTimedOut{Path: li.path}
+	}
+	timeout.LogSuccess(UTCNow().Sub(start))
+	return
 }
 
 // Unlock - block until write lock is released.
@@ -295,11 +341,17 @@ func (li *lockInstance) Unlock() {
 	li.ns.unlock(li.volume, li.path, li.opsID, readLock)
 }
 
-// RLock - block until read lock is taken.
-func (li *lockInstance) RLock() {
+// RLock - block until read lock is taken or timeout has occurred.
+func (li *lockInstance) GetRLock(timeout *dynamicTimeout) (timedOutErr error) {
 	lockSource := getSource()
+	start := UTCNow()
 	readLock := true
-	li.ns.lock(li.volume, li.path, lockSource, li.opsID, readLock)
+	if !li.ns.lock(li.volume, li.path, lockSource, li.opsID, readLock, timeout.Timeout()) {
+		timeout.LogFailure()
+		return OperationTimedOut{Path: li.path}
+	}
+	timeout.LogSuccess(UTCNow().Sub(start))
+	return
 }
 
 // RUnlock - block until read lock is released.