locking: Add Refresh for better locking cleanup (#11535)

Co-authored-by: Anis Elleuch <anis@min.io>
Co-authored-by: Harshavardhana <harsha@minio.io>

cmd/local-locker.go

@@ -27,13 +27,13 @@ import (
 
 // lockRequesterInfo stores various info from the client for each lock that is requested.
 type lockRequesterInfo struct {
-	Name          string    // name of the resource lock was requested for
-	Writer        bool      // Bool whether write or read lock.
-	UID           string    // UID to uniquely identify request of client.
-	Timestamp     time.Time // Timestamp set at the time of initialization.
-	TimeLastCheck time.Time // Timestamp for last check of validity of lock.
-	Source        string    // Contains line, function and filename reqesting the lock.
-	Group         bool      // indicates if it was a group lock.
+	Name            string    // name of the resource lock was requested for
+	Writer          bool      // Bool whether write or read lock.
+	UID             string    // UID to uniquely identify request of client.
+	Timestamp       time.Time // Timestamp set at the time of initialization.
+	TimeLastRefresh time.Time // Timestamp for last lock refresh.
+	Source          string    // Contains line, function and filename reqesting the lock.
+	Group           bool      // indicates if it was a group lock.
 	// Owner represents the UUID of the owner who originally requested the lock
 	// useful in expiry.
 	Owner string
@@ -93,15 +93,15 @@ func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool
 	for _, resource := range args.Resources {
 		l.lockMap[resource] = []lockRequesterInfo{
 			{
-				Name:          resource,
-				Writer:        true,
-				Source:        args.Source,
-				Owner:         args.Owner,
-				UID:           args.UID,
-				Timestamp:     UTCNow(),
-				TimeLastCheck: UTCNow(),
-				Group:         len(args.Resources) > 1,
-				Quorum:        args.Quorum,
+				Name:            resource,
+				Writer:          true,
+				Source:          args.Source,
+				Owner:           args.Owner,
+				UID:             args.UID,
+				Timestamp:       UTCNow(),
+				TimeLastRefresh: UTCNow(),
+				Group:           len(args.Resources) > 1,
+				Quorum:          args.Quorum,
 			},
 		}
 	}
@@ -154,14 +154,14 @@ func (l *localLocker) RLock(ctx context.Context, args dsync.LockArgs) (reply boo
 	defer l.mutex.Unlock()
 	resource := args.Resources[0]
 	lrInfo := lockRequesterInfo{
-		Name:          resource,
-		Writer:        false,
-		Source:        args.Source,
-		Owner:         args.Owner,
-		UID:           args.UID,
-		Timestamp:     UTCNow(),
-		TimeLastCheck: UTCNow(),
-		Quorum:        args.Quorum,
+		Name:            resource,
+		Writer:          false,
+		Source:          args.Source,
+		Owner:           args.Owner,
+		UID:             args.UID,
+		Timestamp:       UTCNow(),
+		TimeLastRefresh: UTCNow(),
+		Quorum:          args.Quorum,
 	}
 	if lri, ok := l.lockMap[resource]; ok {
 		if reply = !isWriteLock(lri); reply {
@@ -236,7 +236,7 @@ func (l *localLocker) ForceUnlock(ctx context.Context, args dsync.LockArgs) (rep
 	}
 }
 
-func (l *localLocker) Expired(ctx context.Context, args dsync.LockArgs) (expired bool, err error) {
+func (l *localLocker) Refresh(ctx context.Context, args dsync.LockArgs) (refreshed bool, err error) {
 	select {
 	case <-ctx.Done():
 		return false, ctx.Err()
@@ -244,45 +244,39 @@ func (l *localLocker) Expired(ctx context.Context, args dsync.LockArgs) (expired
 		l.mutex.Lock()
 		defer l.mutex.Unlock()
 
-		resource := args.Resources[0] // expiry check is always per resource.
+		resource := args.Resources[0] // refresh check is always per resource.
 
 		// Lock found, proceed to verify if belongs to given uid.
 		lri, ok := l.lockMap[resource]
 		if !ok {
-			// lock doesn't exist yet not reason to
-			// expire that doesn't exist yet - it may be
-			// racing with other active lock requests.
+			// lock doesn't exist yet, return false
 			return false, nil
 		}
 
 		// Check whether uid is still active
-		for _, entry := range lri {
-			if entry.UID == args.UID && entry.Owner == args.Owner {
-				ep := globalRemoteEndpoints[args.Owner]
-				if !ep.IsLocal {
-					// check if the owner is online
-					return isServerResolvable(ep, 3*time.Second) != nil, nil
-				}
-				return false, nil
+		for i := range lri {
+			if lri[i].UID == args.UID && lri[i].Owner == args.Owner {
+				lri[i].TimeLastRefresh = UTCNow()
+				return true, nil
 			}
 		}
 
-		return true, nil
+		return false, nil
 	}
 }
 
 // Similar to removeEntry but only removes an entry only if the lock entry exists in map.
 // Caller must hold 'l.mutex' lock.
-func (l *localLocker) removeEntryIfExists(lrip lockRequesterInfo) {
+func (l *localLocker) expireOldLocks(interval time.Duration) {
 	l.mutex.Lock()
 	defer l.mutex.Unlock()
 
-	// Check if entry is still in map (could have been removed altogether by 'concurrent' (R)Unlock of last entry)
-	if lri, ok := l.lockMap[lrip.Name]; ok {
-		// Even if the entry exists, it may not be the same entry which was
-		// considered as expired, so we simply an attempt to remove it if its
-		// not possible there is nothing we need to do.
-		l.removeEntry(lrip.Name, dsync.LockArgs{Owner: lrip.Owner, UID: lrip.UID}, &lri)
+	for _, lris := range l.lockMap {
+		for _, lri := range lris {
+			if time.Since(lri.TimeLastRefresh) > interval {
+				l.removeEntry(lri.Name, dsync.LockArgs{Owner: lri.Owner, UID: lri.UID}, &lris)
+			}
+		}
 	}
 }