fix: handle concurrent lockers with multiple optimizations (#10640)

- select lockers which are non-local and online to have
  affinity towards remote servers for lock contention

- optimize lock retry interval to avoid sending too many
  messages during lock contention, reduces average CPU
  usage as well

- if bucket is not set, when deleteObject fails make sure
  setPutObjHeaders() honors lifecycle only if bucket name
  is set.

- fix top locks to list out always the oldest lockers always,
  avoid getting bogged down into map's unordered nature.

pkg/dsync/drwmutex.go

@@ -133,7 +133,7 @@ func (dm *DRWMutex) GetRLock(ctx context.Context, id, source string, opts Option
 }
 
 const (
-	lockRetryInterval = 100 * time.Millisecond
+	lockRetryInterval = 1 * time.Second
 )
 
 // lockBlocking will try to acquire either a read or a write lock
@@ -204,6 +204,7 @@ func (dm *DRWMutex) lockBlocking(ctx context.Context, id, source string, isReadL
 				dm.m.Unlock()
 				return locked
 			}
+
 			time.Sleep(time.Duration(r.Float64() * float64(lockRetryInterval)))
 		}
 	}
@@ -268,6 +269,7 @@ func lock(ctx context.Context, ds *Dsync, locks *[]string, id, source string, is
 
 	wg.Add(1)
 	go func(isReadLock bool) {
+		defer wg.Done()
 
 		// Wait until we have either
 		//
@@ -317,9 +319,6 @@ func lock(ctx context.Context, ds *Dsync, locks *[]string, id, source string, is
 		// Count locks in order to determine whether we have quorum or not
 		quorumLocked = checkQuorumLocked(locks, quorum)
 
-		// Signal that we have the quorum
-		wg.Done()
-
 		// Wait for the other responses and immediately release the locks
 		// (do not add them to the locks array because the DRWMutex could
 		//  already has been unlocked again by the original calling thread)

pkg/dsync/rpc-client-impl_test.go

@@ -50,6 +50,10 @@ func (rpcClient *ReconnectRPCClient) IsOnline() bool {
 	return rpcClient.rpc != nil
 }
 
+func (rpcClient *ReconnectRPCClient) IsLocal() bool {
+	return false
+}
+
 // Close closes the underlying socket file descriptor.
 func (rpcClient *ReconnectRPCClient) Close() error {
 	rpcClient.mutex.Lock()

pkg/dsync/rpc-client-interface.go

@@ -68,4 +68,7 @@ type NetLocker interface {
 
 	// Is the underlying connection online? (is always true for any local lockers)
 	IsOnline() bool
+
+	// Is the underlying locker local to this server?
+	IsLocal() bool
 }