fix: Pass context all the way down to the network call in lockers (#10161)

Context timeout might race on each other when timeouts are lower
i.e when two lock attempts happened very quickly on the same resource
and the servers were yet trying to establish quorum.

This situation can lead to locks held which wouldn't be unlocked
and subsequent lock attempts would fail.

This would require a complete server restart. A potential of this
issue happening is when server is booting up and we are trying
to hold a 'transaction.lock' in quick bursts of timeout.

pkg/dsync/drwmutex.go

@@ -140,8 +140,8 @@ func (dm *DRWMutex) lockBlocking(ctx context.Context, timeout time.Duration, id,
 		locks := make([]string, len(restClnts))
 
 		// Try to acquire the lock.
-		success := lock(dm.clnt, &locks, id, source, isReadLock, dm.Names...)
-		if !success {
+		locked = lock(retryCtx, dm.clnt, &locks, id, source, isReadLock, dm.Names...)
+		if !locked {
 			continue
 		}
 
@@ -158,16 +158,16 @@ func (dm *DRWMutex) lockBlocking(ctx context.Context, timeout time.Duration, id,
 		}
 
 		dm.m.Unlock()
-		return true
+		return locked
 	}
 
 	// Failed to acquire the lock on this attempt, incrementally wait
 	// for a longer back-off time and try again afterwards.
-	return false
+	return locked
 }
 
 // lock tries to acquire the distributed lock, returning true or false.
-func lock(ds *Dsync, locks *[]string, id, source string, isReadLock bool, lockNames ...string) bool {
+func lock(ctx context.Context, ds *Dsync, locks *[]string, id, source string, isReadLock bool, lockNames ...string) bool {
 
 	restClnts := ds.GetLockersFn()
 
@@ -199,11 +199,11 @@ func lock(ds *Dsync, locks *[]string, id, source string, isReadLock bool, lockNa
 			var locked bool
 			var err error
 			if isReadLock {
-				if locked, err = c.RLock(args); err != nil {
+				if locked, err = c.RLock(ctx, args); err != nil {
 					log("dsync: Unable to call RLock failed with %s for %#v at %s\n", err, args, c)
 				}
 			} else {
-				if locked, err = c.Lock(args); err != nil {
+				if locked, err = c.Lock(ctx, args); err != nil {
 					log("dsync: Unable to call Lock failed with %s for %#v at %s\n", err, args, c)
 				}
 			}

pkg/dsync/rpc-client-impl_test.go

@@ -17,6 +17,7 @@
 package dsync_test
 
 import (
+	"context"
 	"net/rpc"
 	"sync"
 
@@ -89,12 +90,12 @@ func (rpcClient *ReconnectRPCClient) Call(serviceMethod string, args interface{}
 	return err
 }
 
-func (rpcClient *ReconnectRPCClient) RLock(args LockArgs) (status bool, err error) {
+func (rpcClient *ReconnectRPCClient) RLock(ctx context.Context, args LockArgs) (status bool, err error) {
 	err = rpcClient.Call("Dsync.RLock", &args, &status)
 	return status, err
 }
 
-func (rpcClient *ReconnectRPCClient) Lock(args LockArgs) (status bool, err error) {
+func (rpcClient *ReconnectRPCClient) Lock(ctx context.Context, args LockArgs) (status bool, err error) {
 	err = rpcClient.Call("Dsync.Lock", &args, &status)
 	return status, err
 }
@@ -109,7 +110,7 @@ func (rpcClient *ReconnectRPCClient) Unlock(args LockArgs) (status bool, err err
 	return status, err
 }
 
-func (rpcClient *ReconnectRPCClient) Expired(args LockArgs) (expired bool, err error) {
+func (rpcClient *ReconnectRPCClient) Expired(ctx context.Context, args LockArgs) (expired bool, err error) {
 	err = rpcClient.Call("Dsync.Expired", &args, &expired)
 	return expired, err
 }

pkg/dsync/rpc-client-interface.go

@@ -16,6 +16,8 @@
 
 package dsync
 
+import "context"
+
 // LockArgs is minimal required values for any dsync compatible lock operation.
 type LockArgs struct {
 	// Unique ID of lock/unlock request.
@@ -34,12 +36,12 @@ type NetLocker interface {
 	// Do read lock for given LockArgs.  It should return
 	// * a boolean to indicate success/failure of the operation
 	// * an error on failure of lock request operation.
-	RLock(args LockArgs) (bool, error)
+	RLock(ctx context.Context, args LockArgs) (bool, error)
 
 	// Do write lock for given LockArgs. It should return
 	// * a boolean to indicate success/failure of the operation
 	// * an error on failure of lock request operation.
-	Lock(args LockArgs) (bool, error)
+	Lock(ctx context.Context, args LockArgs) (bool, error)
 
 	// Do read unlock for given LockArgs. It should return
 	// * a boolean to indicate success/failure of the operation
@@ -52,7 +54,7 @@ type NetLocker interface {
 	Unlock(args LockArgs) (bool, error)
 
 	// Expired returns if current lock args has expired.
-	Expired(args LockArgs) (bool, error)
+	Expired(ctx context.Context, args LockArgs) (bool, error)
 
 	// Returns underlying endpoint of this lock client instance.
 	String() string