Vendorize minio/dsync for server-side read lock (#2484)

- Prevention of stale lock accumulation.
- Removal of dead code.

vendor/github.com/minio/dsync/dmutex.go

@@ -1,319 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package dsync
-
-import (
-	"log"
-	"math"
-	"math/rand"
-	"net/rpc"
-	"sync"
-	"time"
-)
-
-const DMutexAcquireTimeout = 25 * time.Millisecond
-
-// A DMutex is a distributed mutual exclusion lock.
-type DMutex struct {
-	Name  string
-	locks []bool     // Array of nodes that granted a lock
-	uids  []string   // Array of uids for verification of sending correct release messages
-	m     sync.Mutex // Mutex to prevent multiple simultaneous locks from this node
-
-}
-
-type Granted struct {
-	index  int
-	locked bool
-	uid    string
-}
-
-// Connect to respective lock server nodes on the first Lock() call.
-func connectLazy(dm *DMutex) {
-	if clnts == nil {
-		panic("rpc client connections weren't initialized.")
-	}
-	for i := range clnts {
-		if clnts[i].rpc != nil {
-			continue
-		}
-
-		// Pass in unique path (as required by server.HandleHTTP().
-		// Ignore failure to connect, the lock server node may join the
-		// cluster later.
-		clnt, err := rpc.DialHTTPPath("tcp", nodes[i], rpcPaths[i])
-		if err != nil {
-			clnts[i].SetRPC(nil)
-			continue
-		}
-		clnts[i].SetRPC(clnt)
-	}
-}
-
-// Lock locks dm.
-//
-// If the lock is already in use, the calling goroutine
-// blocks until the mutex is available.
-func (dm *DMutex) Lock() {
-
-	// Shield Lock() with local mutex in order to prevent more than
-	// one broadcast going out at the same time from this node
-	dm.m.Lock()
-	defer dm.m.Unlock()
-
-	runs, backOff := 1, 1
-
-	for {
-		connectLazy(dm)
-
-		// create temp arrays on stack
-		locks := make([]bool, n)
-		ids := make([]string, n)
-
-		// try to acquire the lock
-		success := lock(clnts, &locks, &ids, dm.Name)
-		if success {
-			// if success, copy array to object
-			dm.locks = make([]bool, n)
-			copy(dm.locks, locks[:])
-			dm.uids = make([]string, n)
-			copy(dm.uids, ids[:])
-			return
-		}
-
-		// We timed out on the previous lock, incrementally wait for a longer back-off time,
-		// and try again afterwards
-		time.Sleep(time.Duration(backOff) * time.Millisecond)
-
-		backOff += int(rand.Float64() * math.Pow(2, float64(runs)))
-		if backOff > 1024 {
-			backOff = backOff % 64
-
-			runs = 1 // reset runs
-		} else if runs < 10 {
-			runs++
-		}
-	}
-}
-
-func (dm *DMutex) tryLockTimeout() bool {
-
-	// Shield Lock() with local mutex in order to prevent more than
-	// one broadcast going out at the same time from this node
-	dm.m.Lock()
-	defer dm.m.Unlock()
-
-	// TODO: Implement reconnect
-	connectLazy(dm)
-
-	// create temp arrays on stack
-	locks := make([]bool, n)
-	ids := make([]string, n)
-
-	// try to acquire the lock
-	success := lock(clnts, &locks, &ids, dm.Name)
-	if success {
-		// if success, copy array to object
-		dm.locks = make([]bool, n)
-		copy(dm.locks, locks[:])
-		dm.uids = make([]string, n)
-		copy(dm.uids, ids[:])
-	}
-	return success
-}
-
-// lock tries to acquire the distributed lock, returning true or false
-//
-func lock(clnts []*RPCClient, locks *[]bool, uids *[]string, lockName string) bool {
-
-	// Create buffered channel of quorum size
-	ch := make(chan Granted, n/2+1)
-
-	for index, c := range clnts {
-
-		// broadcast lock request to all nodes
-		go func(index int, c *RPCClient) {
-			// All client methods issuing RPCs are thread-safe and goroutine-safe,
-			// i.e. it is safe to call them from multiple concurrently running go routines.
-			var status bool
-			err := c.Call("Dsync.Lock", lockName, &status)
-
-			locked, uid := false, ""
-			if err == nil {
-				locked = status
-				// TODO: Get UIOD again
-				uid = ""
-			} else {
-				// If rpc call failed due to connection related errors, reset rpc.Client object
-				// to trigger reconnect on subsequent Lock()/Unlock() requests to the same node.
-				if IsRPCError(err) {
-					clnts[index].SetRPC(nil)
-				}
-				// silently ignore error, retry later
-			}
-
-			ch <- Granted{index: index, locked: locked, uid: uid}
-
-		}(index, c)
-	}
-
-	var wg sync.WaitGroup
-	wg.Add(1)
-
-	quorum := false
-
-	go func() {
-
-		// Wait until we have received (minimally) quorum number of responses or timeout
-		i := 0
-		done := false
-		timeout := time.After(DMutexAcquireTimeout)
-
-		for ; i < n; i++ {
-
-			select {
-			case grant := <-ch:
-				if grant.locked {
-					// Mark that this node has acquired the lock
-					(*locks)[grant.index] = true
-					(*uids)[grant.index] = grant.uid
-				} else {
-					done = true
-					//fmt.Println("one lock failed before quorum -- release locks acquired")
-					releaseAll(clnts, locks, uids, lockName)
-				}
-
-			case <-timeout:
-				done = true
-				// timeout happened, maybe one of the nodes is slow, count
-				// number of locks to check whether we have quorum or not
-				if !quorumMet(locks) {
-					//fmt.Println("timed out -- release locks acquired")
-					releaseAll(clnts, locks, uids, lockName)
-				}
-			}
-
-			if done {
-				break
-			}
-		}
-
-		// Count locks in order to determine whterh we have quorum or not
-		quorum = quorumMet(locks)
-
-		// Signal that we have the quorum
-		wg.Done()
-
-		// Wait for the other responses and immediately release the locks
-		// (do not add them to the locks array because the DMutex could
-		//  already has been unlocked again by the original calling thread)
-		for ; i < n; i++ {
-			grantToBeReleased := <-ch
-			if grantToBeReleased.locked {
-				// release lock
-				go sendRelease(clnts[grantToBeReleased.index], lockName, grantToBeReleased.uid)
-			}
-		}
-	}()
-
-	wg.Wait()
-
-	return quorum
-}
-
-// quorumMet determines whether we have acquired n/2+1 underlying locks or not
-func quorumMet(locks *[]bool) bool {
-
-	count := 0
-	for _, locked := range *locks {
-		if locked {
-			count++
-		}
-	}
-
-	return count >= n/2+1
-}
-
-// releaseAll releases all locks that are marked as locked
-func releaseAll(clnts []*RPCClient, locks *[]bool, ids *[]string, lockName string) {
-
-	for lock := 0; lock < n; lock++ {
-		if (*locks)[lock] {
-			go sendRelease(clnts[lock], lockName, (*ids)[lock])
-			(*locks)[lock] = false
-			(*ids)[lock] = ""
-		}
-	}
-
-}
-
-// hasLock returns whether or not a node participated in granting the lock
-func (dm *DMutex) hasLock(node string) bool {
-
-	for index, n := range nodes {
-		if n == node {
-			return dm.locks[index]
-		}
-	}
-
-	return false
-}
-
-// locked returns whether or not we have met the quorum
-func (dm *DMutex) locked() bool {
-
-	locks := make([]bool, n)
-	copy(locks[:], dm.locks[:])
-
-	return quorumMet(&locks)
-}
-
-// Unlock unlocks dm.
-//
-// It is a run-time error if dm is not locked on entry to Unlock.
-func (dm *DMutex) Unlock() {
-
-	// Verify that we have the lock or panic otherwise (similar to sync.mutex)
-	if !dm.locked() {
-		panic("dsync: unlock of unlocked distributed mutex")
-	}
-
-	// We don't need to wait until we have released all the locks (or the quorum)
-	// (a subsequent lock will retry automatically in case it would fail to get
-	//  quorum)
-	for index, c := range clnts {
-
-		if dm.locks[index] {
-			// broadcast lock release to all nodes the granted the lock
-			go sendRelease(c, dm.Name, dm.uids[index])
-
-			dm.locks[index] = false
-		}
-	}
-}
-
-// sendRelease sends a release message to a node that previously granted a lock
-func sendRelease(c *RPCClient, name, uid string) {
-
-	// All client methods issuing RPCs are thread-safe and goroutine-safe,
-	// i.e. it is safe to call them from multiple concurrently running goroutines.
-	var status bool
-	// TODO: Send UID to server
-	if err := c.Call("Dsync.Unlock", name, &status); err != nil {
-		log.Fatal("Unlock on %s failed on client %v", name, c)
-	}
-}

vendor/github.com/minio/dsync/drwmutex.go

@@ -17,127 +17,360 @@
 package dsync
 
 import (
-	"fmt"
+	"math"
+	"math/rand"
+	"net/rpc"
 	"sync"
+	"time"
 )
 
-const maxReaders = 8
+const DRWMutexAcquireTimeout = 25 * time.Millisecond
 
+// A DRWMutex is a distributed mutual exclusion lock.
 type DRWMutex struct {
-	rArray       []*DMutex
+	Name  string
-	rLockedArray []bool
+	locks []bool     // Array of nodes that granted a lock
-	w            DMutex     // held if there are pending writers
+	uids  []string   // Array of uids for verification of sending correct release messages
-	m            sync.Mutex // Mutex to prevent multiple simultaneous locks from this node
+	m     sync.Mutex // Mutex to prevent multiple simultaneous locks from this node
-	m2           sync.Mutex // Mutex to prevent multiple simultaneous locks from this node
 }
 
-func NewDRWMutex(name string) (drw *DRWMutex) {
+type Granted struct {
-
+	index  int
-	rArray := make([]*DMutex, maxReaders)
+	locked bool
-	rLockedArray := make([]bool, maxReaders)
+	uid    string
-
+}
-	for r := 0; r < maxReaders; r++ {
-		rArray[r] = &DMutex{Name: fmt.Sprintf("%s-r%d", name, r)}
-	}
 
+func NewDRWMutex(name string) *DRWMutex {
 	return &DRWMutex{
-		rArray:       rArray,
+		Name:  name,
-		rLockedArray: rLockedArray,
+		locks: make([]bool, dnodeCount),
-		w:            DMutex{Name: name + "-w"}}
+		uids:  make([]string, dnodeCount),
+	}
 }
 
-// RLock locks drw for reading.
+// Connect to respective lock server nodes on the first Lock() call.
-func (drw *DRWMutex) RLock() {
+func connectLazy() {
+	if clnts == nil {
+		panic("rpc client connections weren't initialized.")
+	}
+	for i := range clnts {
+		if clnts[i].rpc != nil {
+			continue
+		}
 
-	drw.m.Lock()
+		// Pass in unique path (as required by server.HandleHTTP().
-	defer drw.m.Unlock()
+		// Ignore failure to connect, the lock server node may join the
+		// cluster later.
+		clnt, err := rpc.DialHTTPPath("tcp", nodes[i], rpcPaths[i])
+		if err != nil {
+			clnts[i].SetRPC(nil)
+			continue
+		}
+		clnts[i].SetRPC(clnt)
+	}
+}
 
-	// Check if no write is active, block otherwise
+// RLock holds a read lock on dm.
-	// Can skip this?
+//
-	drw.w.Lock()
+// If the lock is already in use, the calling goroutine
-	drw.w.Unlock()
+// blocks until the mutex is available.
+func (dm *DRWMutex) RLock() {
+	// Shield RLock() with local mutex in order to prevent more than
+	// one broadcast going out at the same time from this node
+	dm.m.Lock()
+	defer dm.m.Unlock()
 
-	// Lock either one of the reader locks
+	runs, backOff := 1, 1
-	for i := 0; ; i++ {
+
-		drw.rLockedArray[i%maxReaders] = drw.rArray[i%maxReaders].tryLockTimeout()
+	for {
-		if drw.rLockedArray[i%maxReaders] {
+		connectLazy()
+
+		// create temp arrays on stack
+		locks := make([]bool, dnodeCount)
+		ids := make([]string, dnodeCount)
+
+		// try to acquire the lock
+		isReadLock := true
+		success := lock(clnts, &locks, &ids, dm.Name, isReadLock)
+		if success {
+			// if success, copy array to object
+			copy(dm.locks, locks[:])
+			copy(dm.uids, ids[:])
 			return
 		}
-	}
-}
 
-// RUnlock undoes a single RLock call;
+		// We timed out on the previous lock, incrementally wait for a longer back-off time,
-// it does not affect other simultaneous readers.
+		// and try again afterwards
-// It is a run-time error if rw is not locked for reading
+		time.Sleep(time.Duration(backOff) * time.Millisecond)
-// on entry to RUnlock.
-func (drw *DRWMutex) RUnlock() {
 
-	drw.m.Lock()
+		backOff += int(rand.Float64() * math.Pow(2, float64(runs)))
-	defer drw.m.Unlock()
+		if backOff > 1024 {
+			backOff = backOff % 64
 
-	// Unlock whichever readlock that was acquired)
+			runs = 1 // reset runs
-	for r := 0; r < maxReaders; r++ {
+		} else if runs < 10 {
-		if drw.rLockedArray[r] {
+			runs++
-			drw.rArray[r].Unlock()
-			drw.rLockedArray[r] = false
-			// we only want to release a single read lock at a time
-			break
 		}
 	}
 }
 
-// Lock locks rw for writing.
+// Lock locks dm.
-// If the lock is already locked for reading or writing,
+//
-// Lock blocks until the lock is available.
+// If the lock is already in use, the calling goroutine
-// To ensure that the lock eventually becomes available,
+// blocks until the mutex is available.
-// a blocked Lock call excludes new readers from acquiring
+func (dm *DRWMutex) Lock() {
-// the lock.
-func (drw *DRWMutex) Lock() {
 
-	drw.m.Lock()
+	// Shield Lock() with local mutex in order to prevent more than
-	defer drw.m.Unlock()
+	// one broadcast going out at the same time from this node
+	dm.m.Lock()
+	defer dm.m.Unlock()
 
-	// First, resolve competition with other writers.
+	runs, backOff := 1, 1
-	drw.w.Lock()
 
-	// Acquire all read locks.
+	for {
-	var wg sync.WaitGroup
+		connectLazy()
-	wg.Add(maxReaders)
 
-	for r := 0; r < maxReaders; r++ {
+		// create temp arrays on stack
-		go func(r int) {
+		locks := make([]bool, dnodeCount)
-			defer wg.Done()
+		ids := make([]string, dnodeCount)
-			drw.rArray[r].Lock()
+
-			drw.rLockedArray[r] = true
+		// try to acquire the lock
-		}(r)
+		isReadLock := false
+		success := lock(clnts, &locks, &ids, dm.Name, isReadLock)
+		if success {
+			// if success, copy array to object
+			copy(dm.locks, locks[:])
+			copy(dm.uids, ids[:])
+			return
+		}
+
+		// We timed out on the previous lock, incrementally wait for a longer back-off time,
+		// and try again afterwards
+		time.Sleep(time.Duration(backOff) * time.Millisecond)
+
+		backOff += int(rand.Float64() * math.Pow(2, float64(runs)))
+		if backOff > 1024 {
+			backOff = backOff % 64
+
+			runs = 1 // reset runs
+		} else if runs < 10 {
+			runs++
+		}
 	}
+}
+
+// lock tries to acquire the distributed lock, returning true or false
+//
+func lock(clnts []*RPCClient, locks *[]bool, uids *[]string, lockName string, isReadLock bool) bool {
+
+	// Create buffered channel of quorum size
+	ch := make(chan Granted, dquorum)
+
+	for index, c := range clnts {
+
+		// broadcast lock request to all nodes
+		go func(index int, isReadLock bool, c *RPCClient) {
+			// All client methods issuing RPCs are thread-safe and goroutine-safe,
+			// i.e. it is safe to call them from multiple concurrently running go routines.
+			var status bool
+			var err error
+			if isReadLock {
+				err = c.Call("Dsync.RLock", lockName, &status)
+			} else {
+				err = c.Call("Dsync.Lock", lockName, &status)
+			}
+
+			locked, uid := false, ""
+			if err == nil {
+				locked = status
+				// TODO: Get UIOD again
+				uid = ""
+			} else {
+				// If rpc call failed due to connection related errors, reset rpc.Client object
+				// to trigger reconnect on subsequent Lock()/Unlock() requests to the same node.
+				if IsRPCError(err) {
+					clnts[index].SetRPC(nil)
+				}
+				// silently ignore error, retry later
+			}
+
+			ch <- Granted{index: index, locked: locked, uid: uid}
+
+		}(index, isReadLock, c)
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	quorum := false
+
+	go func(isReadLock bool) {
+
+		// Wait until we have received (minimally) quorum number of responses or timeout
+		i := 0
+		done := false
+		timeout := time.After(DRWMutexAcquireTimeout)
+
+		for ; i < dnodeCount; i++ {
+
+			select {
+			case grant := <-ch:
+				if grant.locked {
+					// Mark that this node has acquired the lock
+					(*locks)[grant.index] = true
+					(*uids)[grant.index] = grant.uid
+				} else {
+					done = true
+					//fmt.Println("one lock failed before quorum -- release locks acquired")
+					releaseAll(clnts, locks, uids, lockName, isReadLock)
+				}
+
+			case <-timeout:
+				done = true
+				// timeout happened, maybe one of the nodes is slow, count
+				// number of locks to check whether we have quorum or not
+				if !quorumMet(locks) {
+					//fmt.Println("timed out -- release locks acquired")
+					releaseAll(clnts, locks, uids, lockName, isReadLock)
+				}
+			}
+
+			if done {
+				break
+			}
+		}
+
+		// Count locks in order to determine whterh we have quorum or not
+		quorum = quorumMet(locks)
+
+		// Signal that we have the quorum
+		wg.Done()
+
+		// Wait for the other responses and immediately release the locks
+		// (do not add them to the locks array because the DRWMutex could
+		//  already has been unlocked again by the original calling thread)
+		for ; i < dnodeCount; i++ {
+			grantToBeReleased := <-ch
+			if grantToBeReleased.locked {
+				// release lock
+				sendRelease(clnts[grantToBeReleased.index], lockName, grantToBeReleased.uid, isReadLock)
+			}
+		}
+	}(isReadLock)
 
 	wg.Wait()
+
+	return quorum
 }
 
-// Unlock unlocks rw for writing.  It is a run-time error if rw is
+// quorumMet determines whether we have acquired n/2+1 underlying locks or not
-// not locked for writing on entry to Unlock.
+func quorumMet(locks *[]bool) bool {
-//
-// As with Mutexes, a locked RWMutex is not associated with a particular
-// goroutine.  One goroutine may RLock (Lock) an RWMutex and then
-// arrange for another goroutine to RUnlock (Unlock) it.
-func (drw *DRWMutex) Unlock() {
 
-	drw.m.Lock()
+	count := 0
-	defer drw.m.Unlock()
+	for _, locked := range *locks {
-
+		if locked {
-	for r := 0; r < maxReaders; r++ {
+			count++
-		if !drw.rLockedArray[r] {
-			panic("dsync: unlock of unlocked distributed rwmutex")
 		}
 	}
 
-	// Unlock all read locks
+	return count >= dquorum
-	for r := 0; r < maxReaders; r++ {
+}
-		drw.rArray[r].Unlock()
+
-		drw.rLockedArray[r] = false
+// releaseAll releases all locks that are marked as locked
+func releaseAll(clnts []*RPCClient, locks *[]bool, ids *[]string, lockName string, isReadLock bool) {
+
+	for lock := 0; lock < dnodeCount; lock++ {
+		if (*locks)[lock] {
+			sendRelease(clnts[lock], lockName, (*ids)[lock], isReadLock)
+			(*locks)[lock] = false
+			(*ids)[lock] = ""
+		}
 	}
 
-	// Allow other writers to proceed.
+}
-	drw.w.Unlock()
+
+// RUnlock releases a read lock held on dm.
+//
+// It is a run-time error if dm is not locked on entry to RUnlock.
+func (dm *DRWMutex) RUnlock() {
+	// We don't panic like sync.Mutex, when an unlock is issued on an
+	// un-locked lock, since the lock rpc server may have restarted and
+	// "forgotten" about the lock.
+
+	// We don't need to wait until we have released all the locks (or the quorum)
+	// (a subsequent lock will retry automatically in case it would fail to get
+	//  quorum)
+	for index, c := range clnts {
+
+		if dm.locks[index] {
+			// broadcast lock release to all nodes the granted the lock
+			isReadLock := true
+			sendRelease(c, dm.Name, dm.uids[index], isReadLock)
+
+			dm.locks[index] = false
+		}
+	}
+}
+
+// Unlock unlocks dm.
+//
+// It is a run-time error if dm is not locked on entry to Unlock.
+func (dm *DRWMutex) Unlock() {
+
+	// We don't panic like sync.Mutex, when an unlock is issued on an
+	// un-locked lock, since the lock rpc server may have restarted and
+	// "forgotten" about the lock.
+
+	// We don't need to wait until we have released all the locks (or the quorum)
+	// (a subsequent lock will retry automatically in case it would fail to get
+	//  quorum)
+	for index, c := range clnts {
+
+		if dm.locks[index] {
+			// broadcast lock release to all nodes the granted the lock
+			isReadLock := false
+			sendRelease(c, dm.Name, dm.uids[index], isReadLock)
+
+			dm.locks[index] = false
+		}
+	}
+}
+
+// sendRelease sends a release message to a node that previously granted a lock
+func sendRelease(c *RPCClient, name, uid string, isReadLock bool) {
+
+	backOffArray := []time.Duration{30 * time.Second, 1 * time.Minute, 3 * time.Minute, 10 * time.Minute, 30 * time.Minute, 1 * time.Hour}
+
+	go func(c *RPCClient, name, uid string) {
+
+		for _, backOff := range backOffArray {
+
+			// Make sure we are connected
+			connectLazy()
+
+			// All client methods issuing RPCs are thread-safe and goroutine-safe,
+			// i.e. it is safe to call them from multiple concurrently running goroutines.
+			var status bool
+			var err error
+			// TODO: Send UID to server
+			if isReadLock {
+				if err = c.Call("Dsync.RUnlock", name, &status); err == nil {
+					// RUnlock delivered, exit out
+					return
+				}
+			} else {
+				if err = c.Call("Dsync.Unlock", name, &status); err == nil {
+					// Unlock delivered, exit out
+					return
+				}
+			}
+
+			// If rpc call failed due to connection related errors, reset rpc.Client object
+			// to trigger reconnect on subsequent Lock()/Unlock() requests to the same node.
+			c.SetRPC(nil)
+
+			// wait
+			time.Sleep(backOff)
+		}
+	}(c, name, uid)
 }

vendor/github.com/minio/dsync/dsync.go

@@ -23,16 +23,20 @@ const DebugPath = "/debug"
 
 const DefaultPath = "/rpc/dsync"
 
-var n int
+// Number of nodes participating in the distributed locking.
+var dnodeCount int
+
+// List of nodes participating.
 var nodes []string
+
+// List of rpc paths, one per lock server.
 var rpcPaths []string
+
+// List of rpc client objects, one per lock server.
 var clnts []*RPCClient
 
-func closeClients(clients []*RPCClient) {
+// Simple majority based quorum, set to dNodeCount/2+1
-	for _, clnt := range clients {
+var dquorum int
-		clnt.Close()
-	}
-}
 
 // SetNodesWithPath - initializes package-level global state variables such as
 // nodes, rpcPaths, clnts.
@@ -41,7 +45,7 @@ func closeClients(clients []*RPCClient) {
 func SetNodesWithPath(nodeList []string, paths []string) (err error) {
 
 	// Validate if number of nodes is within allowable range.
-	if n != 0 {
+	if dnodeCount != 0 {
 		return errors.New("Cannot reinitialize dsync package")
 	} else if len(nodeList) < 4 {
 		return errors.New("Dsync not designed for less than 4 nodes")
@@ -53,8 +57,9 @@ func SetNodesWithPath(nodeList []string, paths []string) (err error) {
 	copy(nodes, nodeList[:])
 	rpcPaths = make([]string, len(paths))
 	copy(rpcPaths, paths[:])
-	n = len(nodes)
+	dnodeCount = len(nodes)
-	clnts = make([]*RPCClient, n)
+	dquorum = dnodeCount/2 + 1
+	clnts = make([]*RPCClient, dnodeCount)
 	// Initialize node name and rpc path for each RPCClient object.
 	for i := range clnts {
 		clnts[i] = newClient(nodes[i], rpcPaths[i])

vendor/github.com/minio/dsync/rpc-client.go

@@ -43,11 +43,6 @@ func (rpcClient *RPCClient) SetRPC(rpc *rpc.Client) {
 	defer rpcClient.Unlock()
 	rpcClient.rpc = rpc
 }
-func (rpcClient *RPCClient) Close() error {
-	rpcClient.Lock()
-	defer rpcClient.Unlock()
-	return rpcClient.rpc.Close()
-}
 
 func (rpcClient *RPCClient) Call(serviceMethod string, args interface{}, reply interface{}) error {
 	rpcClient.Lock()

vendor/vendor.json

@@ -98,10 +98,10 @@
 			"revisionTime": "2015-11-18T20:00:48-08:00"
 		},
 		{
-			"checksumSHA1": "Ev8FdU+RSmpHQsLGzRpg5/ka7zE=",
+			"checksumSHA1": "kbVCnnU0gR/i8WA8Gs2I+/7kONY=",
 			"path": "github.com/minio/dsync",
-			"revision": "b26292b87d023da097193c8fe624d4a159e0fd03",
+			"revision": "8f4819554f1f4fffc2e1c8c706b23e5c844997f4",
-			"revisionTime": "2016-08-11T06:53:13Z"
+			"revisionTime": "2016-08-17T23:34:37Z"
 		},
 		{
 			"path": "github.com/minio/go-homedir",