Add bucket metadata state client/handler (Fixes #3022) (#3152)

- Adds an interface to update in-memory bucket metadata state called
  BucketMetaState - this interface has functions to:
     - update bucket notification configuration,
     - bucket listener configuration,
     - bucket policy configuration, and
     - send bucket event

- This interface is implemented by `localBMS` a type for manipulating
  local node in-memory bucket metadata, and by `remoteBMS` a type for
  manipulating remote node in-memory bucket metadata.

- The remote node interface, makes an RPC call, but the local node
  interface does not - it updates in-memory bucket state directly.

- Rename mkPeersFromEndpoints to makeS3Peers and refactored it.

- Use arrayslice instead of map in s3Peers struct

- `s3Peers.SendUpdate` now receives an arrayslice of peer indexes to
  send the request to, with a special nil value slice indicating that
  all peers should be sent the update.

- `s3Peers.SendUpdate` now returns an arrayslice of errors, representing
  errors from peers when sending an update. The array positions
  correspond to peer array s3Peers.peers

Improve globalS3Peers:

- Make isDistXL a global `globalIsDistXL` and remove from s3Peers

- Make globalS3Peers an array of (address, bucket-meta-state) pairs.

- Fix code and tests.

cmd/server-main.go

@@ -103,7 +103,6 @@ type serverCmdConfig struct {
 	serverAddr       string
 	endpoints        []*url.URL
 	ignoredEndpoints []*url.URL
-	isDistXL         bool // True only if its distributed XL.
 	storageDisks     []StorageAPI
 }
 
@@ -266,17 +265,16 @@ func checkSufficientDisks(eps []*url.URL) error {
 }
 
 // Returns if slice of disks is a distributed setup.
-func isDistributedSetup(eps []*url.URL) (isDist bool) {
+func isDistributedSetup(eps []*url.URL) bool {
 	// Validate if one the disks is not local.
 	for _, ep := range eps {
 		if !isLocalStorage(ep) {
-			// One or more disks supplied as arguments are not
-			// attached to the local node.
-			isDist = true
-			break
+			// One or more disks supplied as arguments are
+			// not attached to the local node.
+			return true
 		}
 	}
-	return isDist
+	return false
 }
 
 // We just exit for invalid endpoints.
@@ -446,7 +444,7 @@ func serverMain(c *cli.Context) {
 	firstDisk := isLocalStorage(endpoints[0])
 
 	// Check if endpoints are part of distributed setup.
-	isDistXL := isDistributedSetup(endpoints)
+	globalIsDistXL = isDistributedSetup(endpoints)
 
 	// Configure server.
 	srvConfig := serverCmdConfig{
@@ -454,7 +452,6 @@ func serverMain(c *cli.Context) {
 		endpoints:        endpoints,
 		ignoredEndpoints: ignoredEndpoints,
 		storageDisks:     storageDisks,
-		isDistXL:         isDistXL,
 	}
 
 	// Configure server.
@@ -462,12 +459,12 @@ func serverMain(c *cli.Context) {
 	fatalIf(err, "Unable to configure one of server's RPC services.")
 
 	// Set nodes for dsync for distributed setup.
-	if isDistXL {
+	if globalIsDistXL {
 		fatalIf(initDsyncNodes(endpoints), "Unable to initialize distributed locking")
 	}
 
 	// Initialize name space lock.
-	initNSLock(isDistXL)
+	initNSLock(globalIsDistXL)
 
 	// Initialize a new HTTP server.
 	apiServer := NewServerMux(serverAddr, handler)