Add bucket metadata state client/handler (Fixes #3022) (#3152)

- Adds an interface to update in-memory bucket metadata state called
  BucketMetaState - this interface has functions to:
     - update bucket notification configuration,
     - bucket listener configuration,
     - bucket policy configuration, and
     - send bucket event

- This interface is implemented by `localBMS` a type for manipulating
  local node in-memory bucket metadata, and by `remoteBMS` a type for
  manipulating remote node in-memory bucket metadata.

- The remote node interface, makes an RPC call, but the local node
  interface does not - it updates in-memory bucket state directly.

- Rename mkPeersFromEndpoints to makeS3Peers and refactored it.

- Use arrayslice instead of map in s3Peers struct

- `s3Peers.SendUpdate` now receives an arrayslice of peer indexes to
  send the request to, with a special nil value slice indicating that
  all peers should be sent the update.

- `s3Peers.SendUpdate` now returns an arrayslice of errors, representing
  errors from peers when sending an update. The array positions
  correspond to peer array s3Peers.peers

Improve globalS3Peers:

- Make isDistXL a global `globalIsDistXL` and remove from s3Peers

- Make globalS3Peers an array of (address, bucket-meta-state) pairs.

- Fix code and tests.

cmd/notify-listener.go

@@ -27,6 +27,7 @@ import (
 type listenerConn struct {
 	TargetAddr  string
 	ListenerARN string
+	BMSClient   BucketMetaState
 }
 
 type listenerLogger struct {
@@ -35,7 +36,8 @@ type listenerLogger struct {
 }
 
 func newListenerLogger(listenerArn, targetAddr string) (*listenerLogger, error) {
-	if globalS3Peers.GetPeerClient(targetAddr) == nil {
+	bmsClient := globalS3Peers.GetPeerClient(targetAddr)
+	if bmsClient == nil {
 		return nil, fmt.Errorf(
 			"Peer %s was not initialized - bug!",
 			targetAddr,
@@ -44,6 +46,7 @@ func newListenerLogger(listenerArn, targetAddr string) (*listenerLogger, error)
 	lc := listenerConn{
 		TargetAddr:  targetAddr,
 		ListenerARN: listenerArn,
+		BMSClient:   bmsClient,
 	}
 
 	lcLog := logrus.New()
@@ -66,21 +69,14 @@ func (lc listenerConn) Fire(entry *logrus.Entry) error {
 		return nil
 	}
 
-	// Fetch peer client object
-	client := globalS3Peers.GetPeerClient(lc.TargetAddr)
-	if client == nil {
-		return fmt.Errorf("Target %s client RPC object not available!", lc.TargetAddr)
-	}
-
 	// Send Event RPC call and return error
 	arg := EventArgs{Event: notificationEvent, Arn: lc.ListenerARN}
-	reply := GenericReply{}
-	err := client.Call("S3.Event", &arg, &reply)
+	err := lc.BMSClient.SendEvent(&arg)
 
 	// In case connection is shutdown, retry once.
 	if err != nil {
 		if err.Error() == rpc.ErrShutdown.Error() {
-			err = client.Call("S3.Event", &arg, &reply)
+			err = lc.BMSClient.SendEvent(&arg)
 		}
 	}
 	return err