Add bucket metadata state client/handler (Fixes #3022) (#3152)

- Adds an interface to update in-memory bucket metadata state called
  BucketMetaState - this interface has functions to:
     - update bucket notification configuration,
     - bucket listener configuration,
     - bucket policy configuration, and
     - send bucket event

- This interface is implemented by `localBMS` a type for manipulating
  local node in-memory bucket metadata, and by `remoteBMS` a type for
  manipulating remote node in-memory bucket metadata.

- The remote node interface, makes an RPC call, but the local node
  interface does not - it updates in-memory bucket state directly.

- Rename mkPeersFromEndpoints to makeS3Peers and refactored it.

- Use arrayslice instead of map in s3Peers struct

- `s3Peers.SendUpdate` now receives an arrayslice of peer indexes to
  send the request to, with a special nil value slice indicating that
  all peers should be sent the update.

- `s3Peers.SendUpdate` now returns an arrayslice of errors, representing
  errors from peers when sending an update. The array positions
  correspond to peer array s3Peers.peers

Improve globalS3Peers:

- Make isDistXL a global `globalIsDistXL` and remove from s3Peers

- Make globalS3Peers an array of (address, bucket-meta-state) pairs.

- Fix code and tests.

cmd/s3-peer-rpc-handlers.go

@@ -16,10 +16,7 @@
 
 package cmd
 
-import (
-	"encoding/json"
-	"time"
-)
+import "time"
 
 func (s3 *s3PeerAPIHandlers) LoginHandler(args *RPCLoginArgs, reply *RPCLoginReply) error {
 	jwt, err := newJWT(defaultInterNodeJWTExpiry)
@@ -57,16 +54,7 @@ func (s3 *s3PeerAPIHandlers) SetBucketNotificationPeer(args *SetBNPArgs, reply *
 		return errInvalidToken
 	}
 
-	// check if object layer is available.
-	objAPI := s3.ObjectAPI()
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	// Update in-memory notification config.
-	globalEventNotifier.SetBucketNotificationConfig(args.Bucket, args.NCfg)
-
-	return nil
+	return s3.UpdateBucketNotification(args)
 }
 
 // SetBLPArgs - Arguments collection to SetBucketListenerPeer RPC call
@@ -80,20 +68,13 @@ type SetBLPArgs struct {
 	LCfg []listenerConfig
 }
 
-func (s3 *s3PeerAPIHandlers) SetBucketListenerPeer(args SetBLPArgs, reply *GenericReply) error {
+func (s3 *s3PeerAPIHandlers) SetBucketListenerPeer(args *SetBLPArgs, reply *GenericReply) error {
 	// check auth
 	if !isRPCTokenValid(args.Token) {
 		return errInvalidToken
 	}
 
-	// check if object layer is available.
-	objAPI := s3.ObjectAPI()
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	// Update in-memory notification config.
-	return globalEventNotifier.SetBucketListenerConfig(args.Bucket, args.LCfg)
+	return s3.UpdateBucketListener(args)
 }
 
 // EventArgs - Arguments collection for Event RPC call
@@ -115,13 +96,7 @@ func (s3 *s3PeerAPIHandlers) Event(args *EventArgs, reply *GenericReply) error {
 		return errInvalidToken
 	}
 
-	// check if object layer is available.
-	objAPI := s3.ObjectAPI()
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	return globalEventNotifier.SendListenerEvent(args.Arn, args.Event)
+	return s3.SendEvent(args)
 }
 
 // SetBPPArgs - Arguments collection for SetBucketPolicyPeer RPC call
@@ -136,22 +111,11 @@ type SetBPPArgs struct {
 }
 
 // tell receiving server to update a bucket policy
-func (s3 *s3PeerAPIHandlers) SetBucketPolicyPeer(args SetBPPArgs, reply *GenericReply) error {
+func (s3 *s3PeerAPIHandlers) SetBucketPolicyPeer(args *SetBPPArgs, reply *GenericReply) error {
 	// check auth
 	if !isRPCTokenValid(args.Token) {
 		return errInvalidToken
 	}
 
-	// check if object layer is available.
-	objAPI := s3.ObjectAPI()
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	var pCh policyChange
-	if err := json.Unmarshal(args.PChBytes, &pCh); err != nil {
-		return err
-	}
-
-	return globalBucketPolicies.SetBucketPolicy(args.Bucket, pCh)
+	return s3.UpdateBucketPolicy(args)
 }