Propagate creds change to cluster (Fixes #2855) (#2929)

cmd/routers.go

@@ -112,6 +112,11 @@ func configureServerHandler(srvCmdConfig serverCmdConfig) (http.Handler, error)
 	// set environmental variable MINIO_BROWSER=off to disable minio web browser.
 	// By default minio web browser is enabled.
 	if !strings.EqualFold(os.Getenv("MINIO_BROWSER"), "off") {
+		// Register RPC router for web related calls.
+		if err = registerBrowserRPCRouter(mux); err != nil {
+			return nil, err
+		}
+
 		if err = registerWebRouter(mux); err != nil {
 			return nil, err
 		}

cmd/web-handlers.go

@@ -338,6 +338,7 @@ type SetAuthArgs struct {
 type SetAuthReply struct {
 	Token       string            `json:"token"`
 	UIVersion   string            `json:"uiVersion"`
+	PeerErrMsgs map[string]string `json:"peerErrMsgs"`
 }
 
 // SetAuth - Set accessKey and secretKey credentials.
@@ -351,24 +352,68 @@ func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *Se
 	if !isValidSecretKey.MatchString(args.SecretKey) {
 		return &json2.Error{Message: "Invalid Secret Key"}
 	}
+
 	cred := credential{args.AccessKey, args.SecretKey}
-	serverConfig.SetCredential(cred)
+	unexpErrsMsg := "ALERT: Unexpected error(s) happened - please check the server logs."
-	if err := serverConfig.Save(); err != nil {
+	gaveUpMsg := func(errMsg error, moreErrors bool) *json2.Error {
-		return &json2.Error{Message: err.Error()}
+		msg := fmt.Sprintf(
+			"ALERT: We gave up due to: '%s', but there were more errors. Please check the server logs.",
+			errMsg.Error(),
+		)
+		if moreErrors {
+			return &json2.Error{Message: msg}
+		}
+		return &json2.Error{Message: errMsg.Error()}
 	}
 
+	// Notify all other Minio peers to update credentials
+	errsMap := updateCredsOnPeers(cred)
+
+	// Update local credentials
+	serverConfig.SetCredential(cred)
+	if err := serverConfig.Save(); err != nil {
+		errsMap[globalMinioAddr] = err
+	}
+
+	// Log all the peer related error messages, and populate the
+	// PeerErrMsgs map.
+	reply.PeerErrMsgs = make(map[string]string)
+	for svr, errVal := range errsMap {
+		tErr := fmt.Errorf("Unable to change credentials on %s: %v", svr, errVal)
+		errorIf(tErr, "Credentials change could not be propagated successfully!")
+		reply.PeerErrMsgs[svr] = errVal.Error()
+	}
+
+	// If we were unable to update locally, we return an error to
+	// the user/browser.
+	if errsMap[globalMinioAddr] != nil {
+		// Since the error message may be very long to display
+		// on the browser, we tell the user to check the
+		// server logs.
+		return &json2.Error{Message: unexpErrsMsg}
+	}
+
+	// Did we have peer errors?
+	var moreErrors bool
+	if len(errsMap) > 0 {
+		moreErrors = true
+	}
+
+	// If we were able to update locally, we try to generate a new
+	// token and complete the request.
 	jwt, err := newJWT(defaultJWTExpiry) // JWT Expiry set to 24Hrs.
 	if err != nil {
-		return &json2.Error{Message: err.Error()}
+		return gaveUpMsg(err, moreErrors)
 	}
 
 	if err = jwt.Authenticate(args.AccessKey, args.SecretKey); err != nil {
-		return &json2.Error{Message: err.Error()}
+		return gaveUpMsg(err, moreErrors)
 	}
 	token, err := jwt.GenerateToken(args.AccessKey)
 	if err != nil {
-		return &json2.Error{Message: err.Error()}
+		return gaveUpMsg(err, moreErrors)
 	}
+
 	reply.Token = token
 	reply.UIVersion = miniobrowser.UIVersion
 	return nil

cmd/web-peer-rpc.go

@@ -0,0 +1,146 @@
+/*
+ * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"path"
+	"sync"
+	"time"
+)
+
+func (br *browserAPIHandlers) LoginHandler(args *RPCLoginArgs, reply *RPCLoginReply) error {
+	jwt, err := newJWT(defaultInterNodeJWTExpiry)
+	if err != nil {
+		return err
+	}
+	if err = jwt.Authenticate(args.Username, args.Password); err != nil {
+		return err
+	}
+	token, err := jwt.GenerateToken(args.Username)
+	if err != nil {
+		return err
+	}
+	reply.Token = token
+	reply.ServerVersion = Version
+	reply.Timestamp = time.Now().UTC()
+	return nil
+}
+
+// SetAuthPeerArgs - Arguments collection for SetAuth RPC call
+type SetAuthPeerArgs struct {
+	// For Auth
+	GenericArgs
+
+	// New credentials that receiving peer should update to.
+	Creds credential
+}
+
+// SetAuthPeer - Update to new credentials sent from a peer Minio
+// server. Since credentials are already validated on the sending
+// peer, here we just persist to file and update in-memory config. All
+// subsequently running isRPCTokenValid() calls will fail, and clients
+// will be forced to re-establish connections. Connections will be
+// re-established only when the sending client has also updated its
+// credentials.
+func (br *browserAPIHandlers) SetAuthPeer(args SetAuthPeerArgs, reply *GenericReply) error {
+	// Check auth
+	if !isRPCTokenValid(args.Token) {
+		return errInvalidToken
+	}
+
+	// Update credentials in memory
+	serverConfig.SetCredential(args.Creds)
+
+	// Save credentials to config file
+	if err := serverConfig.Save(); err != nil {
+		errorIf(err, "Error updating config file with new credentials sent from browser RPC.")
+		return err
+	}
+
+	return nil
+}
+
+// Sends SetAuthPeer RPCs to all peers in the Minio cluster
+func updateCredsOnPeers(creds credential) map[string]error {
+	// Get list of peers (from globalS3Peers)
+	peers := globalS3Peers.GetPeers()
+
+	// Array of errors for each peer
+	errs := make([]error, len(peers))
+	var wg sync.WaitGroup
+
+	// Launch go routines to send request to each peer in
+	// parallel.
+	for ix := range peers {
+		wg.Add(1)
+		go func(ix int) {
+			defer wg.Done()
+
+			// Exclude self to avoid race with
+			// invalidating the RPC token.
+			if peers[ix] == globalMinioAddr {
+				errs[ix] = nil
+				return
+			}
+
+			// Initialize client
+			client := newAuthClient(&authConfig{
+				accessKey:   serverConfig.GetCredential().AccessKeyID,
+				secretKey:   serverConfig.GetCredential().SecretAccessKey,
+				address:     peers[ix],
+				path:        path.Join(reservedBucket, browserPath),
+				loginMethod: "Browser.LoginHandler",
+			})
+
+			// Construct RPC call arguments.
+			args := SetAuthPeerArgs{Creds: creds}
+
+			// Make RPC call - we only care about error
+			// response and not the reply.
+			err := client.Call("Browser.SetAuthPeer", &args, &GenericReply{})
+
+			// we try a bit hard (3 attempts with 1 second
+			// delay) to set creds on peers in case of
+			// failure.
+			if err != nil {
+				for i := 0; i < 2; i++ {
+					time.Sleep(1 * time.Second)
+					err = client.Call("Browser.SetAuthPeer", &args, &GenericReply{})
+					if err == nil {
+						break
+					}
+				}
+			}
+
+			// Send result down the channel
+			errs[ix] = err
+		}(ix)
+	}
+
+	// Wait for requests to complete.
+	wg.Wait()
+
+	// Put errors into map.
+	errsMap := make(map[string]error)
+	for i, err := range errs {
+		if err != nil {
+			errsMap[peers[i]] = err
+		}
+	}
+
+	return errsMap
+}

cmd/web-rpc-router.go

@@ -0,0 +1,50 @@
+/*
+ * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net/rpc"
+
+	router "github.com/gorilla/mux"
+)
+
+// Set up an RPC endpoint that receives browser related calls. The
+// original motivation is for propagating credentials change
+// throughout Minio cluster, initiated from a Minio browser session.
+
+const (
+	browserPath = "/browser/setauth"
+)
+
+// The Type exporting methods exposed for RPC calls.
+type browserAPIHandlers struct {
+}
+
+// Register RPC router
+func registerBrowserRPCRouter(mux *router.Router) error {
+	browserHandlers := &browserAPIHandlers{}
+
+	browserRPCServer := rpc.NewServer()
+	err := browserRPCServer.RegisterName("Browser", browserHandlers)
+	if err != nil {
+		return traceError(err)
+	}
+
+	browserRouter := mux.NewRoute().PathPrefix(reservedBucket).Subrouter()
+	browserRouter.Path(browserPath).Handler(browserRPCServer)
+	return nil
+}