Migrate all Peer communication to common Notification subsystem (#7031)

Deprecate the use of Admin Peers concept and migrate all peer
communication to Notification subsystem. This finally allows
for a common subsystem for all peer notification in case of
distributed server deployments.

cmd/admin-handlers.go

@@ -17,22 +17,22 @@
 package cmd
 
 import (
-	"archive/zip"
 	"bytes"
 	"context"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
-	"fmt"
 	"io"
 	"net/http"
 	"os"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/tidwall/gjson"
+	"github.com/tidwall/sjson"
+
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/cpu"
@@ -41,9 +41,8 @@ import (
 	"github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/mem"
+	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/quick"
-	"github.com/tidwall/gjson"
-	"github.com/tidwall/sjson"
 )
 
 const (
@@ -97,6 +96,11 @@ func (a adminAPIHandlers) VersionHandler(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServiceStatus")
 
+	if globalNotificationSys == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
 	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
@@ -109,13 +113,8 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
 		CommitID: CommitID,
 	}
 
-	// Fetch uptimes from all peers. This may fail to due to lack
+	// Fetch uptimes from all peers and pick the latest.
-	// of read-quorum availability.
+	uptime := getPeerUptimes(globalNotificationSys.ServerInfo(ctx))
-	uptime, err := getPeerUptimes(globalAdminPeers)
-	if err != nil {
-		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
-		return
-	}
 
 	// Create API response
 	serverStatus := madmin.ServiceStatus{
@@ -129,6 +128,7 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
 		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
+
 	// Reply with storage information (across nodes in a
 	// distributed setup) as json.
 	writeSuccessResponseJSON(w, jsonBytes)
@@ -142,6 +142,11 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
 func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServiceStopNRestart")
 
+	if globalNotificationSys == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
 	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
@@ -151,7 +156,6 @@ func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *h
 	var sa madmin.ServiceAction
 	err := json.NewDecoder(r.Body).Decode(&sa)
 	if err != nil {
-		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(w, ErrRequestBodyParse, r.URL)
 		return
 	}
@@ -171,7 +175,15 @@ func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *h
 	// Reply to the client before restarting minio server.
 	writeSuccessResponseHeadersOnly(w)
 
-	sendServiceCmd(globalAdminPeers, serviceSig)
+	// Notify all other Minio peers signal service.
+	for _, nerr := range globalNotificationSys.SignalService(serviceSig) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+
+	globalServiceSignalCh <- serviceSig
 }
 
 // ServerProperties holds some server information such as, version, region
@@ -235,6 +247,12 @@ type ServerInfo struct {
 func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServerInfo")
 
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
 	// Authenticate request
 
 	// Setting the region as empty so as the mc server info command is irrespective to the region.
@@ -244,39 +262,33 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	// Web service response
+	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
-	reply := make([]ServerInfo, len(globalAdminPeers))
+	if err != nil {
-
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
-	var wg sync.WaitGroup
+		return
-
-	// Gather server information for all nodes
-	for i, p := range globalAdminPeers {
-		wg.Add(1)
-
-		// Gather information from a peer in a goroutine
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-
-			// Initialize server info at index
-			reply[idx] = ServerInfo{Addr: peer.addr}
-
-			serverInfoData, err := peer.cmdRunner.ServerInfo()
-			if err != nil {
-				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", peer.addr)
-				ctx := logger.SetReqInfo(ctx, reqInfo)
-				logger.LogIf(ctx, err)
-				reply[idx].Error = err.Error()
-				return
-			}
-
-			reply[idx].Data = &serverInfoData
-		}(i, p)
 	}
 
-	wg.Wait()
+	serverInfo := globalNotificationSys.ServerInfo(ctx)
+	// Once we have recieved all the ServerInfo from peers
+	// add the local peer server info as well.
+	serverInfo = append(serverInfo, ServerInfo{
+		Addr: thisAddr.String(),
+		Data: &ServerInfoData{
+			StorageInfo: objectAPI.StorageInfo(ctx),
+			ConnStats:   globalConnStats.toServerConnStats(),
+			HTTPStats:   globalHTTPStats.toServerHTTPStats(),
+			Properties: ServerProperties{
+				Uptime:   UTCNow().Sub(globalBootTime),
+				Version:  Version,
+				CommitID: CommitID,
+				SQSARN:   globalNotificationSys.GetARNList(),
+				Region:   globalServerConfig.GetRegion(),
+			},
+		},
+	})
 
 	// Marshal API response
-	jsonBytes, err := json.Marshal(reply)
+	jsonBytes, err := json.Marshal(serverInfo)
 	if err != nil {
 		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
@@ -323,7 +335,7 @@ func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request
 
 	// Get object layer instance.
 	objLayer := newObjectLayerFn()
-	if objLayer == nil {
+	if objLayer == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
 		return
 	}
@@ -417,6 +429,11 @@ type StartProfilingResult struct {
 func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "StartProfiling")
 
+	if globalNotificationSys == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
 	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
@@ -426,31 +443,53 @@ func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.R
 	vars := mux.Vars(r)
 	profiler := vars["profilerType"]
 
-	startProfilingResult := make([]StartProfilingResult, len(globalAdminPeers))
+	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
-
+	if err != nil {
-	// Call StartProfiling function on all nodes and save results
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
-	wg := sync.WaitGroup{}
+		return
-	for i, peer := range globalAdminPeers {
+	}
-		wg.Add(1)
+
-		go func(idx int, peer adminPeer) {
+	// Start profiling on remote servers.
-			defer wg.Done()
+	hostErrs := globalNotificationSys.StartProfiling(profiler)
-			result := StartProfilingResult{NodeName: peer.addr}
+
-			if err := peer.cmdRunner.StartProfiling(profiler); err != nil {
+	// Start profiling locally as well.
-				result.Error = err.Error()
+	{
-				return
+		if globalProfiler != nil {
-			}
+			globalProfiler.Stop()
-			result.Success = true
+		}
-			startProfilingResult[idx] = result
+		prof, err := startProfiler(profiler, "")
-		}(i, peer)
+		if err != nil {
+			hostErrs = append(hostErrs, NotificationPeerErr{
+				Host: *thisAddr,
+				Err:  err,
+			})
+		} else {
+			globalProfiler = prof
+			hostErrs = append(hostErrs, NotificationPeerErr{
+				Host: *thisAddr,
+			})
+		}
+	}
+
+	var startProfilingResult []StartProfilingResult
+
+	for _, nerr := range hostErrs {
+		result := StartProfilingResult{NodeName: nerr.Host.String()}
+		if nerr.Err != nil {
+			result.Error = nerr.Err.Error()
+		} else {
+			result.Success = true
+		}
+		startProfilingResult = append(startProfilingResult, result)
 	}
-	wg.Wait()
 
 	// Create JSON result and send it to the client
 	startProfilingResultInBytes, err := json.Marshal(startProfilingResult)
 	if err != nil {
-		writeCustomErrorResponseJSON(w, http.StatusInternalServerError, err.Error(), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
+
 	writeSuccessResponseJSON(w, []byte(startProfilingResultInBytes))
 }
 
@@ -478,51 +517,18 @@ func (f dummyFileInfo) Sys() interface{}   { return f.sys }
 func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DownloadProfiling")
 
+	if globalNotificationSys == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
 	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
 	}
 
-	profilingDataFound := false
+	if !globalNotificationSys.DownloadProfilingData(ctx, w) {
-
-	// Initialize a zip writer which will provide a zipped content
-	// of profiling data of all nodes
-	zipWriter := zip.NewWriter(w)
-	defer zipWriter.Close()
-
-	for i, peer := range globalAdminPeers {
-		// Get profiling data from a node
-		data, err := peer.cmdRunner.DownloadProfilingData()
-		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to download profiling data from node `%s`, reason: %s", peer.addr, err.Error()))
-			continue
-		}
-
-		profilingDataFound = true
-
-		// Send profiling data to zip as file
-		header, err := zip.FileInfoHeader(dummyFileInfo{
-			name:    fmt.Sprintf("profiling-%d", i),
-			size:    int64(len(data)),
-			mode:    0600,
-			modTime: time.Now().UTC(),
-			isDir:   false,
-			sys:     nil,
-		})
-		if err != nil {
-			continue
-		}
-		writer, err := zipWriter.CreateHeader(header)
-		if err != nil {
-			continue
-		}
-		if _, err = io.Copy(writer, bytes.NewBuffer(data)); err != nil {
-			return
-		}
-	}
-
-	if !profilingDataFound {
 		writeErrorResponseJSON(w, ErrAdminProfilerNotEnabled, r.URL)
 		return
 	}
@@ -952,6 +958,11 @@ func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetUserStatus")
 
+	if globalNotificationSys == nil || globalIAMSys == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
@@ -988,10 +999,10 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 	}
 
 	// Notify all other Minio peers to reload users
-	for host, err := range globalNotificationSys.LoadUsers() {
+	for _, nerr := range globalNotificationSys.LoadUsers() {
-		if err != nil {
+		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, err)
+			logger.LogIf(ctx, nerr.Err)
 		}
 	}
 }
@@ -1002,7 +1013,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
+	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
 		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
 		return
 	}
@@ -1056,10 +1067,10 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Notify all other Minio peers to reload users
-	for host, err := range globalNotificationSys.LoadUsers() {
+	for _, nerr := range globalNotificationSys.LoadUsers() {
-		if err != nil {
+		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, err)
+			logger.LogIf(ctx, nerr.Err)
 		}
 	}
 }
@@ -1070,7 +1081,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
+	if objectAPI == nil || globalIAMSys == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
 		return
 	}
@@ -1102,7 +1113,7 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
+	if objectAPI == nil || globalIAMSys == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
 		return
 	}
@@ -1129,10 +1140,10 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 	}
 
 	// Notify all other Minio peers to reload users
-	for host, err := range globalNotificationSys.LoadUsers() {
+	for _, nerr := range globalNotificationSys.LoadUsers() {
-		if err != nil {
+		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, err)
+			logger.LogIf(ctx, nerr.Err)
 		}
 	}
 }
@@ -1143,7 +1154,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
+	if objectAPI == nil || globalIAMSys == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
 		return
 	}
@@ -1194,10 +1205,10 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 	}
 
 	// Notify all other Minio peers to reload users
-	for host, err := range globalNotificationSys.LoadUsers() {
+	for _, nerr := range globalNotificationSys.LoadUsers() {
-		if err != nil {
+		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, err)
+			logger.LogIf(ctx, nerr.Err)
 		}
 	}
 }
@@ -1208,7 +1219,7 @@ func (a adminAPIHandlers) SetUserPolicy(w http.ResponseWriter, r *http.Request)
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
+	if objectAPI == nil || globalIAMSys == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
 		return
 	}
@@ -1241,10 +1252,10 @@ func (a adminAPIHandlers) SetUserPolicy(w http.ResponseWriter, r *http.Request)
 	}
 
 	// Notify all other Minio peers to reload users
-	for host, err := range globalNotificationSys.LoadUsers() {
+	for _, nerr := range globalNotificationSys.LoadUsers() {
-		if err != nil {
+		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, err)
+			logger.LogIf(ctx, nerr.Err)
 		}
 	}
 }
@@ -1255,7 +1266,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
+	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
 		return
 	}
@@ -1472,7 +1483,7 @@ func (a adminAPIHandlers) UpdateAdminCredentialsHandler(w http.ResponseWriter,
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
+	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
 		return
 	}
@@ -1535,10 +1546,10 @@ func (a adminAPIHandlers) UpdateAdminCredentialsHandler(w http.ResponseWriter,
 	}
 
 	// Notify all other Minio peers to update credentials
-	for host, err := range globalNotificationSys.LoadCredentials() {
+	for _, nerr := range globalNotificationSys.LoadCredentials() {
-		if err != nil {
+		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, err)
+			logger.LogIf(ctx, nerr.Err)
 		}
 	}
 

cmd/admin-handlers_test.go

@@ -533,7 +533,6 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	// single node setup, this degenerates to a simple function
 	// call under the hood.
 	globalMinioAddr = "127.0.0.1:9000"
-	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	var wg sync.WaitGroup
 
@@ -607,7 +606,6 @@ func TestServiceSetCreds(t *testing.T) {
 	// single node setup, this degenerates to a simple function
 	// call under the hood.
 	globalMinioAddr = "127.0.0.1:9000"
-	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	credentials := globalServerConfig.GetCredential()
 
@@ -706,7 +704,6 @@ func TestGetConfigHandler(t *testing.T) {
 
 	// Initialize admin peers to make admin RPC calls.
 	globalMinioAddr = "127.0.0.1:9000"
-	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	// Prepare query params for get-config mgmt REST API.
 	queryVal := url.Values{}
@@ -735,7 +732,6 @@ func TestSetConfigHandler(t *testing.T) {
 
 	// Initialize admin peers to make admin RPC calls.
 	globalMinioAddr = "127.0.0.1:9000"
-	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	// Prepare query params for set-config mgmt REST API.
 	queryVal := url.Values{}
@@ -807,7 +803,6 @@ func TestAdminServerInfo(t *testing.T) {
 
 	// Initialize admin peers to make admin RPC calls.
 	globalMinioAddr = "127.0.0.1:9000"
-	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	// Prepare query params for set-config mgmt REST API.
 	queryVal := url.Values{}

cmd/admin-heal-ops.go

@@ -641,10 +641,10 @@ func (h *healSequence) healDiskFormat() error {
 	// Healing succeeded notify the peers to reload format and re-initialize disks.
 	// We will not notify peers only if healing succeeded.
 	if err == nil {
-		for host, rerr := range globalNotificationSys.ReloadFormat(h.settings.DryRun) {
+		for _, nerr := range globalNotificationSys.ReloadFormat(h.settings.DryRun) {
-			if rerr != nil {
+			if nerr.Err != nil {
-				logger.GetReqInfo(h.ctx).SetTags("peerAddress", host.String())
+				logger.GetReqInfo(h.ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(h.ctx, rerr)
+				logger.LogIf(h.ctx, nerr.Err)
 			}
 		}
 	}

cmd/admin-rpc-client.go

@@ -1,261 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2014, 2015, 2016, 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-	"crypto/tls"
-	"fmt"
-	"net"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/minio/minio/cmd/logger"
-	xnet "github.com/minio/minio/pkg/net"
-)
-
-var errUnsupportedSignal = fmt.Errorf("unsupported signal: only restart and stop signals are supported")
-
-// AdminRPCClient - admin RPC client talks to admin RPC server.
-type AdminRPCClient struct {
-	*RPCClient
-}
-
-// SignalService - calls SignalService RPC.
-func (rpcClient *AdminRPCClient) SignalService(signal serviceSignal) (err error) {
-	args := SignalServiceArgs{Sig: signal}
-	reply := VoidReply{}
-
-	return rpcClient.Call(adminServiceName+".SignalService", &args, &reply)
-}
-
-// ServerInfo - returns the server info of the server to which the RPC call is made.
-func (rpcClient *AdminRPCClient) ServerInfo() (sid ServerInfoData, err error) {
-	err = rpcClient.Call(adminServiceName+".ServerInfo", &AuthArgs{}, &sid)
-	return sid, err
-}
-
-// StartProfiling - starts profiling in the remote server.
-func (rpcClient *AdminRPCClient) StartProfiling(profiler string) error {
-	args := StartProfilingArgs{Profiler: profiler}
-	reply := VoidReply{}
-	return rpcClient.Call(adminServiceName+".StartProfiling", &args, &reply)
-}
-
-// DownloadProfilingData - returns profiling data of the remote server.
-func (rpcClient *AdminRPCClient) DownloadProfilingData() ([]byte, error) {
-	args := AuthArgs{}
-	var reply []byte
-
-	err := rpcClient.Call(adminServiceName+".DownloadProfilingData", &args, &reply)
-	return reply, err
-}
-
-// NewAdminRPCClient - returns new admin RPC client.
-func NewAdminRPCClient(host *xnet.Host) (*AdminRPCClient, error) {
-	scheme := "http"
-	if globalIsSSL {
-		scheme = "https"
-	}
-
-	serviceURL := &xnet.URL{
-		Scheme: scheme,
-		Host:   host.String(),
-		Path:   adminServicePath,
-	}
-
-	var tlsConfig *tls.Config
-	if globalIsSSL {
-		tlsConfig = &tls.Config{
-			ServerName: host.Name,
-			RootCAs:    globalRootCAs,
-		}
-	}
-
-	rpcClient, err := NewRPCClient(
-		RPCClientArgs{
-			NewAuthTokenFunc: newAuthToken,
-			RPCVersion:       globalRPCAPIVersion,
-			ServiceName:      adminServiceName,
-			ServiceURL:       serviceURL,
-			TLSConfig:        tlsConfig,
-		},
-	)
-	if err != nil {
-		return nil, err
-	}
-
-	return &AdminRPCClient{rpcClient}, nil
-}
-
-// adminCmdRunner - abstracts local and remote execution of admin
-// commands like service stop and service restart.
-type adminCmdRunner interface {
-	SignalService(s serviceSignal) error
-	ServerInfo() (ServerInfoData, error)
-	StartProfiling(string) error
-	DownloadProfilingData() ([]byte, error)
-}
-
-// adminPeer - represents an entity that implements admin API RPCs.
-type adminPeer struct {
-	addr      string
-	cmdRunner adminCmdRunner
-	isLocal   bool
-}
-
-// type alias for a collection of adminPeer.
-type adminPeers []adminPeer
-
-// makeAdminPeers - helper function to construct a collection of adminPeer.
-func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
-	localAddr := GetLocalPeer(endpoints)
-	if strings.HasPrefix(localAddr, "127.0.0.1:") {
-		// Use first IPv4 instead of loopback address.
-		localAddr = net.JoinHostPort(sortIPs(localIP4.ToSlice())[0], globalMinioPort)
-	}
-	if strings.HasPrefix(localAddr, "[::1]:") {
-		// Use first IPv4 instead of loopback address.
-		localAddr = net.JoinHostPort(localIP6.ToSlice()[0], globalMinioPort)
-	}
-
-	adminPeerList = append(adminPeerList, adminPeer{
-		addr:      localAddr,
-		cmdRunner: localAdminClient{},
-		isLocal:   true,
-	})
-
-	for _, hostStr := range GetRemotePeers(endpoints) {
-		host, err := xnet.ParseHost(hostStr)
-		logger.FatalIf(err, "Unable to parse Admin RPC Host")
-		rpcClient, err := NewAdminRPCClient(host)
-		logger.FatalIf(err, "Unable to initialize Admin RPC Client")
-		adminPeerList = append(adminPeerList, adminPeer{
-			addr:      hostStr,
-			cmdRunner: rpcClient,
-		})
-	}
-
-	return adminPeerList
-}
-
-// Initialize global adminPeer collection.
-func initGlobalAdminPeers(endpoints EndpointList) {
-	globalAdminPeers = makeAdminPeers(endpoints)
-}
-
-// invokeServiceCmd - Invoke Restart/Stop command.
-func invokeServiceCmd(cp adminPeer, cmd serviceSignal) (err error) {
-	switch cmd {
-	case serviceRestart, serviceStop:
-		err = cp.cmdRunner.SignalService(cmd)
-	}
-	return err
-}
-
-// sendServiceCmd - Invoke Restart command on remote peers
-// adminPeer followed by on the local peer.
-func sendServiceCmd(cps adminPeers, cmd serviceSignal) {
-	// Send service command like stop or restart to all remote nodes and finally run on local node.
-	errs := make([]error, len(cps))
-	var wg sync.WaitGroup
-	remotePeers := cps[1:]
-	for i := range remotePeers {
-		wg.Add(1)
-		go func(idx int) {
-			defer wg.Done()
-			// we use idx+1 because remotePeers slice is 1 position shifted w.r.t cps
-			errs[idx+1] = invokeServiceCmd(remotePeers[idx], cmd)
-		}(i)
-	}
-	wg.Wait()
-	errs[0] = invokeServiceCmd(cps[0], cmd)
-}
-
-// uptimeSlice - used to sort uptimes in chronological order.
-type uptimeSlice []struct {
-	err    error
-	uptime time.Duration
-}
-
-func (ts uptimeSlice) Len() int {
-	return len(ts)
-}
-
-func (ts uptimeSlice) Less(i, j int) bool {
-	return ts[i].uptime < ts[j].uptime
-}
-
-func (ts uptimeSlice) Swap(i, j int) {
-	ts[i], ts[j] = ts[j], ts[i]
-}
-
-// getPeerUptimes - returns the uptime since the last time read quorum
-// was established on success. Otherwise returns errXLReadQuorum.
-func getPeerUptimes(peers adminPeers) (time.Duration, error) {
-	// In a single node Erasure or FS backend setup the uptime of
-	// the setup is the uptime of the single minio server
-	// instance.
-	if !globalIsDistXL {
-		return UTCNow().Sub(globalBootTime), nil
-	}
-
-	uptimes := make(uptimeSlice, len(peers))
-
-	// Get up time of all servers.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			serverInfoData, rpcErr := peer.cmdRunner.ServerInfo()
-			uptimes[idx].uptime, uptimes[idx].err = serverInfoData.Properties.Uptime, rpcErr
-		}(i, peer)
-	}
-	wg.Wait()
-
-	// Sort uptimes in chronological order.
-	sort.Sort(uptimes)
-
-	// Pick the readQuorum'th uptime in chronological order. i.e,
-	// the time at which read quorum was (re-)established.
-	readQuorum := len(uptimes) / 2
-	validCount := 0
-	latestUptime := time.Duration(0)
-	for _, uptime := range uptimes {
-		if uptime.err != nil {
-			logger.LogIf(context.Background(), uptime.err)
-			continue
-		}
-
-		validCount++
-		if validCount >= readQuorum {
-			latestUptime = uptime.uptime
-			break
-		}
-	}
-
-	// Less than readQuorum "Admin.Uptime" RPC call returned
-	// successfully, so read-quorum unavailable.
-	if validCount < readQuorum {
-		return time.Duration(0), InsufficientReadQuorum{}
-	}
-
-	return latestUptime, nil
-}

cmd/admin-rpc-server.go

@@ -1,86 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"path"
-
-	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/logger"
-	xrpc "github.com/minio/minio/cmd/rpc"
-)
-
-const adminServiceName = "Admin"
-const adminServiceSubPath = "/admin"
-
-var adminServicePath = path.Join(minioReservedBucketPath, adminServiceSubPath)
-
-// adminRPCReceiver - Admin RPC receiver for admin RPC server.
-type adminRPCReceiver struct {
-	local *localAdminClient
-}
-
-// SignalServiceArgs - provides the signal argument to SignalService RPC
-type SignalServiceArgs struct {
-	AuthArgs
-	Sig serviceSignal
-}
-
-// SignalService - Send a restart or stop signal to the service
-func (receiver *adminRPCReceiver) SignalService(args *SignalServiceArgs, reply *VoidReply) error {
-	return receiver.local.SignalService(args.Sig)
-}
-
-// ServerInfo - returns the server info when object layer was initialized on this server.
-func (receiver *adminRPCReceiver) ServerInfo(args *AuthArgs, reply *ServerInfoData) (err error) {
-	*reply, err = receiver.local.ServerInfo()
-	return err
-}
-
-// StartProfilingArgs - holds the RPC argument for StartingProfiling RPC call
-type StartProfilingArgs struct {
-	AuthArgs
-	Profiler string
-}
-
-// StartProfiling - starts profiling of this server
-func (receiver *adminRPCReceiver) StartProfiling(args *StartProfilingArgs, reply *VoidReply) error {
-	return receiver.local.StartProfiling(args.Profiler)
-}
-
-// DownloadProfilingData - stops and returns profiling data of this server
-func (receiver *adminRPCReceiver) DownloadProfilingData(args *AuthArgs, reply *[]byte) (err error) {
-	*reply, err = receiver.local.DownloadProfilingData()
-	return
-}
-
-// NewAdminRPCServer - returns new admin RPC server.
-func NewAdminRPCServer() (*xrpc.Server, error) {
-	rpcServer := xrpc.NewServer()
-	if err := rpcServer.RegisterName(adminServiceName, &adminRPCReceiver{&localAdminClient{}}); err != nil {
-		return nil, err
-	}
-	return rpcServer, nil
-}
-
-// registerAdminRPCRouter - creates and registers Admin RPC server and its router.
-func registerAdminRPCRouter(router *mux.Router) {
-	rpcServer, err := NewAdminRPCServer()
-	logger.FatalIf(err, "Unable to initialize Lock RPC Server")
-	subrouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
-	subrouter.Path(adminServiceSubPath).HandlerFunc(httpTraceHdrs(rpcServer.ServeHTTP))
-}

cmd/admin-rpc_test.go

@@ -1,161 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"net/http"
-	"net/http/httptest"
-	"testing"
-	"time"
-
-	xnet "github.com/minio/minio/pkg/net"
-)
-
-///////////////////////////////////////////////////////////////////////////////
-//
-// localAdminClient and AdminRPCClient are adminCmdRunner interface compatible,
-// hence below test functions are available for both clients.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-///////////////////////////////////////////////////////////////////////////////
-//
-// Admin RPC server, adminRPCReceiver and AdminRPCClient are
-// inter-dependent, below test functions are sufficient to test all of them.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-func testAdminCmdRunnerSignalService(t *testing.T, client adminCmdRunner) {
-	defer func(sigChan chan serviceSignal) { globalServiceSignalCh = sigChan }(globalServiceSignalCh)
-	globalServiceSignalCh = make(chan serviceSignal, 10)
-
-	testCases := []struct {
-		signal    serviceSignal
-		expectErr bool
-	}{
-		{serviceRestart, false},
-		{serviceStop, false},
-		{serviceStatus, true},
-		{serviceSignal(100), true},
-	}
-
-	for i, testCase := range testCases {
-		err := client.SignalService(testCase.signal)
-		expectErr := (err != nil)
-
-		if expectErr != testCase.expectErr {
-			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
-		}
-	}
-}
-
-func testAdminCmdRunnerServerInfo(t *testing.T, client adminCmdRunner) {
-	tmpGlobalBootTime := globalBootTime
-	tmpGlobalObjectAPI := globalObjectAPI
-	tmpGlobalConnStats := globalConnStats
-	tmpGlobalHTTPStats := globalHTTPStats
-	tmpGlobalNotificationSys := globalNotificationSys
-	defer func() {
-		globalBootTime = tmpGlobalBootTime
-		globalObjectAPI = tmpGlobalObjectAPI
-		globalConnStats = tmpGlobalConnStats
-		globalHTTPStats = tmpGlobalHTTPStats
-		globalNotificationSys = tmpGlobalNotificationSys
-	}()
-
-	endpoints := new(EndpointList)
-
-	notificationSys := NewNotificationSys(globalServerConfig, *endpoints)
-
-	testCases := []struct {
-		bootTime        time.Time
-		objectAPI       ObjectLayer
-		connStats       *ConnStats
-		httpStats       *HTTPStats
-		notificationSys *NotificationSys
-		expectErr       bool
-	}{
-		{UTCNow(), &DummyObjectLayer{}, newConnStats(), newHTTPStats(), notificationSys, false},
-		{time.Time{}, nil, nil, nil, nil, true},
-		{UTCNow(), nil, nil, nil, nil, true},
-	}
-
-	for i, testCase := range testCases {
-		globalBootTime = testCase.bootTime
-		globalObjectAPI = testCase.objectAPI
-		globalConnStats = testCase.connStats
-		globalHTTPStats = testCase.httpStats
-		globalNotificationSys = testCase.notificationSys
-		_, err := client.ServerInfo()
-		expectErr := (err != nil)
-
-		if expectErr != testCase.expectErr {
-			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
-		}
-	}
-}
-
-func newAdminRPCHTTPServerClient(t *testing.T) (*httptest.Server, *AdminRPCClient, *serverConfig) {
-	rpcServer, err := NewAdminRPCServer()
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		rpcServer.ServeHTTP(w, r)
-	}))
-
-	url, err := xnet.ParseURL(httpServer.URL)
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	host, err := xnet.ParseHost(url.Host)
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	prevGlobalServerConfig := globalServerConfig
-	globalServerConfig = newServerConfig()
-
-	rpcClient, err := NewAdminRPCClient(host)
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	return httpServer, rpcClient, prevGlobalServerConfig
-}
-
-func TestAdminRPCClientSignalService(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
-	defer httpServer.Close()
-	defer func() {
-		globalServerConfig = prevGlobalServerConfig
-	}()
-
-	testAdminCmdRunnerSignalService(t, rpcClient)
-}
-
-func TestAdminRPCClientServerInfo(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
-	defer httpServer.Close()
-	defer func() {
-		globalServerConfig = prevGlobalServerConfig
-	}()
-
-	testAdminCmdRunnerServerInfo(t, rpcClient)
-}

cmd/gateway-main.go

@@ -176,15 +176,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Add API router.
 	registerAPIRouter(router, encryptionEnabled)
 
-	// Dummy endpoint representing gateway instance.
-	globalEndpoints = []Endpoint{{
-		URL:     &url.URL{Path: "/minio/gateway"},
-		IsLocal: true,
-	}}
-
-	// Initialize Admin Peers.
-	initGlobalAdminPeers(globalEndpoints)
-
 	var getCert certs.GetCertificateFunc
 	if globalTLSCerts != nil {
 		getCert = globalTLSCerts.GetCertificate

cmd/globals.go

@@ -162,9 +162,6 @@ var (
 	// File to log HTTP request/response headers and body.
 	globalHTTPTraceFile *os.File
 
-	// List of admin peers.
-	globalAdminPeers = adminPeers{}
-
 	// Minio server user agent string.
 	globalServerUserAgent = "Minio/" + ReleaseTag + " (" + runtime.GOOS + "; " + runtime.GOARCH + ")"
 

cmd/local-admin-client.go

@@ -1,112 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-	"errors"
-	"os"
-
-	"io/ioutil"
-)
-
-// localAdminClient - represents admin operation to be executed locally.
-type localAdminClient struct{}
-
-// SignalService - sends a restart or stop signal to the local server
-func (lc localAdminClient) SignalService(s serviceSignal) error {
-	switch s {
-	case serviceRestart, serviceStop:
-		globalServiceSignalCh <- s
-	default:
-		return errUnsupportedSignal
-	}
-	return nil
-}
-
-// ReInitFormat - re-initialize disk format.
-func (lc localAdminClient) ReInitFormat(dryRun bool) error {
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-	return objectAPI.ReloadFormat(context.Background(), dryRun)
-}
-
-// ServerInfo - Returns the server info of this server.
-func (lc localAdminClient) ServerInfo() (sid ServerInfoData, e error) {
-	if globalBootTime.IsZero() {
-		return sid, errServerNotInitialized
-	}
-
-	// Build storage info
-	objLayer := newObjectLayerFn()
-	if objLayer == nil {
-		return sid, errServerNotInitialized
-	}
-	storage := objLayer.StorageInfo(context.Background())
-
-	return ServerInfoData{
-		StorageInfo: storage,
-		ConnStats:   globalConnStats.toServerConnStats(),
-		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
-		Properties: ServerProperties{
-			Uptime:   UTCNow().Sub(globalBootTime),
-			Version:  Version,
-			CommitID: CommitID,
-			SQSARN:   globalNotificationSys.GetARNList(),
-			Region:   globalServerConfig.GetRegion(),
-		},
-	}, nil
-}
-
-// StartProfiling - starts profiling on the local server.
-func (lc localAdminClient) StartProfiling(profiler string) error {
-	if globalProfiler != nil {
-		globalProfiler.Stop()
-	}
-	prof, err := startProfiler(profiler, "")
-	if err != nil {
-		return err
-	}
-	globalProfiler = prof
-	return nil
-}
-
-// DownloadProfilingData - stops and returns profiling data of the local server.
-func (lc localAdminClient) DownloadProfilingData() ([]byte, error) {
-	if globalProfiler == nil {
-		return nil, errors.New("profiler not enabled")
-	}
-
-	profilerPath := globalProfiler.Path()
-
-	// Stop the profiler
-	globalProfiler.Stop()
-
-	profilerFile, err := os.Open(profilerPath)
-	if err != nil {
-		return nil, err
-	}
-
-	data, err := ioutil.ReadAll(profilerFile)
-	if err != nil {
-		return nil, err
-	}
-
-	return data, nil
-}

cmd/local-admin-client_test.go

@@ -1,29 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"testing"
-)
-
-func TestLocalAdminClientSignalService(t *testing.T) {
-	testAdminCmdRunnerSignalService(t, &localAdminClient{})
-}
-
-func TestLocalAdminClientServerInfo(t *testing.T) {
-	testAdminCmdRunnerServerInfo(t, &localAdminClient{})
-}

cmd/notification.go

@@ -17,11 +17,13 @@
 package cmd
 
 import (
+	"archive/zip"
 	"bytes"
 	"context"
 	"encoding/json"
 	"encoding/xml"
 	"fmt"
+	"io"
 	"net"
 	"net/url"
 	"path"
@@ -90,85 +92,252 @@ func (sys *NotificationSys) DeleteBucket(ctx context.Context, bucketName string)
 	}()
 }
 
-// ReloadFormat - calls ReloadFormat RPC call on all peers.
+// A NotificationGroup is a collection of goroutines working on subtasks that are part of
-func (sys *NotificationSys) ReloadFormat(dryRun bool) map[xnet.Host]error {
+// the same overall task.
-	errors := make(map[xnet.Host]error)
+//
-	var wg sync.WaitGroup
+// A zero NotificationGroup is valid and does not cancel on error.
-	for addr, client := range sys.peerRPCClientMap {
+type NotificationGroup struct {
-		wg.Add(1)
+	wg   sync.WaitGroup
-		go func(addr xnet.Host, client *PeerRPCClient) {
+	errs []NotificationPeerErr
-			defer wg.Done()
+}
-			// Try to load format in three attempts, before giving up.
+
-			for i := 0; i < 3; i++ {
+// WithNPeers returns a new NotificationGroup with length of errs slice upto nerrs,
-				err := client.ReloadFormat(dryRun)
+// upon Wait() errors are returned collected from all tasks.
-				if err == nil {
+func WithNPeers(nerrs int) *NotificationGroup {
-					break
+	return &NotificationGroup{errs: make([]NotificationPeerErr, nerrs)}
+}
+
+// Wait blocks until all function calls from the Go method have returned, then
+// returns the slice of errors from all function calls.
+func (g *NotificationGroup) Wait() []NotificationPeerErr {
+	g.wg.Wait()
+	return g.errs
+}
+
+// Go calls the given function in a new goroutine.
+//
+// The first call to return a non-nil error will be
+// collected in errs slice and returned by Wait().
+func (g *NotificationGroup) Go(ctx context.Context, f func() error, index int, addr xnet.Host) {
+	g.wg.Add(1)
+
+	go func() {
+		defer g.wg.Done()
+		g.errs[index] = NotificationPeerErr{
+			Host: addr,
+		}
+		for i := 0; i < 3; i++ {
+			if err := f(); err != nil {
+				g.errs[index].Err = err
+				// Last iteration log the error.
+				if i == 2 {
+					reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
+					ctx := logger.SetReqInfo(ctx, reqInfo)
+					logger.LogIf(ctx, err)
 				}
-				errors[addr] = err
 				// Wait for one second and no need wait after last attempt.
 				if i < 2 {
 					time.Sleep(1 * time.Second)
 				}
+				continue
 			}
-		}(addr, client)
+			break
-	}
+		}
-	wg.Wait()
+	}()
+}
 
-	return errors
+// ReloadFormat - calls ReloadFormat RPC call on all peers.
+func (sys *NotificationSys) ReloadFormat(dryRun bool) []NotificationPeerErr {
+	var idx = 0
+	ng := WithNPeers(len(sys.peerRPCClientMap))
+	for addr, client := range sys.peerRPCClientMap {
+		client := client
+		ng.Go(context.Background(), func() error {
+			return client.ReloadFormat(dryRun)
+		}, idx, addr)
+		idx++
+	}
+	return ng.Wait()
 }
 
 // LoadUsers - calls LoadUsers RPC call on all peers.
-func (sys *NotificationSys) LoadUsers() map[xnet.Host]error {
+func (sys *NotificationSys) LoadUsers() []NotificationPeerErr {
-	errors := make(map[xnet.Host]error)
+	var idx = 0
-	var wg sync.WaitGroup
+	ng := WithNPeers(len(sys.peerRPCClientMap))
 	for addr, client := range sys.peerRPCClientMap {
-		wg.Add(1)
+		ng.Go(context.Background(), client.LoadUsers, idx, addr)
-		go func(addr xnet.Host, client *PeerRPCClient) {
+		idx++
-			defer wg.Done()
-			// Try to load users in three attempts.
-			for i := 0; i < 3; i++ {
-				err := client.LoadUsers()
-				if err == nil {
-					break
-				}
-				errors[addr] = err
-				// Wait for one second and no need wait after last attempt.
-				if i < 2 {
-					time.Sleep(1 * time.Second)
-				}
-			}
-		}(addr, client)
 	}
-	wg.Wait()
+	return ng.Wait()
-
-	return errors
 }
 
 // LoadCredentials - calls LoadCredentials RPC call on all peers.
-func (sys *NotificationSys) LoadCredentials() map[xnet.Host]error {
+func (sys *NotificationSys) LoadCredentials() []NotificationPeerErr {
-	errors := make(map[xnet.Host]error)
+	var idx = 0
+	ng := WithNPeers(len(sys.peerRPCClientMap))
+	for addr, client := range sys.peerRPCClientMap {
+		ng.Go(context.Background(), client.LoadCredentials, idx, addr)
+		idx++
+	}
+	return ng.Wait()
+}
+
+// StartProfiling - start profiling on remote peers, by initiating a remote RPC.
+func (sys *NotificationSys) StartProfiling(profiler string) []NotificationPeerErr {
+	var idx = 0
+	ng := WithNPeers(len(sys.peerRPCClientMap))
+	for addr, client := range sys.peerRPCClientMap {
+		client := client
+		ng.Go(context.Background(), func() error {
+			return client.StartProfiling(profiler)
+		}, idx, addr)
+		idx++
+	}
+	return ng.Wait()
+}
+
+// DownloadProfilingData - download profiling data from all remote peers.
+func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io.Writer) bool {
+	profilingDataFound := false
+
+	// Initialize a zip writer which will provide a zipped content
+	// of profiling data of all nodes
+	zipWriter := zip.NewWriter(writer)
+	defer zipWriter.Close()
+
+	for addr, client := range sys.peerRPCClientMap {
+		data, err := client.DownloadProfilingData()
+		if err != nil {
+			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
+			ctx := logger.SetReqInfo(ctx, reqInfo)
+			logger.LogIf(ctx, err)
+			continue
+		}
+
+		profilingDataFound = true
+
+		// Send profiling data to zip as file
+		header, zerr := zip.FileInfoHeader(dummyFileInfo{
+			name:    fmt.Sprintf("profiling-%s.pprof", addr),
+			size:    int64(len(data)),
+			mode:    0600,
+			modTime: UTCNow(),
+			isDir:   false,
+			sys:     nil,
+		})
+		if zerr != nil {
+			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
+			ctx := logger.SetReqInfo(ctx, reqInfo)
+			logger.LogIf(ctx, zerr)
+			continue
+		}
+		zwriter, zerr := zipWriter.CreateHeader(header)
+		if zerr != nil {
+			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
+			ctx := logger.SetReqInfo(ctx, reqInfo)
+			logger.LogIf(ctx, zerr)
+			continue
+		}
+		if _, err = io.Copy(zwriter, bytes.NewBuffer(data)); err != nil {
+			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
+			ctx := logger.SetReqInfo(ctx, reqInfo)
+			logger.LogIf(ctx, err)
+			continue
+		}
+	}
+
+	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return profilingDataFound
+	}
+
+	data, err := getProfileData()
+	if err != nil {
+		reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", thisAddr.String())
+		ctx := logger.SetReqInfo(ctx, reqInfo)
+		logger.LogIf(ctx, err)
+		return profilingDataFound
+	}
+
+	profilingDataFound = true
+
+	// Send profiling data to zip as file
+	header, zerr := zip.FileInfoHeader(dummyFileInfo{
+		name:    fmt.Sprintf("profiling-%s.pprof", thisAddr),
+		size:    int64(len(data)),
+		mode:    0600,
+		modTime: UTCNow(),
+		isDir:   false,
+		sys:     nil,
+	})
+
+	zwriter, zerr := zipWriter.CreateHeader(header)
+	if zerr != nil {
+		return profilingDataFound
+	}
+
+	if _, err = io.Copy(zwriter, bytes.NewBuffer(data)); err != nil {
+		return profilingDataFound
+	}
+
+	return profilingDataFound
+}
+
+// SignalService - calls signal service RPC call on all peers.
+func (sys *NotificationSys) SignalService(sig serviceSignal) []NotificationPeerErr {
+	var idx = 0
+	ng := WithNPeers(len(sys.peerRPCClientMap))
+	for addr, client := range sys.peerRPCClientMap {
+		client := client
+		ng.Go(context.Background(), func() error {
+			return client.SignalService(sig)
+		}, idx, addr)
+		idx++
+	}
+	return ng.Wait()
+}
+
+// ServerInfo - calls ServerInfo RPC call on all peers.
+func (sys *NotificationSys) ServerInfo(ctx context.Context) []ServerInfo {
+	var idx = 0
+	serverInfo := make([]ServerInfo, len(sys.peerRPCClientMap))
 	var wg sync.WaitGroup
 	for addr, client := range sys.peerRPCClientMap {
 		wg.Add(1)
-		go func(addr xnet.Host, client *PeerRPCClient) {
+		go func(idx int, addr xnet.Host, client *PeerRPCClient) {
 			defer wg.Done()
-			// Try to load credentials in three attempts.
+			// Try to fetch serverInfo remotely in three attempts.
 			for i := 0; i < 3; i++ {
-				err := client.LoadCredentials()
+				info, err := client.ServerInfo()
 				if err == nil {
-					break
+					serverInfo[idx] = ServerInfo{
+						Addr: addr.String(),
+						Data: &info,
+					}
+					return
+				}
+				serverInfo[idx] = ServerInfo{
+					Addr:  addr.String(),
+					Data:  &info,
+					Error: err.Error(),
+				}
+				// Last iteration log the error.
+				if i == 2 {
+					reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
+					ctx := logger.SetReqInfo(ctx, reqInfo)
+					logger.LogIf(ctx, err)
 				}
-				errors[addr] = err
 				// Wait for one second and no need wait after last attempt.
 				if i < 2 {
 					time.Sleep(1 * time.Second)
 				}
 			}
-		}(addr, client)
+		}(idx, addr, client)
+		idx++
 	}
 	wg.Wait()
-
+	return serverInfo
-	return errors
 }
 
 // SetBucketPolicy - calls SetBucketPolicy RPC call on all peers.

cmd/peer-rpc-client.go

@@ -168,6 +168,38 @@ func (rpcClient *PeerRPCClient) CPULoadInfo() (ServerCPULoadInfo, error) {
 	return reply, err
 }
 
+// StartProfiling - starts profiling on the remote server.
+func (rpcClient *PeerRPCClient) StartProfiling(profiler string) error {
+	args := StartProfilingArgs{Profiler: profiler}
+	reply := VoidReply{}
+	return rpcClient.Call(peerServiceName+".StartProfiling", &args, &reply)
+}
+
+// DownloadProfilingData - download already running profiling on the remote server.
+func (rpcClient *PeerRPCClient) DownloadProfilingData() ([]byte, error) {
+	args := AuthArgs{}
+	var reply []byte
+	err := rpcClient.Call(peerServiceName+".DownloadProfilingData", &args, &reply)
+	return reply, err
+}
+
+// SignalService - calls load server info RPC.
+func (rpcClient *PeerRPCClient) SignalService(sig serviceSignal) error {
+	args := SignalServiceArgs{Sig: sig}
+	reply := VoidReply{}
+
+	return rpcClient.Call(peerServiceName+".SignalService", &args, &reply)
+}
+
+// ServerInfo - calls load server info RPC.
+func (rpcClient *PeerRPCClient) ServerInfo() (ServerInfoData, error) {
+	args := AuthArgs{}
+	reply := ServerInfoData{}
+
+	err := rpcClient.Call(peerServiceName+".ServerInfo", &args, &reply)
+	return reply, err
+}
+
 // NewPeerRPCClient - returns new peer RPC client.
 func NewPeerRPCClient(host *xnet.Host) (*PeerRPCClient, error) {
 	scheme := "http"

cmd/peer-rpc-server.go

@@ -20,6 +20,8 @@ import (
 	"context"
 	"fmt"
 	"path"
+	"sort"
+	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger"
@@ -30,7 +32,7 @@ import (
 )
 
 const peerServiceName = "Peer"
-const peerServiceSubPath = "/s3/remote"
+const peerServiceSubPath = "/peer/remote"
 
 var peerServicePath = path.Join(minioReservedBucketPath, peerServiceSubPath)
 
@@ -118,7 +120,8 @@ type ListenBucketNotificationArgs struct {
 	Addr       xnet.Host      `json:"addr"`
 }
 
-// ListenBucketNotification - handles listen bucket notification RPC call. It creates PeerRPCClient target which pushes requested events to target in remote peer.
+// ListenBucketNotification - handles listen bucket notification RPC call.
+// It creates PeerRPCClient target which pushes requested events to target in remote peer.
 func (receiver *peerRPCReceiver) ListenBucketNotification(args *ListenBucketNotificationArgs, reply *VoidReply) error {
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
@@ -269,6 +272,117 @@ func (receiver *peerRPCReceiver) MemUsageInfo(args *AuthArgs, reply *ServerMemUs
 	return nil
 }
 
+// uptimes - used to sort uptimes in chronological order.
+type uptimes []time.Duration
+
+func (ts uptimes) Len() int {
+	return len(ts)
+}
+
+func (ts uptimes) Less(i, j int) bool {
+	return ts[i] < ts[j]
+}
+
+func (ts uptimes) Swap(i, j int) {
+	ts[i], ts[j] = ts[j], ts[i]
+}
+
+// getPeerUptimes - returns the uptime.
+func getPeerUptimes(serverInfo []ServerInfo) time.Duration {
+	// In a single node Erasure or FS backend setup the uptime of
+	// the setup is the uptime of the single minio server
+	// instance.
+	if !globalIsDistXL {
+		return UTCNow().Sub(globalBootTime)
+	}
+
+	var times []time.Duration
+
+	for _, info := range serverInfo {
+		if info.Error != "" {
+			continue
+		}
+		times = append(times, info.Data.Properties.Uptime)
+	}
+
+	// Sort uptimes in chronological order.
+	sort.Sort(uptimes(times))
+
+	// Return the latest time as the uptime.
+	return times[0]
+}
+
+// StartProfilingArgs - holds the RPC argument for StartingProfiling RPC call
+type StartProfilingArgs struct {
+	AuthArgs
+	Profiler string
+}
+
+// StartProfiling - profiling server receiver.
+func (receiver *peerRPCReceiver) StartProfiling(args *StartProfilingArgs, reply *VoidReply) error {
+	if globalProfiler != nil {
+		globalProfiler.Stop()
+	}
+	var err error
+	globalProfiler, err = startProfiler(args.Profiler, "")
+	return err
+}
+
+// DownloadProfilingData - download profiling data.
+func (receiver *peerRPCReceiver) DownloadProfilingData(args *AuthArgs, reply *[]byte) error {
+	var err error
+	*reply, err = getProfileData()
+	return err
+}
+
+var errUnsupportedSignal = fmt.Errorf("unsupported signal: only restart and stop signals are supported")
+
+// SignalServiceArgs - send event RPC arguments.
+type SignalServiceArgs struct {
+	AuthArgs
+	Sig serviceSignal
+}
+
+// SignalService - signal service receiver.
+func (receiver *peerRPCReceiver) SignalService(args *SignalServiceArgs, reply *VoidReply) error {
+	switch args.Sig {
+	case serviceRestart, serviceStop:
+		globalServiceSignalCh <- args.Sig
+	default:
+		return errUnsupportedSignal
+	}
+	return nil
+}
+
+// ServerInfo - server info receiver.
+func (receiver *peerRPCReceiver) ServerInfo(args *AuthArgs, reply *ServerInfoData) error {
+	if globalBootTime.IsZero() {
+		return errServerNotInitialized
+	}
+
+	// Build storage info
+	objLayer := newObjectLayerFn()
+	if objLayer == nil {
+		return errServerNotInitialized
+	}
+
+	// Server info data.
+	*reply = ServerInfoData{
+		StorageInfo: objLayer.StorageInfo(context.Background()),
+		ConnStats:   globalConnStats.toServerConnStats(),
+		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
+		Properties: ServerProperties{
+			Uptime:   UTCNow().Sub(globalBootTime),
+			Version:  Version,
+			CommitID: CommitID,
+			SQSARN:   globalNotificationSys.GetARNList(),
+			Region:   globalServerConfig.GetRegion(),
+		},
+	}
+
+	return nil
+}
+
 // NewPeerRPCServer - returns new peer RPC server.
 func NewPeerRPCServer() (*xrpc.Server, error) {
 	rpcServer := xrpc.NewServer()

cmd/routers.go

@@ -41,7 +41,7 @@ func registerDistXLRouters(router *mux.Router, endpoints EndpointList) {
 	// Register distributed namespace lock.
 	registerDistNSLockRouter(router)
 
-	// Register S3 peer communication router.
+	// Register peer communication router.
 	registerPeerRPCRouter(router)
 }
 
@@ -104,9 +104,6 @@ func configureServerHandler(endpoints EndpointList) (http.Handler, error) {
 	// Add STS router always.
 	registerSTSRouter(router)
 
-	// Add Admin RPC router
-	registerAdminRPCRouter(router)
-
 	// Add Admin router, all APIs are enabled in server mode.
 	registerAdminRouter(router, true)
 

cmd/server-main.go

@@ -293,9 +293,6 @@ func serverMain(ctx *cli.Context) {
 		logger.Fatal(uiErrUnexpectedError(err), "Unable to configure one of server's RPC services")
 	}
 
-	// Initialize Admin Peers inter-node communication only in distributed setup.
-	initGlobalAdminPeers(globalEndpoints)
-
 	var getCert certs.GetCertificateFunc
 	if globalTLSCerts != nil {
 		getCert = globalTLSCerts.GetCertificate

cmd/sts-handlers.go

@@ -123,10 +123,10 @@ func (sts *stsAPIHandlers) AssumeRoleWithWebIdentity(w http.ResponseWriter, r *h
 	}
 
 	// Notify all other Minio peers to reload temp users
-	for host, err := range globalNotificationSys.LoadUsers() {
+	for _, nerr := range globalNotificationSys.LoadUsers() {
-		if err != nil {
+		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, err)
+			logger.LogIf(ctx, nerr.Err)
 		}
 	}
 
@@ -207,10 +207,10 @@ func (sts *stsAPIHandlers) AssumeRoleWithClientGrants(w http.ResponseWriter, r *
 	}
 
 	// Notify all other Minio peers to reload temp users
-	for host, err := range globalNotificationSys.LoadUsers() {
+	for _, nerr := range globalNotificationSys.LoadUsers() {
-		if err != nil {
+		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, err)
+			logger.LogIf(ctx, nerr.Err)
 		}
 	}
 

cmd/utils.go

@@ -208,6 +208,26 @@ func (p profilerWrapper) Path() string {
 	return p.pathFn()
 }
 
+// Returns current profile data, returns error if there is no active
+// profiling in progress. Stops an active profile.
+func getProfileData() ([]byte, error) {
+	if globalProfiler == nil {
+		return nil, errors.New("profiler not enabled")
+	}
+
+	profilerPath := globalProfiler.Path()
+
+	// Stop the profiler
+	globalProfiler.Stop()
+
+	profilerFile, err := os.Open(profilerPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return ioutil.ReadAll(profilerFile)
+}
+
 // Starts a profiler returns nil if profiler is not enabled, caller needs to handle this.
 func startProfiler(profilerType, dirPath string) (interface {
 	Stop()

cmd/web-handlers.go

@@ -752,10 +752,10 @@ func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *Se
 
 	if errs := globalNotificationSys.LoadCredentials(); len(errs) != 0 {
 		reply.PeerErrMsgs = make(map[string]string)
-		for host, err := range errs {
+		for _, nerr := range errs {
-			err = fmt.Errorf("Unable to update credentials on server %v: %v", host, err)
+			err = fmt.Errorf("Unable to update credentials on server %v: %v", nerr.Host, nerr.Err)
 			logger.LogIf(context.Background(), err)
-			reply.PeerErrMsgs[host.String()] = err.Error()
+			reply.PeerErrMsgs[nerr.Host.String()] = err.Error()
 		}
 	} else {
 		reply.Token, err = authenticateWeb(creds.AccessKey, creds.SecretKey)