mirror of
https://github.com/minio/minio.git
synced 2025-11-09 05:34:56 -05:00
Migrate all Peer communication to common Notification subsystem (#7031)
Deprecate the use of Admin Peers concept and migrate all peer communication to Notification subsystem. This finally allows for a common subsystem for all peer notification in case of distributed server deployments.
This commit is contained in:
Harshavardhana
GitHub
@@ -17,11 +17,13 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/url"
|
||||
"path"
|
||||
@@ -90,85 +92,252 @@ func (sys *NotificationSys) DeleteBucket(ctx context.Context, bucketName string)
|
||||
}()
|
||||
}
|
||||
|
||||
// ReloadFormat - calls ReloadFormat RPC call on all peers.
|
||||
func (sys *NotificationSys) ReloadFormat(dryRun bool) map[xnet.Host]error {
|
||||
errors := make(map[xnet.Host]error)
|
||||
var wg sync.WaitGroup
|
||||
for addr, client := range sys.peerRPCClientMap {
|
||||
wg.Add(1)
|
||||
go func(addr xnet.Host, client *PeerRPCClient) {
|
||||
defer wg.Done()
|
||||
// Try to load format in three attempts, before giving up.
|
||||
for i := 0; i < 3; i++ {
|
||||
err := client.ReloadFormat(dryRun)
|
||||
if err == nil {
|
||||
break
|
||||
// A NotificationGroup is a collection of goroutines working on subtasks that are part of
|
||||
// the same overall task.
|
||||
//
|
||||
// A zero NotificationGroup is valid and does not cancel on error.
|
||||
type NotificationGroup struct {
|
||||
wg sync.WaitGroup
|
||||
errs []NotificationPeerErr
|
||||
}
|
||||
|
||||
// WithNPeers returns a new NotificationGroup with length of errs slice upto nerrs,
|
||||
// upon Wait() errors are returned collected from all tasks.
|
||||
func WithNPeers(nerrs int) *NotificationGroup {
|
||||
return &NotificationGroup{errs: make([]NotificationPeerErr, nerrs)}
|
||||
}
|
||||
|
||||
// Wait blocks until all function calls from the Go method have returned, then
|
||||
// returns the slice of errors from all function calls.
|
||||
func (g *NotificationGroup) Wait() []NotificationPeerErr {
|
||||
g.wg.Wait()
|
||||
return g.errs
|
||||
}
|
||||
|
||||
// Go calls the given function in a new goroutine.
|
||||
//
|
||||
// The first call to return a non-nil error will be
|
||||
// collected in errs slice and returned by Wait().
|
||||
func (g *NotificationGroup) Go(ctx context.Context, f func() error, index int, addr xnet.Host) {
|
||||
g.wg.Add(1)
|
||||
|
||||
go func() {
|
||||
defer g.wg.Done()
|
||||
g.errs[index] = NotificationPeerErr{
|
||||
Host: addr,
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := f(); err != nil {
|
||||
g.errs[index].Err = err
|
||||
// Last iteration log the error.
|
||||
if i == 2 {
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
|
||||
ctx := logger.SetReqInfo(ctx, reqInfo)
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
errors[addr] = err
|
||||
// Wait for one second and no need wait after last attempt.
|
||||
if i < 2 {
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
continue
|
||||
}
|
||||
}(addr, client)
|
||||
}
|
||||
wg.Wait()
|
||||
break
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
return errors
|
||||
// ReloadFormat - calls ReloadFormat RPC call on all peers.
|
||||
func (sys *NotificationSys) ReloadFormat(dryRun bool) []NotificationPeerErr {
|
||||
var idx = 0
|
||||
ng := WithNPeers(len(sys.peerRPCClientMap))
|
||||
for addr, client := range sys.peerRPCClientMap {
|
||||
client := client
|
||||
ng.Go(context.Background(), func() error {
|
||||
return client.ReloadFormat(dryRun)
|
||||
}, idx, addr)
|
||||
idx++
|
||||
}
|
||||
return ng.Wait()
|
||||
}
|
||||
|
||||
// LoadUsers - calls LoadUsers RPC call on all peers.
|
||||
func (sys *NotificationSys) LoadUsers() map[xnet.Host]error {
|
||||
errors := make(map[xnet.Host]error)
|
||||
var wg sync.WaitGroup
|
||||
func (sys *NotificationSys) LoadUsers() []NotificationPeerErr {
|
||||
var idx = 0
|
||||
ng := WithNPeers(len(sys.peerRPCClientMap))
|
||||
for addr, client := range sys.peerRPCClientMap {
|
||||
wg.Add(1)
|
||||
go func(addr xnet.Host, client *PeerRPCClient) {
|
||||
defer wg.Done()
|
||||
// Try to load users in three attempts.
|
||||
for i := 0; i < 3; i++ {
|
||||
err := client.LoadUsers()
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
errors[addr] = err
|
||||
// Wait for one second and no need wait after last attempt.
|
||||
if i < 2 {
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
}
|
||||
}(addr, client)
|
||||
ng.Go(context.Background(), client.LoadUsers, idx, addr)
|
||||
idx++
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
return errors
|
||||
return ng.Wait()
|
||||
}
|
||||
|
||||
// LoadCredentials - calls LoadCredentials RPC call on all peers.
|
||||
func (sys *NotificationSys) LoadCredentials() map[xnet.Host]error {
|
||||
errors := make(map[xnet.Host]error)
|
||||
func (sys *NotificationSys) LoadCredentials() []NotificationPeerErr {
|
||||
var idx = 0
|
||||
ng := WithNPeers(len(sys.peerRPCClientMap))
|
||||
for addr, client := range sys.peerRPCClientMap {
|
||||
ng.Go(context.Background(), client.LoadCredentials, idx, addr)
|
||||
idx++
|
||||
}
|
||||
return ng.Wait()
|
||||
}
|
||||
|
||||
// StartProfiling - start profiling on remote peers, by initiating a remote RPC.
|
||||
func (sys *NotificationSys) StartProfiling(profiler string) []NotificationPeerErr {
|
||||
var idx = 0
|
||||
ng := WithNPeers(len(sys.peerRPCClientMap))
|
||||
for addr, client := range sys.peerRPCClientMap {
|
||||
client := client
|
||||
ng.Go(context.Background(), func() error {
|
||||
return client.StartProfiling(profiler)
|
||||
}, idx, addr)
|
||||
idx++
|
||||
}
|
||||
return ng.Wait()
|
||||
}
|
||||
|
||||
// DownloadProfilingData - download profiling data from all remote peers.
|
||||
func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io.Writer) bool {
|
||||
profilingDataFound := false
|
||||
|
||||
// Initialize a zip writer which will provide a zipped content
|
||||
// of profiling data of all nodes
|
||||
zipWriter := zip.NewWriter(writer)
|
||||
defer zipWriter.Close()
|
||||
|
||||
for addr, client := range sys.peerRPCClientMap {
|
||||
data, err := client.DownloadProfilingData()
|
||||
if err != nil {
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
|
||||
ctx := logger.SetReqInfo(ctx, reqInfo)
|
||||
logger.LogIf(ctx, err)
|
||||
continue
|
||||
}
|
||||
|
||||
profilingDataFound = true
|
||||
|
||||
// Send profiling data to zip as file
|
||||
header, zerr := zip.FileInfoHeader(dummyFileInfo{
|
||||
name: fmt.Sprintf("profiling-%s.pprof", addr),
|
||||
size: int64(len(data)),
|
||||
mode: 0600,
|
||||
modTime: UTCNow(),
|
||||
isDir: false,
|
||||
sys: nil,
|
||||
})
|
||||
if zerr != nil {
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
|
||||
ctx := logger.SetReqInfo(ctx, reqInfo)
|
||||
logger.LogIf(ctx, zerr)
|
||||
continue
|
||||
}
|
||||
zwriter, zerr := zipWriter.CreateHeader(header)
|
||||
if zerr != nil {
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
|
||||
ctx := logger.SetReqInfo(ctx, reqInfo)
|
||||
logger.LogIf(ctx, zerr)
|
||||
continue
|
||||
}
|
||||
if _, err = io.Copy(zwriter, bytes.NewBuffer(data)); err != nil {
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
|
||||
ctx := logger.SetReqInfo(ctx, reqInfo)
|
||||
logger.LogIf(ctx, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return profilingDataFound
|
||||
}
|
||||
|
||||
data, err := getProfileData()
|
||||
if err != nil {
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", thisAddr.String())
|
||||
ctx := logger.SetReqInfo(ctx, reqInfo)
|
||||
logger.LogIf(ctx, err)
|
||||
return profilingDataFound
|
||||
}
|
||||
|
||||
profilingDataFound = true
|
||||
|
||||
// Send profiling data to zip as file
|
||||
header, zerr := zip.FileInfoHeader(dummyFileInfo{
|
||||
name: fmt.Sprintf("profiling-%s.pprof", thisAddr),
|
||||
size: int64(len(data)),
|
||||
mode: 0600,
|
||||
modTime: UTCNow(),
|
||||
isDir: false,
|
||||
sys: nil,
|
||||
})
|
||||
|
||||
zwriter, zerr := zipWriter.CreateHeader(header)
|
||||
if zerr != nil {
|
||||
return profilingDataFound
|
||||
}
|
||||
|
||||
if _, err = io.Copy(zwriter, bytes.NewBuffer(data)); err != nil {
|
||||
return profilingDataFound
|
||||
}
|
||||
|
||||
return profilingDataFound
|
||||
}
|
||||
|
||||
// SignalService - calls signal service RPC call on all peers.
|
||||
func (sys *NotificationSys) SignalService(sig serviceSignal) []NotificationPeerErr {
|
||||
var idx = 0
|
||||
ng := WithNPeers(len(sys.peerRPCClientMap))
|
||||
for addr, client := range sys.peerRPCClientMap {
|
||||
client := client
|
||||
ng.Go(context.Background(), func() error {
|
||||
return client.SignalService(sig)
|
||||
}, idx, addr)
|
||||
idx++
|
||||
}
|
||||
return ng.Wait()
|
||||
}
|
||||
|
||||
// ServerInfo - calls ServerInfo RPC call on all peers.
|
||||
func (sys *NotificationSys) ServerInfo(ctx context.Context) []ServerInfo {
|
||||
var idx = 0
|
||||
serverInfo := make([]ServerInfo, len(sys.peerRPCClientMap))
|
||||
var wg sync.WaitGroup
|
||||
for addr, client := range sys.peerRPCClientMap {
|
||||
wg.Add(1)
|
||||
go func(addr xnet.Host, client *PeerRPCClient) {
|
||||
go func(idx int, addr xnet.Host, client *PeerRPCClient) {
|
||||
defer wg.Done()
|
||||
// Try to load credentials in three attempts.
|
||||
// Try to fetch serverInfo remotely in three attempts.
|
||||
for i := 0; i < 3; i++ {
|
||||
err := client.LoadCredentials()
|
||||
info, err := client.ServerInfo()
|
||||
if err == nil {
|
||||
break
|
||||
serverInfo[idx] = ServerInfo{
|
||||
Addr: addr.String(),
|
||||
Data: &info,
|
||||
}
|
||||
return
|
||||
}
|
||||
serverInfo[idx] = ServerInfo{
|
||||
Addr: addr.String(),
|
||||
Data: &info,
|
||||
Error: err.Error(),
|
||||
}
|
||||
// Last iteration log the error.
|
||||
if i == 2 {
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
|
||||
ctx := logger.SetReqInfo(ctx, reqInfo)
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
errors[addr] = err
|
||||
// Wait for one second and no need wait after last attempt.
|
||||
if i < 2 {
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
}
|
||||
}(addr, client)
|
||||
}(idx, addr, client)
|
||||
idx++
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
return errors
|
||||
return serverInfo
|
||||
}
|
||||
|
||||
// SetBucketPolicy - calls SetBucketPolicy RPC call on all peers.
|
||||
|
||||
Reference in New Issue
Block a user