From 48b590e14b3b9889095c2262a2d68c85afd62176 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 7 Mar 2024 10:24:07 -0800 Subject: [PATCH] fix: same server to be part of multiple pools (#19216) our PoolNumber calculation was costly, while we already had this information per endpoint, we needed to deduce it appropriately. This PR addresses this by assigning PoolNumbers field that carries all the pool numbers that belong to a server. properties.PoolNumber still carries a valid value only when len(properties.PoolNumbers) == 1, otherwise properties.PoolNumber is set to math.MaxInt (indicating that this value is undefined) and then one must rely on properties.PoolNumbers for server participation in multiple pools. addresses the issue originating from #11327 --- cmd/admin-handlers.go | 40 ++++++++++++++-------------------------- cmd/admin-server-info.go | 17 +++++++++++++++++ go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 34 insertions(+), 29 deletions(-) diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index e31037d1f..d4db5c2a3 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -2280,8 +2280,6 @@ func getServerInfo(ctx context.Context, pools, metrics bool, r *http.Request) ma servers := globalNotificationSys.ServerInfo(metrics) servers = append(servers, local) - assignPoolNumbers(servers) - var poolsInfo map[int]map[int]madmin.ErasureSetInfo var backend madmin.ErasureBackend @@ -2718,14 +2716,20 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur for _, server := range infoMessage.Servers { anonEndpoint := anonAddr(server.Endpoint) servers = append(servers, madmin.ServerInfo{ - State: server.State, - Endpoint: anonEndpoint, - Uptime: server.Uptime, - Version: server.Version, - CommitID: server.CommitID, - Network: anonymizeNetwork(server.Network), - Drives: anonymizeDrives(server.Disks), - PoolNumber: server.PoolNumber, + State: server.State, + Endpoint: anonEndpoint, + Uptime: server.Uptime, + Version: server.Version, + CommitID: server.CommitID, + Network: anonymizeNetwork(server.Network), + Drives: anonymizeDrives(server.Disks), + PoolNumber: func() int { + if len(server.PoolNumbers) == 1 { + return server.PoolNumbers[0] + } + return math.MaxInt // this indicates that its unset. + }(), + PoolNumbers: server.PoolNumbers, MemStats: madmin.MemStats{ Alloc: server.MemStats.Alloc, TotalAlloc: server.MemStats.TotalAlloc, @@ -2915,22 +2919,6 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque writeSuccessResponseJSON(w, jsonBytes) } -func assignPoolNumbers(servers []madmin.ServerProperties) { - for i := range servers { - for idx, ge := range globalEndpoints { - for _, endpoint := range ge.Endpoints { - if servers[i].Endpoint == endpoint.Host { - servers[i].PoolNumber = idx + 1 - } else if host, err := xnet.ParseHost(servers[i].Endpoint); err == nil { - if host.Name == endpoint.Hostname() { - servers[i].PoolNumber = idx + 1 - } - } - } - } - } -} - func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus { lambdaMap := make(map[string][]madmin.TargetIDStatus) diff --git a/cmd/admin-server-info.go b/cmd/admin-server-info.go index 291bc6158..7e57cb652 100644 --- a/cmd/admin-server-info.go +++ b/cmd/admin-server-info.go @@ -19,10 +19,12 @@ package cmd import ( "context" + "math" "net/http" "os" "runtime" "runtime/debug" + "sort" "strings" "time" @@ -42,9 +44,13 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req if globalIsDistErasure { addr = globalLocalNodeName } + poolNumbers := make(map[int]struct{}) network := make(map[string]string) for _, ep := range endpointServerPools { for _, endpoint := range ep.Endpoints { + if endpoint.IsLocal { + poolNumbers[endpoint.PoolIdx+1] = struct{}{} + } nodeName := endpoint.Host if nodeName == "" { nodeName = addr @@ -112,6 +118,17 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req MinioEnvVars: make(map[string]string, 10), } + for poolNumber := range poolNumbers { + props.PoolNumbers = append(props.PoolNumbers, poolNumber) + } + sort.Ints(props.PoolNumbers) + props.PoolNumber = func() int { + if len(props.PoolNumbers) == 1 { + return props.PoolNumbers[0] + } + return math.MaxInt // this indicates that its unset. + }() + sensitive := map[string]struct{}{ config.EnvAccessKey: {}, config.EnvSecretKey: {}, diff --git a/go.mod b/go.mod index bebd03da1..8f3961e2b 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/minio/dperf v0.5.3 github.com/minio/highwayhash v1.0.2 github.com/minio/kms-go/kes v0.3.0 - github.com/minio/madmin-go/v3 v3.0.49 + github.com/minio/madmin-go/v3 v3.0.50-0.20240307080957-112c599cb563 github.com/minio/minio-go/v7 v7.0.68 github.com/minio/mux v1.9.0 github.com/minio/pkg/v2 v2.0.9-0.20240209124402-7990a27fd79d diff --git a/go.sum b/go.sum index 332c6c56b..e5822752d 100644 --- a/go.sum +++ b/go.sum @@ -446,8 +446,8 @@ github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/kms-go/kes v0.3.0 h1:SU8VGVM/Hk9w1OiSby3OatkcojooUqIdDHl6dtM6NkY= github.com/minio/kms-go/kes v0.3.0/go.mod h1:w6DeVT878qEOU3nUrYVy1WOT5H1Ig9hbDIh698NYJKY= -github.com/minio/madmin-go/v3 v3.0.49 h1:Ag5eyYUf9K1MvW9hiErEJhGfqlf//pOtlhdoepb9AwY= -github.com/minio/madmin-go/v3 v3.0.49/go.mod h1:ZDF7kf5fhmxLhbGTqyq5efs4ao0v4eWf7nOuef/ljJs= +github.com/minio/madmin-go/v3 v3.0.50-0.20240307080957-112c599cb563 h1:Sewy11CUNITOocZ2KUfW0l2zm0wNp+d8tb22ivztGRg= +github.com/minio/madmin-go/v3 v3.0.50-0.20240307080957-112c599cb563/go.mod h1:ZDF7kf5fhmxLhbGTqyq5efs4ao0v4eWf7nOuef/ljJs= github.com/minio/mc v0.0.0-20240209221824-669cb0a9a475 h1:yfLzMougcV2xkVlWgwYwVRoT8pnXrcCV4oOQW+pI2EQ= github.com/minio/mc v0.0.0-20240209221824-669cb0a9a475/go.mod h1:MmDLdb7NWd/OYhcKcXKvwErq2GNa/Zq6xtTWuhdC4II= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=