Add admin info timeouts (#20249)

Since a lot of operations load from storage, do remote calls, add a 10 second timeout to each operation.

This should make `mc admin info` return values even under extreme conditions.
This commit is contained in:
Klaus Post 2024-08-12 10:24:29 -07:00 committed by GitHub
parent d8f0e0ea6e
commit 53eb7656de
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 14 additions and 3 deletions

View File

@ -2331,6 +2331,7 @@ func getPoolsInfo(ctx context.Context, allDisks []madmin.Disk) (map[int]map[int]
}
func getServerInfo(ctx context.Context, pools, metrics bool, r *http.Request) madmin.InfoMessage {
const operationTimeout = 10 * time.Second
ldap := madmin.LDAP{}
if globalIAMSys.LDAPConfig.Enabled() {
ldapConn, err := globalIAMSys.LDAPConfig.LDAP.Connect()
@ -2371,7 +2372,9 @@ func getServerInfo(ctx context.Context, pools, metrics bool, r *http.Request) ma
mode = madmin.ItemOnline
// Load data usage
dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
ctx2, cancel := context.WithTimeout(ctx, operationTimeout)
dataUsageInfo, err := loadDataUsageFromBackend(ctx2, objectAPI)
cancel()
if err == nil {
buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount}
objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount}
@ -2405,18 +2408,24 @@ func getServerInfo(ctx context.Context, pools, metrics bool, r *http.Request) ma
}
if pools {
poolsInfo, _ = getPoolsInfo(ctx, allDisks)
ctx2, cancel := context.WithTimeout(ctx, operationTimeout)
poolsInfo, _ = getPoolsInfo(ctx2, allDisks)
cancel()
}
}
domain := globalDomainNames
services := madmin.Services{
KMSStatus: fetchKMSStatus(ctx),
LDAP: ldap,
Logger: log,
Audit: audit,
Notifications: notifyTarget,
}
{
ctx2, cancel := context.WithTimeout(ctx, operationTimeout)
services.KMSStatus = fetchKMSStatus(ctx2)
cancel()
}
return madmin.InfoMessage{
Mode: string(mode),

View File

@ -1099,6 +1099,8 @@ func (sys *NotificationSys) ServerInfo(ctx context.Context, metrics bool) []madm
wg.Add(1)
go func(client *peerRESTClient, idx int) {
defer wg.Done()
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
info, err := client.ServerInfo(ctx, metrics)
if err != nil {
info.Endpoint = client.host.String()