Refresh tier config periodically (#19049)

- Increase the parity for tier-config.bin object
- Refresh globalTierConfigMgr cached value once every 15 mins

cmd/tier-handlers.go

@@ -140,6 +140,7 @@ func (api adminAPIHandlers) ListTierHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
+	w.Header().Set(tierCfgRefreshAtHdr, globalTierConfigMgr.refreshedAt().String())
 	writeSuccessResponseJSON(w, data)
 }
 

cmd/tier.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2023 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -23,6 +23,7 @@ import (
 	"encoding/base64"
 	"encoding/binary"
 	"fmt"
+	"math/rand"
 	"net/http"
 	"path"
 	"strings"
@@ -33,6 +34,7 @@ import (
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/kms"
+	"github.com/minio/minio/internal/logger"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -74,12 +76,15 @@ const (
 // tierConfigPath refers to remote tier config object name
 var tierConfigPath = path.Join(minioConfigPrefix, tierConfigFile)
 
+const tierCfgRefreshAtHdr = "X-MinIO-TierCfg-RefreshedAt"
+
 // TierConfigMgr holds the collection of remote tiers configured in this deployment.
 type TierConfigMgr struct {
 	sync.RWMutex `msg:"-"`
 	drivercache  map[string]WarmBackend `msg:"-"`
 
-	Tiers map[string]madmin.TierConfig `json:"tiers"`
+	Tiers           map[string]madmin.TierConfig `json:"tiers"`
+	lastRefreshedAt time.Time                    `msg:"-"`
 }
 
 type tierMetrics struct {
@@ -172,6 +177,12 @@ func (t *tierMetrics) Report() []Metric {
 	return metrics
 }
 
+func (config *TierConfigMgr) refreshedAt() time.Time {
+	config.RLock()
+	defer config.RUnlock()
+	return config.lastRefreshedAt
+}
+
 // IsTierValid returns true if there exists a remote tier by name tierName,
 // otherwise returns false.
 func (config *TierConfigMgr) IsTierValid(tierName string) bool {
@@ -413,7 +424,7 @@ func (config *TierConfigMgr) configReader(ctx context.Context) (*PutObjReader, *
 		return nil, nil, err
 	}
 	if GlobalKMS == nil {
-		return NewPutObjReader(hr), &ObjectOptions{}, nil
+		return NewPutObjReader(hr), &ObjectOptions{MaxParity: true}, nil
 	}
 
 	// Note: Local variables with names ek, oek, etc are named inline with
@@ -443,6 +454,7 @@ func (config *TierConfigMgr) configReader(ctx context.Context) (*PutObjReader, *
 	opts := &ObjectOptions{
 		UserDefined: metadata,
 		MTime:       UTCNow(),
+		MaxParity:   true,
 	}
 
 	return pReader, opts, nil
@@ -455,6 +467,9 @@ func (config *TierConfigMgr) Reload(ctx context.Context, objAPI ObjectLayer) err
 	case nil:
 		break
 	case errConfigNotFound: // nothing to reload
+		// To maintain the invariance that lastRefreshedAt records the
+		// timestamp of last successful refresh
+		config.lastRefreshedAt = UTCNow()
 		return nil
 	default:
 		return err
@@ -474,7 +489,7 @@ func (config *TierConfigMgr) Reload(ctx context.Context, objAPI ObjectLayer) err
 	for tier, cfg := range newConfig.Tiers {
 		config.Tiers[tier] = cfg
 	}
-
+	config.lastRefreshedAt = UTCNow()
 	return nil
 }
 
@@ -501,6 +516,31 @@ func NewTierConfigMgr() *TierConfigMgr {
 	}
 }
 
+func (config *TierConfigMgr) refreshTierConfig(ctx context.Context, objAPI ObjectLayer) {
+	const tierCfgRefresh = 15 * time.Minute
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	randInterval := func() time.Duration {
+		return time.Duration(r.Float64() * 5 * float64(time.Second))
+	}
+
+	// To avoid all MinIO nodes reading the tier config object at the same
+	// time.
+	t := time.NewTimer(tierCfgRefresh + randInterval())
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			err := config.Reload(ctx, objAPI)
+			if err != nil {
+				logger.LogIf(ctx, err)
+			}
+		}
+		t.Reset(tierCfgRefresh + randInterval())
+	}
+}
+
 // loadTierConfig loads remote tier configuration from objAPI.
 func loadTierConfig(ctx context.Context, objAPI ObjectLayer) (*TierConfigMgr, error) {
 	if objAPI == nil {
@@ -536,19 +576,11 @@ func loadTierConfig(ctx context.Context, objAPI ObjectLayer) (*TierConfigMgr, er
 	return cfg, nil
 }
 
-// Reset clears remote tier configured and clears tier driver cache.
-func (config *TierConfigMgr) Reset() {
-	config.Lock()
-	for k := range config.drivercache {
-		delete(config.drivercache, k)
-	}
-	for k := range config.Tiers {
-		delete(config.Tiers, k)
-	}
-	config.Unlock()
-}
-
 // Init initializes tier configuration reading from objAPI
 func (config *TierConfigMgr) Init(ctx context.Context, objAPI ObjectLayer) error {
-	return config.Reload(ctx, objAPI)
+	err := config.Reload(ctx, objAPI)
+	if globalIsDistErasure {
+		go config.refreshTierConfig(ctx, objAPI)
+	}
+	return err
 }