diff --git a/cmd/bootstrap-messages.go b/cmd/bootstrap-messages.go
index 7df3b956a..39d82e5f0 100644
--- a/cmd/bootstrap-messages.go
+++ b/cmd/bootstrap-messages.go
@@ -19,98 +19,51 @@ package cmd
import (
"context"
- "fmt"
"sync"
- "time"
"github.com/minio/madmin-go/v3"
"github.com/minio/minio/internal/pubsub"
)
-const bootstrapMsgsLimit = 4 << 10
+const bootstrapTraceLimit = 4 << 10
-type bootstrapInfo struct {
- msg string
- ts time.Time
- source string
-}
type bootstrapTracer struct {
- mu sync.RWMutex
- idx int
- info [bootstrapMsgsLimit]bootstrapInfo
- lastUpdate time.Time
+ mu sync.RWMutex
+ info []madmin.TraceInfo
}
var globalBootstrapTracer = &bootstrapTracer{}
-func (bs *bootstrapTracer) DropEvents() {
+func (bs *bootstrapTracer) Record(info madmin.TraceInfo) {
bs.mu.Lock()
defer bs.mu.Unlock()
- if time.Now().UTC().Sub(bs.lastUpdate) > 24*time.Hour {
- bs.info = [4096]bootstrapInfo{}
- bs.idx = 0
+ if len(bs.info) > bootstrapTraceLimit {
+ return
}
-}
-
-func (bs *bootstrapTracer) Empty() bool {
- var empty bool
- bs.mu.RLock()
- empty = bs.info[0].msg == ""
- bs.mu.RUnlock()
-
- return empty
-}
-
-func (bs *bootstrapTracer) Record(msg string, skip int) {
- source := getSource(skip + 1)
- bs.mu.Lock()
- now := time.Now().UTC()
- bs.info[bs.idx] = bootstrapInfo{
- msg: msg,
- ts: now,
- source: source,
- }
- bs.lastUpdate = now
- bs.idx = (bs.idx + 1) % bootstrapMsgsLimit
- bs.mu.Unlock()
+ bs.info = append(bs.info, info)
}
func (bs *bootstrapTracer) Events() []madmin.TraceInfo {
- traceInfo := make([]madmin.TraceInfo, 0, bootstrapMsgsLimit)
-
- // Add all messages in order
- addAll := func(info []bootstrapInfo) {
- for _, msg := range info {
- if msg.ts.IsZero() {
- continue // skip empty events
- }
- traceInfo = append(traceInfo, madmin.TraceInfo{
- TraceType: madmin.TraceBootstrap,
- Time: msg.ts,
- NodeName: globalLocalNodeName,
- FuncName: "BOOTSTRAP",
- Message: fmt.Sprintf("%s %s", msg.source, msg.msg),
- })
- }
- }
+ traceInfo := make([]madmin.TraceInfo, 0, bootstrapTraceLimit)
bs.mu.RLock()
- addAll(bs.info[bs.idx:])
- addAll(bs.info[:bs.idx])
+ for _, i := range bs.info {
+ traceInfo = append(traceInfo, i)
+ }
bs.mu.RUnlock()
+
return traceInfo
}
func (bs *bootstrapTracer) Publish(ctx context.Context, trace *pubsub.PubSub[madmin.TraceInfo, madmin.TraceType]) {
- if bs.Empty() {
- return
- }
for _, bsEvent := range bs.Events() {
- select {
- case <-ctx.Done():
- default:
- trace.Publish(bsEvent)
+ if bsEvent.Message != "" {
+ select {
+ case <-ctx.Done():
+ default:
+ trace.Publish(bsEvent)
+ }
}
}
}
diff --git a/cmd/bootstrap-messages_test.go b/cmd/bootstrap-messages_test.go
deleted file mode 100644
index dbe2c59ce..000000000
--- a/cmd/bootstrap-messages_test.go
+++ /dev/null
@@ -1,60 +0,0 @@
-// Copyright (c) 2015-2023 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see .
-
-package cmd
-
-import (
- "fmt"
- "strings"
- "testing"
- "time"
-)
-
-func TestBootstrap(t *testing.T) {
- // Bootstrap events exceed bootstrap messages limit
- bsTracer := &bootstrapTracer{}
- for i := 0; i < bootstrapMsgsLimit+10; i++ {
- bsTracer.Record(fmt.Sprintf("msg-%d", i), 1)
- }
-
- traceInfos := bsTracer.Events()
- if len(traceInfos) != bootstrapMsgsLimit {
- t.Fatalf("Expected length of events %d but got %d", bootstrapMsgsLimit, len(traceInfos))
- }
-
- // Simulate the case where bootstrap events were updated a day ago
- bsTracer.lastUpdate = time.Now().UTC().Add(-25 * time.Hour)
- bsTracer.DropEvents()
- if !bsTracer.Empty() {
- t.Fatalf("Expected all bootstrap events to have been dropped, but found %d events", len(bsTracer.Events()))
- }
-
- // Fewer than 4K bootstrap events
- for i := 0; i < 10; i++ {
- bsTracer.Record(fmt.Sprintf("msg-%d", i), 1)
- }
- events := bsTracer.Events()
- if len(events) != 10 {
- t.Fatalf("Expected length of events %d but got %d", 10, len(events))
- }
- for i, traceInfo := range bsTracer.Events() {
- msg := fmt.Sprintf("msg-%d", i)
- if !strings.HasSuffix(traceInfo.Message, msg) {
- t.Fatalf("Expected %s but got %s", msg, traceInfo.Message)
- }
- }
-}
diff --git a/cmd/bootstrap-peer-server.go b/cmd/bootstrap-peer-server.go
index c290299d6..b6d199ceb 100644
--- a/cmd/bootstrap-peer-server.go
+++ b/cmd/bootstrap-peer-server.go
@@ -221,7 +221,7 @@ func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointS
for onlineServers < len(clnts)/2 {
for _, clnt := range clnts {
if err := clnt.Verify(ctx, srcCfg); err != nil {
- bootstrapTrace(fmt.Sprintf("clnt.Verify: %v, endpoint: %v", err, clnt.endpoint))
+ bootstrapTraceMsg(fmt.Sprintf("clnt.Verify: %v, endpoint: %v", err, clnt.endpoint))
if !isNetworkError(err) {
logger.LogOnceIf(ctx, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err), clnt.String())
incorrectConfigs = append(incorrectConfigs, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err))
diff --git a/cmd/config-current.go b/cmd/config-current.go
index 1be6725ad..006fd1b68 100644
--- a/cmd/config-current.go
+++ b/cmd/config-current.go
@@ -440,7 +440,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
}
if err == nil && dnsURL != "" {
- bootstrapTrace("initialize remote bucket DNS store")
+ bootstrapTraceMsg("initialize remote bucket DNS store")
globalDNSConfig, err = dns.NewOperatorDNS(dnsURL,
dns.Authentication(dnsUser, dnsPass),
dns.RootCAs(globalRootCAs))
@@ -455,7 +455,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
}
if etcdCfg.Enabled {
- bootstrapTrace("initialize etcd store")
+ bootstrapTraceMsg("initialize etcd store")
globalEtcdClient, err = etcd.New(etcdCfg)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
@@ -514,19 +514,19 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
transport := NewHTTPTransport()
- bootstrapTrace("initialize the event notification targets")
+ bootstrapTraceMsg("initialize the event notification targets")
globalNotifyTargetList, err = notify.FetchEnabledTargets(GlobalContext, s, transport)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
}
- bootstrapTrace("initialize the lambda targets")
+ bootstrapTraceMsg("initialize the lambda targets")
globalLambdaTargetList, err = lambda.FetchEnabledTargets(GlobalContext, s, transport)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize lambda target(s): %w", err))
}
- bootstrapTrace("applying the dynamic configuration")
+ bootstrapTraceMsg("applying the dynamic configuration")
// Apply dynamic config values
if err := applyDynamicConfig(ctx, objAPI, s); err != nil {
logger.LogIf(ctx, err)
@@ -786,13 +786,13 @@ func getValidConfig(objAPI ObjectLayer) (config.Config, error) {
// from env if found and valid
// data is optional. If nil it will be loaded from backend.
func loadConfig(objAPI ObjectLayer, data []byte) error {
- bootstrapTrace("load the configuration")
+ bootstrapTraceMsg("load the configuration")
srvCfg, err := readServerConfig(GlobalContext, objAPI, data)
if err != nil {
return err
}
- bootstrapTrace("lookup the configuration")
+ bootstrapTraceMsg("lookup the configuration")
// Override any values from ENVs.
lookupConfigs(srvCfg, objAPI)
diff --git a/cmd/config-migrate.go b/cmd/config-migrate.go
index b65b68b51..7dfd24974 100644
--- a/cmd/config-migrate.go
+++ b/cmd/config-migrate.go
@@ -2423,7 +2423,7 @@ func migrateV27ToV28() error {
// Migrates ${HOME}/.minio/config.json to '/.minio.sys/config/config.json'
// if etcd is configured then migrates /config/config.json to '/.minio.sys/config/config.json'
func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
- bootstrapTrace("migrate config to .minio.sys/config/config.json")
+ bootstrapTraceMsg("migrate config to .minio.sys/config/config.json")
// Construct path to config.json for the given bucket.
configFile := path.Join(minioConfigPrefix, minioConfigFile)
diff --git a/cmd/config.go b/cmd/config.go
index 8cdbfc563..3a6dd6957 100644
--- a/cmd/config.go
+++ b/cmd/config.go
@@ -197,10 +197,10 @@ func NewConfigSys() *ConfigSys {
// Initialize and load config from remote etcd or local config directory
func initConfig(objAPI ObjectLayer) (err error) {
- bootstrapTrace("load the configuration")
+ bootstrapTraceMsg("load the configuration")
defer func() {
if err != nil {
- bootstrapTrace(fmt.Sprintf("loading configuration failed: %v", err))
+ bootstrapTraceMsg(fmt.Sprintf("loading configuration failed: %v", err))
}
}()
@@ -213,7 +213,7 @@ func initConfig(objAPI ObjectLayer) (err error) {
return err
}
- bootstrapTrace("lookup the configuration")
+ bootstrapTraceMsg("lookup the configuration")
// Override any values from ENVs.
lookupConfigs(srvCfg, objAPI)
diff --git a/cmd/iam-object-store.go b/cmd/iam-object-store.go
index 8ae8d0ceb..65c394160 100644
--- a/cmd/iam-object-store.go
+++ b/cmd/iam-object-store.go
@@ -384,10 +384,12 @@ func (iamOS *IAMObjectStore) listAllIAMConfigItems(ctx context.Context) (map[str
// Assumes cache is locked by caller.
func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iamCache) error {
- bootstrapTrace("loading all IAM items")
if iamOS.objAPI == nil {
return errServerNotInitialized
}
+
+ bootstrapTraceMsg("loading all IAM items")
+
listedConfigItems, err := iamOS.listAllIAMConfigItems(ctx)
if err != nil {
return fmt.Errorf("unable to list IAM data: %w", err)
@@ -395,7 +397,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
// Loads things in the same order as `LoadIAMCache()`
- bootstrapTrace("loading policy documents")
+ bootstrapTraceMsg("loading policy documents")
policiesList := listedConfigItems[policiesListKey]
for _, item := range policiesList {
@@ -407,7 +409,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
setDefaultCannedPolicies(cache.iamPolicyDocsMap)
if iamOS.usersSysType == MinIOUsersSysType {
- bootstrapTrace("loading regular IAM users")
+ bootstrapTraceMsg("loading regular IAM users")
regUsersList := listedConfigItems[usersListKey]
for _, item := range regUsersList {
userName := path.Dir(item)
@@ -416,7 +418,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
}
}
- bootstrapTrace("loading regular IAM groups")
+ bootstrapTraceMsg("loading regular IAM groups")
groupsList := listedConfigItems[groupsListKey]
for _, item := range groupsList {
group := path.Dir(item)
@@ -426,7 +428,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
}
}
- bootstrapTrace("loading user policy mapping")
+ bootstrapTraceMsg("loading user policy mapping")
userPolicyMappingsList := listedConfigItems[policyDBUsersListKey]
for _, item := range userPolicyMappingsList {
userName := strings.TrimSuffix(item, ".json")
@@ -435,7 +437,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
}
}
- bootstrapTrace("loading group policy mapping")
+ bootstrapTraceMsg("loading group policy mapping")
groupPolicyMappingsList := listedConfigItems[policyDBGroupsListKey]
for _, item := range groupPolicyMappingsList {
groupName := strings.TrimSuffix(item, ".json")
@@ -444,7 +446,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
}
}
- bootstrapTrace("loading service accounts")
+ bootstrapTraceMsg("loading service accounts")
svcAccList := listedConfigItems[svcAccListKey]
for _, item := range svcAccList {
userName := path.Dir(item)
@@ -453,7 +455,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
}
}
- bootstrapTrace("loading STS users")
+ bootstrapTraceMsg("loading STS users")
stsUsersList := listedConfigItems[stsListKey]
for _, item := range stsUsersList {
userName := path.Dir(item)
@@ -462,7 +464,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
}
}
- bootstrapTrace("loading STS policy mapping")
+ bootstrapTraceMsg("loading STS policy mapping")
stsPolicyMappingsList := listedConfigItems[policyDBSTSUsersListKey]
for _, item := range stsPolicyMappingsList {
stsName := strings.TrimSuffix(item, ".json")
diff --git a/cmd/iam-store.go b/cmd/iam-store.go
index 497a324cd..481605d74 100644
--- a/cmd/iam-store.go
+++ b/cmd/iam-store.go
@@ -109,7 +109,7 @@ func getUserIdentityPath(user string, userType IAMUserType) string {
}
func saveIAMFormat(ctx context.Context, store IAMStorageAPI) error {
- bootstrapTrace("Load IAM format file")
+ bootstrapTraceMsg("Load IAM format file")
var iamFmt iamFormat
path := getIAMFormatFilePath()
if err := store.loadIAMConfig(ctx, &iamFmt, path); err != nil {
@@ -127,7 +127,7 @@ func saveIAMFormat(ctx context.Context, store IAMStorageAPI) error {
return nil
}
- bootstrapTrace("Write IAM format file")
+ bootstrapTraceMsg("Write IAM format file")
// Save iam format to version 1.
if err := store.saveIAMConfig(ctx, newIAMFormatVersion1(), path); err != nil {
logger.LogIf(ctx, err)
@@ -462,7 +462,7 @@ func setDefaultCannedPolicies(policies map[string]PolicyDoc) {
// LoadIAMCache reads all IAM items and populates a new iamCache object and
// replaces the in-memory cache object.
func (store *IAMStoreSys) LoadIAMCache(ctx context.Context) error {
- bootstrapTrace("loading IAM data")
+ bootstrapTraceMsg("loading IAM data")
newCache := newIamCache()
@@ -475,7 +475,7 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context) error {
}
} else {
- bootstrapTrace("loading policy documents")
+ bootstrapTraceMsg("loading policy documents")
if err := store.loadPolicyDocs(ctx, newCache.iamPolicyDocsMap); err != nil {
return err
}
@@ -484,41 +484,41 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context) error {
setDefaultCannedPolicies(newCache.iamPolicyDocsMap)
if store.getUsersSysType() == MinIOUsersSysType {
- bootstrapTrace("loading regular users")
+ bootstrapTraceMsg("loading regular users")
if err := store.loadUsers(ctx, regUser, newCache.iamUsersMap); err != nil {
return err
}
- bootstrapTrace("loading regular groups")
+ bootstrapTraceMsg("loading regular groups")
if err := store.loadGroups(ctx, newCache.iamGroupsMap); err != nil {
return err
}
}
- bootstrapTrace("loading user policy mapping")
+ bootstrapTraceMsg("loading user policy mapping")
// load polices mapped to users
if err := store.loadMappedPolicies(ctx, regUser, false, newCache.iamUserPolicyMap); err != nil {
return err
}
- bootstrapTrace("loading group policy mapping")
+ bootstrapTraceMsg("loading group policy mapping")
// load policies mapped to groups
if err := store.loadMappedPolicies(ctx, regUser, true, newCache.iamGroupPolicyMap); err != nil {
return err
}
- bootstrapTrace("loading service accounts")
+ bootstrapTraceMsg("loading service accounts")
// load service accounts
if err := store.loadUsers(ctx, svcUser, newCache.iamUsersMap); err != nil {
return err
}
- bootstrapTrace("loading STS users")
+ bootstrapTraceMsg("loading STS users")
// load STS temp users
if err := store.loadUsers(ctx, stsUser, newCache.iamUsersMap); err != nil {
return err
}
- bootstrapTrace("loading STS policy mapping")
+ bootstrapTraceMsg("loading STS policy mapping")
// load STS policy mappings
if err := store.loadMappedPolicies(ctx, stsUser, false, newCache.iamUserPolicyMap); err != nil {
return err
diff --git a/cmd/iam.go b/cmd/iam.go
index 85edbdf1d..3e1087105 100644
--- a/cmd/iam.go
+++ b/cmd/iam.go
@@ -187,7 +187,7 @@ func (sys *IAMSys) Initialized() bool {
}
// Load - loads all credentials, policies and policy mappings.
-func (sys *IAMSys) Load(ctx context.Context) error {
+func (sys *IAMSys) Load(ctx context.Context, firstTime bool) error {
loadStartTime := time.Now()
err := sys.store.LoadIAMCache(ctx)
if err != nil {
@@ -200,6 +200,10 @@ func (sys *IAMSys) Load(ctx context.Context) error {
atomic.StoreUint64(&sys.LastRefreshTimeUnixNano, uint64(loadStartTime.Add(loadDuration).UnixNano()))
atomic.AddUint64(&sys.TotalRefreshSuccesses, 1)
+ if firstTime {
+ bootstrapTraceMsg(fmt.Sprintf("globalIAMSys.Load(): (duration: %s)", loadDuration))
+ }
+
select {
case <-sys.configLoaded:
default:
@@ -210,7 +214,7 @@ func (sys *IAMSys) Load(ctx context.Context) error {
// Init - initializes config system by reading entries from config/iam
func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etcd.Client, iamRefreshInterval time.Duration) {
- bootstrapTrace("IAM initialization started")
+ bootstrapTraceMsg("IAM initialization started")
globalServerConfigMu.RLock()
s := globalServerConfig
globalServerConfigMu.RUnlock()
@@ -300,7 +304,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
// Load IAM data from storage.
for {
- if err := sys.Load(retryCtx); err != nil {
+ if err := sys.Load(retryCtx, true); err != nil {
if configRetriableErrors(err) {
logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. possible cause (%v)", err)
time.Sleep(time.Duration(r.Float64() * float64(5*time.Second)))
@@ -313,8 +317,6 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
break
}
- bootstrapTrace("finishing IAM loading")
-
refreshInterval := sys.iamRefreshInterval
// Set up polling for expired accounts and credentials purging.
@@ -371,6 +373,8 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
}
sys.printIAMRoles()
+
+ bootstrapTraceMsg("finishing IAM loading")
}
func (sys *IAMSys) validateAndAddRolePolicyMappings(ctx context.Context, m map[arn.ARN]string) {
@@ -446,7 +450,7 @@ func (sys *IAMSys) watch(ctx context.Context) {
select {
case <-timer.C:
refreshStart := time.Now()
- if err := sys.Load(ctx); err != nil {
+ if err := sys.Load(ctx, false); err != nil {
logger.LogIf(ctx, fmt.Errorf("Failure in periodic refresh for IAM (took %.2fs): %v", time.Since(refreshStart).Seconds(), err))
} else {
took := time.Since(refreshStart).Seconds()
@@ -590,12 +594,7 @@ func (sys *IAMSys) ListPolicies(ctx context.Context, bucketName string) (map[str
return nil, errServerNotInitialized
}
- select {
- case <-sys.configLoaded:
- return sys.store.ListPolicies(ctx, bucketName)
- case <-ctx.Done():
- return nil, ctx.Err()
- }
+ return sys.store.ListPolicies(ctx, bucketName)
}
// ListPolicyDocs - lists all canned policy docs.
@@ -604,12 +603,7 @@ func (sys *IAMSys) ListPolicyDocs(ctx context.Context, bucketName string) (map[s
return nil, errServerNotInitialized
}
- select {
- case <-sys.configLoaded:
- return sys.store.ListPolicyDocs(ctx, bucketName)
- case <-ctx.Done():
- return nil, ctx.Err()
- }
+ return sys.store.ListPolicyDocs(ctx, bucketName)
}
// SetPolicy - sets a new named policy.
diff --git a/cmd/server-main.go b/cmd/server-main.go
index 07756c8dc..71a3963db 100644
--- a/cmd/server-main.go
+++ b/cmd/server-main.go
@@ -392,23 +392,51 @@ func configRetriableErrors(err error) bool {
errors.Is(err, os.ErrDeadlineExceeded)
}
-func bootstrapTrace(msg string) {
- globalBootstrapTracer.Record(msg, 2)
+func bootstrapTraceMsg(msg string) {
+ info := madmin.TraceInfo{
+ TraceType: madmin.TraceBootstrap,
+ Time: UTCNow(),
+ NodeName: globalLocalNodeName,
+ FuncName: "BOOTSTRAP",
+ Message: fmt.Sprintf("%s %s", getSource(2), msg),
+ }
+ globalBootstrapTracer.Record(info)
+
if serverDebugLog {
logger.Info(fmt.Sprint(time.Now().Round(time.Millisecond).Format(time.RFC3339), " bootstrap: ", msg))
}
+ noSubs := globalTrace.NumSubscribers(madmin.TraceBootstrap) == 0
+ if noSubs {
+ return
+ }
+
+ globalTrace.Publish(info)
+}
+
+func bootstrapTrace(msg string, worker func()) {
+ if serverDebugLog {
+ logger.Info(fmt.Sprint(time.Now().Round(time.Millisecond).Format(time.RFC3339), " bootstrap: ", msg))
+ }
+
+ now := time.Now()
+ worker()
+ dur := time.Since(now)
+
+ info := madmin.TraceInfo{
+ TraceType: madmin.TraceBootstrap,
+ Time: UTCNow(),
+ NodeName: globalLocalNodeName,
+ FuncName: "BOOTSTRAP",
+ Message: fmt.Sprintf("%s %s (duration: %s)", getSource(2), msg, dur),
+ }
+ globalBootstrapTracer.Record(info)
+
if globalTrace.NumSubscribers(madmin.TraceBootstrap) == 0 {
return
}
- globalTrace.Publish(madmin.TraceInfo{
- TraceType: madmin.TraceBootstrap,
- Time: time.Now().UTC(),
- NodeName: globalLocalNodeName,
- FuncName: "BOOTSTRAP",
- Message: fmt.Sprintf("%s %s", getSource(2), msg),
- })
+ globalTrace.Publish(info)
}
func initServerConfig(ctx context.Context, newObject ObjectLayer) error {
@@ -548,40 +576,40 @@ func serverMain(ctx *cli.Context) {
setDefaultProfilerRates()
- // Initialize globalConsoleSys system
- bootstrapTrace("newConsoleLogger")
- globalConsoleSys = NewConsoleLogger(GlobalContext)
- logger.AddSystemTarget(GlobalContext, globalConsoleSys)
-
- // Perform any self-tests
- bootstrapTrace("selftests")
- bitrotSelfTest()
- erasureSelfTest()
- compressSelfTest()
-
// Handle all server environment vars.
- bootstrapTrace("serverHandleEnvVars")
serverHandleEnvVars()
// Handle all server command args.
- bootstrapTrace("serverHandleCmdArgs")
- serverHandleCmdArgs(ctx)
+ bootstrapTrace("serverHandleCmdArgs", func() {
+ serverHandleCmdArgs(ctx)
+ })
+
+ // Initialize globalConsoleSys system
+ bootstrapTrace("newConsoleLogger", func() {
+ globalConsoleSys = NewConsoleLogger(GlobalContext)
+ logger.AddSystemTarget(GlobalContext, globalConsoleSys)
+
+ // Set node name, only set for distributed setup.
+ globalConsoleSys.SetNodeName(globalLocalNodeName)
+ })
+
+ // Perform any self-tests
+ bootstrapTrace("selftests", func() {
+ bitrotSelfTest()
+ erasureSelfTest()
+ compressSelfTest()
+ })
// Initialize KMS configuration
- bootstrapTrace("handleKMSConfig")
- handleKMSConfig()
-
- // Set node name, only set for distributed setup.
- bootstrapTrace("setNodeName")
- globalConsoleSys.SetNodeName(globalLocalNodeName)
+ bootstrapTrace("handleKMSConfig", handleKMSConfig)
// Initialize all help
- bootstrapTrace("initHelp")
- initHelp()
+ bootstrapTrace("initHelp", initHelp)
// Initialize all sub-systems
- bootstrapTrace("initAllSubsystems")
- initAllSubsystems(GlobalContext)
+ bootstrapTrace("initAllSubsystems", func() {
+ initAllSubsystems(GlobalContext)
+ })
// Is distributed setup, error out if no certificates are found for HTTPS endpoints.
if globalIsDistErasure {
@@ -597,14 +625,16 @@ func serverMain(ctx *cli.Context) {
go func() {
if !globalCLIContext.Quiet && !globalInplaceUpdateDisabled {
// Check for new updates from dl.min.io.
- bootstrapTrace("checkUpdate")
- checkUpdate(getMinioMode())
+ bootstrapTrace("checkUpdate", func() {
+ checkUpdate(getMinioMode())
+ })
}
}()
// Set system resources to maximum.
- bootstrapTrace("setMaxResources")
- setMaxResources()
+ bootstrapTrace("setMaxResources", func() {
+ _ = setMaxResources()
+ })
// Verify kernel release and version.
if oldLinux() {
@@ -617,64 +647,65 @@ func serverMain(ctx *cli.Context) {
logger.Info(color.RedBoldf("WARNING: Detected GOMAXPROCS(%d) < NumCPU(%d), please make sure to provide all PROCS to MinIO for optimal performance", maxProcs, cpuProcs))
}
- // Configure server.
- bootstrapTrace("configureServerHandler")
- handler, err := configureServerHandler(globalEndpoints)
- if err != nil {
- logger.Fatal(config.ErrUnexpectedError(err), "Unable to configure one of server's RPC services")
- }
-
var getCert certs.GetCertificateFunc
if globalTLSCerts != nil {
getCert = globalTLSCerts.GetCertificate
}
- bootstrapTrace("xhttp.NewServer")
- httpServer := xhttp.NewServer(getServerListenAddrs()).
- UseHandler(setCriticalErrorHandler(corsHandler(handler))).
- UseTLSConfig(newTLSConfig(getCert)).
- UseShutdownTimeout(ctx.Duration("shutdown-timeout")).
- UseIdleTimeout(ctx.Duration("idle-timeout")).
- UseReadHeaderTimeout(ctx.Duration("read-header-timeout")).
- UseBaseContext(GlobalContext).
- UseCustomLogger(log.New(io.Discard, "", 0)). // Turn-off random logging by Go stdlib
- UseTCPOptions(globalTCPOptions)
-
- httpServer.TCPOptions.Trace = bootstrapTrace
- go func() {
- serveFn, err := httpServer.Init(GlobalContext, func(listenAddr string, err error) {
- logger.LogIf(GlobalContext, fmt.Errorf("Unable to listen on `%s`: %v", listenAddr, err))
- })
+ // Configure server.
+ bootstrapTrace("configureServer", func() {
+ handler, err := configureServerHandler(globalEndpoints)
if err != nil {
- globalHTTPServerErrorCh <- err
- return
+ logger.Fatal(config.ErrUnexpectedError(err), "Unable to configure one of server's RPC services")
}
- globalHTTPServerErrorCh <- serveFn()
- }()
- bootstrapTrace("setHTTPServer")
- setHTTPServer(httpServer)
+ httpServer := xhttp.NewServer(getServerListenAddrs()).
+ UseHandler(setCriticalErrorHandler(corsHandler(handler))).
+ UseTLSConfig(newTLSConfig(getCert)).
+ UseShutdownTimeout(ctx.Duration("shutdown-timeout")).
+ UseIdleTimeout(ctx.Duration("idle-timeout")).
+ UseReadHeaderTimeout(ctx.Duration("read-header-timeout")).
+ UseBaseContext(GlobalContext).
+ UseCustomLogger(log.New(io.Discard, "", 0)). // Turn-off random logging by Go stdlib
+ UseTCPOptions(globalTCPOptions)
+
+ httpServer.TCPOptions.Trace = bootstrapTraceMsg
+ go func() {
+ serveFn, err := httpServer.Init(GlobalContext, func(listenAddr string, err error) {
+ logger.LogIf(GlobalContext, fmt.Errorf("Unable to listen on `%s`: %v", listenAddr, err))
+ })
+ if err != nil {
+ globalHTTPServerErrorCh <- err
+ return
+ }
+ globalHTTPServerErrorCh <- serveFn()
+ }()
+
+ setHTTPServer(httpServer)
+ })
if globalIsDistErasure {
- bootstrapTrace("verifying system configuration")
- // Additionally in distributed setup, validate the setup and configuration.
- if err := verifyServerSystemConfig(GlobalContext, globalEndpoints); err != nil {
- logger.Fatal(err, "Unable to start the server")
- }
+ bootstrapTrace("verifying system configuration", func() {
+ // Additionally in distributed setup, validate the setup and configuration.
+ if err := verifyServerSystemConfig(GlobalContext, globalEndpoints); err != nil {
+ logger.Fatal(err, "Unable to start the server")
+ }
+ })
}
if !globalDisableFreezeOnBoot {
// Freeze the services until the bucket notification subsystem gets initialized.
- bootstrapTrace("freezeServices")
- freezeServices()
+ bootstrapTrace("freezeServices", freezeServices)
}
- bootstrapTrace("newObjectLayer")
- newObject, err := newObjectLayer(GlobalContext, globalEndpoints)
- if err != nil {
- logFatalErrs(err, Endpoint{}, true)
- }
- bootstrapTrace("newObjectLayer (initialized)")
+ var newObject ObjectLayer
+ bootstrapTrace("newObjectLayer", func() {
+ var err error
+ newObject, err = newObjectLayer(GlobalContext, globalEndpoints)
+ if err != nil {
+ logFatalErrs(err, Endpoint{}, true)
+ }
+ })
xhttp.SetDeploymentID(globalDeploymentID)
xhttp.SetMinIOVersion(Version)
@@ -688,43 +719,49 @@ func serverMain(ctx *cli.Context) {
globalNodeNamesHex[hex.EncodeToString(nodeNameSum[:])] = struct{}{}
}
- bootstrapTrace("newSharedLock")
- globalLeaderLock = newSharedLock(GlobalContext, newObject, "leader.lock")
+ bootstrapTrace("newSharedLock", func() {
+ globalLeaderLock = newSharedLock(GlobalContext, newObject, "leader.lock")
+ })
// Enable background operations on
//
// - Disk auto healing
// - MRF (most recently failed) healing
// - Background expiration routine for lifecycle policies
- bootstrapTrace("initAutoHeal")
- initAutoHeal(GlobalContext, newObject)
+ bootstrapTrace("initAutoHeal", func() {
+ initAutoHeal(GlobalContext, newObject)
+ })
- bootstrapTrace("initHealMRF")
- initHealMRF(GlobalContext, newObject)
+ bootstrapTrace("initHealMRF", func() {
+ initHealMRF(GlobalContext, newObject)
+ })
- bootstrapTrace("initBackgroundExpiry")
- initBackgroundExpiry(GlobalContext, newObject)
+ bootstrapTrace("initBackgroundExpiry", func() {
+ initBackgroundExpiry(GlobalContext, newObject)
+ })
- bootstrapTrace("initServerConfig")
- if err = initServerConfig(GlobalContext, newObject); err != nil {
- var cerr config.Err
- // For any config error, we don't need to drop into safe-mode
- // instead its a user error and should be fixed by user.
- if errors.As(err, &cerr) {
- logger.FatalIf(err, "Unable to initialize the server")
+ var err error
+ bootstrapTrace("initServerConfig", func() {
+ if err = initServerConfig(GlobalContext, newObject); err != nil {
+ var cerr config.Err
+ // For any config error, we don't need to drop into safe-mode
+ // instead its a user error and should be fixed by user.
+ if errors.As(err, &cerr) {
+ logger.FatalIf(err, "Unable to initialize the server")
+ }
+
+ // If context was canceled
+ if errors.Is(err, context.Canceled) {
+ logger.FatalIf(err, "Server startup canceled upon user request")
+ }
+
+ logger.LogIf(GlobalContext, err)
}
- // If context was canceled
- if errors.Is(err, context.Canceled) {
- logger.FatalIf(err, "Server startup canceled upon user request")
+ if !globalCLIContext.StrictS3Compat {
+ logger.Info(color.RedBold("WARNING: Strict AWS S3 compatible incoming PUT, POST content payload validation is turned off, caution is advised do not use in production"))
}
-
- logger.LogIf(GlobalContext, err)
- }
-
- if !globalCLIContext.StrictS3Compat {
- logger.Info(color.RedBold("WARNING: Strict AWS S3 compatible incoming PUT, POST content payload validation is turned off, caution is advised do not use in production"))
- }
+ })
if globalActiveCred.Equal(auth.DefaultCredentials) {
msg := fmt.Sprintf("WARNING: Detected default credentials '%s', we recommend that you change these values with 'MINIO_ROOT_USER' and 'MINIO_ROOT_PASSWORD' environment variables",
@@ -734,43 +771,44 @@ func serverMain(ctx *cli.Context) {
// Initialize users credentials and policies in background right after config has initialized.
go func() {
- bootstrapTrace("globalIAMSys.Init")
- globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalRefreshIAMInterval)
- bootstrapTrace("globalIAMSys.Initialized")
+ bootstrapTrace("globalIAMSys.Init", func() {
+ globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalRefreshIAMInterval)
+ })
// Initialize Console UI
if globalBrowserEnabled {
- bootstrapTrace("initConsoleServer")
- srv, err := initConsoleServer()
- if err != nil {
- logger.FatalIf(err, "Unable to initialize console service")
- }
+ bootstrapTrace("initConsoleServer", func() {
+ srv, err := initConsoleServer()
+ if err != nil {
+ logger.FatalIf(err, "Unable to initialize console service")
+ }
- bootstrapTrace("setConsoleSrv")
- setConsoleSrv(srv)
+ setConsoleSrv(srv)
- go func() {
- logger.FatalIf(newConsoleServerFn().Serve(), "Unable to initialize console server")
- }()
+ go func() {
+ logger.FatalIf(newConsoleServerFn().Serve(), "Unable to initialize console server")
+ }()
+ })
}
// if we see FTP args, start FTP if possible
if len(ctx.StringSlice("ftp")) > 0 {
- bootstrapTrace("startFTPServer")
- go startFTPServer(ctx)
+ bootstrapTrace("go startFTPServer", func() {
+ go startFTPServer(ctx)
+ })
}
// If we see SFTP args, start SFTP if possible
if len(ctx.StringSlice("sftp")) > 0 {
- bootstrapTrace("startFTPServer")
- go startSFTPServer(ctx)
+ bootstrapTrace("go startFTPServer", func() {
+ go startSFTPServer(ctx)
+ })
}
}()
go func() {
if !globalDisableFreezeOnBoot {
- defer unfreezeServices()
- defer bootstrapTrace("unfreezeServices")
+ defer bootstrapTrace("unfreezeServices", unfreezeServices)
t := time.AfterFunc(5*time.Minute, func() {
logger.Info(color.Yellow("WARNING: Taking more time to initialize the config subsystem. Please set '_MINIO_DISABLE_API_FREEZE_ON_BOOT=true' to not freeze the APIs"))
})
@@ -778,32 +816,38 @@ func serverMain(ctx *cli.Context) {
}
// Initialize data scanner.
- bootstrapTrace("initDataScanner")
- initDataScanner(GlobalContext, newObject)
+ bootstrapTrace("initDataScanner", func() {
+ initDataScanner(GlobalContext, newObject)
+ })
// Initialize background replication
- bootstrapTrace("initBackgroundReplication")
- initBackgroundReplication(GlobalContext, newObject)
+ bootstrapTrace("initBackgroundReplication", func() {
+ initBackgroundReplication(GlobalContext, newObject)
+ })
- bootstrapTrace("globalTransitionState.Init")
- globalTransitionState.Init(newObject)
+ bootstrapTrace("globalTransitionState.Init", func() {
+ globalTransitionState.Init(newObject)
+ })
// Initialize batch job pool.
- bootstrapTrace("newBatchJobPool")
- globalBatchJobPool = newBatchJobPool(GlobalContext, newObject, 100)
+ bootstrapTrace("newBatchJobPool", func() {
+ globalBatchJobPool = newBatchJobPool(GlobalContext, newObject, 100)
+ })
// Initialize the license update job
- bootstrapTrace("initLicenseUpdateJob")
- initLicenseUpdateJob(GlobalContext, newObject)
+ bootstrapTrace("initLicenseUpdateJob", func() {
+ initLicenseUpdateJob(GlobalContext, newObject)
+ })
go func() {
// Initialize transition tier configuration manager
- bootstrapTrace("globalTierConfigMgr.Init")
- if err := globalTierConfigMgr.Init(GlobalContext, newObject); err != nil {
- logger.LogIf(GlobalContext, err)
- } else {
- logger.FatalIf(globalTierJournal.Init(GlobalContext), "Unable to initialize remote tier pending deletes journal")
- }
+ bootstrapTrace("globalTierConfigMgr.Init", func() {
+ if err := globalTierConfigMgr.Init(GlobalContext, newObject); err != nil {
+ logger.LogIf(GlobalContext, err)
+ } else {
+ logger.FatalIf(globalTierJournal.Init(GlobalContext), "Unable to initialize remote tier pending deletes journal")
+ }
+ })
}()
// initialize the new disk cache objects.
@@ -817,40 +861,45 @@ func serverMain(ctx *cli.Context) {
}
// Initialize bucket notification system.
- bootstrapTrace("initBucketTargets")
- logger.LogIf(GlobalContext, globalEventNotifier.InitBucketTargets(GlobalContext, newObject))
+ bootstrapTrace("initBucketTargets", func() {
+ logger.LogIf(GlobalContext, globalEventNotifier.InitBucketTargets(GlobalContext, newObject))
+ })
+ var buckets []BucketInfo
// List buckets to initialize bucket metadata sub-sys.
- bootstrapTrace("listBuckets")
- buckets, err := newObject.ListBuckets(GlobalContext, BucketOptions{})
- if err != nil {
- logger.LogIf(GlobalContext, fmt.Errorf("Unable to list buckets to initialize bucket metadata sub-system: %w", err))
- }
+ bootstrapTrace("listBuckets", func() {
+ buckets, err = newObject.ListBuckets(GlobalContext, BucketOptions{})
+ if err != nil {
+ logger.LogIf(GlobalContext, fmt.Errorf("Unable to list buckets to initialize bucket metadata sub-system: %w", err))
+ }
+ })
// Initialize bucket metadata sub-system.
- bootstrapTrace("globalBucketMetadataSys.Init")
- globalBucketMetadataSys.Init(GlobalContext, buckets, newObject)
- bootstrapTrace("globalBucketMetadataSys.Initialized")
+ bootstrapTrace("globalBucketMetadataSys.Init", func() {
+ globalBucketMetadataSys.Init(GlobalContext, buckets, newObject)
+ })
// initialize replication resync state.
- bootstrapTrace("initResync")
- globalReplicationPool.initResync(GlobalContext, buckets, newObject)
+ bootstrapTrace("initResync", func() {
+ globalReplicationPool.initResync(GlobalContext, buckets, newObject)
+ })
// Initialize site replication manager after bucket metadata
- bootstrapTrace("globalSiteReplicationSys.Init")
- globalSiteReplicationSys.Init(GlobalContext, newObject)
- bootstrapTrace("globalSiteReplicationSys.Initialized")
+ bootstrapTrace("globalSiteReplicationSys.Init", func() {
+ globalSiteReplicationSys.Init(GlobalContext, newObject)
+ })
// Initialize quota manager.
- bootstrapTrace("globalBucketQuotaSys.Init")
- globalBucketQuotaSys.Init(newObject)
- bootstrapTrace("globalBucketQuotaSys.Initialized")
+ bootstrapTrace("globalBucketQuotaSys.Init", func() {
+ globalBucketQuotaSys.Init(newObject)
+ })
// Populate existing buckets to the etcd backend
if globalDNSConfig != nil {
// Background this operation.
- bootstrapTrace("go initFederatorBackend")
- go initFederatorBackend(buckets, newObject)
+ bootstrapTrace("go initFederatorBackend", func() {
+ go initFederatorBackend(buckets, newObject)
+ })
}
// Prints the formatted startup message, if err is not nil then it prints additional information as well.
@@ -866,14 +915,15 @@ func serverMain(ctx *cli.Context) {
if region == "" {
region = "us-east-1"
}
- bootstrapTrace("globalMinioClient")
- globalMinioClient, err = minio.New(globalLocalNodeName, &minio.Options{
- Creds: credentials.NewStaticV4(globalActiveCred.AccessKey, globalActiveCred.SecretKey, ""),
- Secure: globalIsTLS,
- Transport: globalProxyTransport,
- Region: region,
+ bootstrapTrace("globalMinioClient", func() {
+ globalMinioClient, err = minio.New(globalLocalNodeName, &minio.Options{
+ Creds: credentials.NewStaticV4(globalActiveCred.AccessKey, globalActiveCred.SecretKey, ""),
+ Secure: globalIsTLS,
+ Transport: globalProxyTransport,
+ Region: region,
+ })
+ logger.FatalIf(err, "Unable to initialize MinIO client")
})
- logger.FatalIf(err, "Unable to initialize MinIO client")
// Add User-Agent to differentiate the requests.
globalMinioClient.SetAppInfo("minio-perf-test", ReleaseTag)