Add more bootstrap info in debug mode (#17362)

cmd/bootstrap-peer-server.go

@@ -24,6 +24,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"os"
 	"reflect"
 	"time"
 
@@ -216,6 +217,7 @@ func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointS
 	for onlineServers < len(clnts)/2 {
 		for _, clnt := range clnts {
 			if err := clnt.Verify(ctx, srcCfg); err != nil {
+				bootstrapTrace(fmt.Sprintf("clnt.Verify: %v, endpoint: %v", err, clnt.endpoint))
 				if !isNetworkError(err) {
 					logger.LogOnceIf(ctx, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err), clnt.String())
 					incorrectConfigs = append(incorrectConfigs, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err))
@@ -264,7 +266,11 @@ func newBootstrapRESTClients(endpointServerPools EndpointServerPools) []*bootstr
 
 			// Only proceed for remote endpoints.
 			if !endpoint.IsLocal {
-				clnts = append(clnts, newBootstrapRESTClient(endpoint))
+				cl := newBootstrapRESTClient(endpoint)
+				if serverDebugLog {
+					cl.restClient.TraceOutput = os.Stdout
+				}
+				clnts = append(clnts, cl)
 			}
 		}
 	}

cmd/server-main.go

@@ -396,6 +396,9 @@ func configRetriableErrors(err error) bool {
 
 func bootstrapTrace(msg string) {
 	globalBootstrapTracer.Record(msg, 2)
+	if serverDebugLog {
+		logger.Info(fmt.Sprint(time.Now().Round(time.Millisecond).Format(time.RFC3339), " bootstrap: ", msg))
+	}
 
 	if globalTrace.NumSubscribers(madmin.TraceBootstrap) == 0 {
 		return
@@ -516,30 +519,38 @@ func serverMain(ctx *cli.Context) {
 	setDefaultProfilerRates()
 
 	// Initialize globalConsoleSys system
+	bootstrapTrace("newConsoleLogger")
 	globalConsoleSys = NewConsoleLogger(GlobalContext)
 	logger.AddSystemTarget(GlobalContext, globalConsoleSys)
 
 	// Perform any self-tests
+	bootstrapTrace("selftests")
 	bitrotSelfTest()
 	erasureSelfTest()
 	compressSelfTest()
 
 	// Handle all server environment vars.
+	bootstrapTrace("serverHandleEnvVars")
 	serverHandleEnvVars()
 
 	// Handle all server command args.
+	bootstrapTrace("serverHandleCmdArgs")
 	serverHandleCmdArgs(ctx)
 
 	// Initialize KMS configuration
+	bootstrapTrace("handleKMSConfig")
 	handleKMSConfig()
 
 	// Set node name, only set for distributed setup.
+	bootstrapTrace("setNodeName")
 	globalConsoleSys.SetNodeName(globalLocalNodeName)
 
 	// Initialize all help
+	bootstrapTrace("initHelp")
 	initHelp()
 
 	// Initialize all sub-systems
+	bootstrapTrace("initAllSubsystems")
 	initAllSubsystems(GlobalContext)
 
 	// Is distributed setup, error out if no certificates are found for HTTPS endpoints.
@@ -556,11 +567,13 @@ func serverMain(ctx *cli.Context) {
 	go func() {
 		if !globalCLIContext.Quiet && !globalInplaceUpdateDisabled {
 			// Check for new updates from dl.min.io.
+			bootstrapTrace("checkUpdate")
 			checkUpdate(getMinioMode())
 		}
 	}()
 
 	// Set system resources to maximum.
+	bootstrapTrace("setMaxResources")
 	setMaxResources()
 
 	// Verify kernel release and version.
@@ -575,6 +588,7 @@ func serverMain(ctx *cli.Context) {
 	}
 
 	// Configure server.
+	bootstrapTrace("configureServerHandler")
 	handler, err := configureServerHandler(globalEndpoints)
 	if err != nil {
 		logger.Fatal(config.ErrUnexpectedError(err), "Unable to configure one of server's RPC services")
@@ -585,6 +599,7 @@ func serverMain(ctx *cli.Context) {
 		getCert = globalTLSCerts.GetCertificate
 	}
 
+	bootstrapTrace("xhttp.NewServer")
 	httpServer := xhttp.NewServer(getServerListenAddrs()).
 		UseHandler(setCriticalErrorHandler(corsHandler(handler))).
 		UseTLSConfig(newTLSConfig(getCert)).
@@ -595,10 +610,12 @@ func serverMain(ctx *cli.Context) {
 		UseCustomLogger(log.New(io.Discard, "", 0)). // Turn-off random logging by Go stdlib
 		UseTCPOptions(globalTCPOptions)
 
+	httpServer.TCPOptions.Trace = bootstrapTrace
 	go func() {
 		globalHTTPServerErrorCh <- httpServer.Start(GlobalContext)
 	}()
 
+	bootstrapTrace("setHTTPServer")
 	setHTTPServer(httpServer)
 
 	if globalIsDistErasure {
@@ -609,6 +626,7 @@ func serverMain(ctx *cli.Context) {
 		}
 	}
 
+	bootstrapTrace("newObjectLayer")
 	newObject, err := newObjectLayer(GlobalContext, globalEndpoints)
 	if err != nil {
 		logFatalErrs(err, Endpoint{}, true)
@@ -617,11 +635,15 @@ func serverMain(ctx *cli.Context) {
 	xhttp.SetDeploymentID(globalDeploymentID)
 	xhttp.SetMinIOVersion(Version)
 
+	bootstrapTrace("newSharedLock")
 	globalLeaderLock = newSharedLock(GlobalContext, newObject, "leader.lock")
 
 	// Enable background operations for erasure coding
+	bootstrapTrace("initAutoHeal")
 	initAutoHeal(GlobalContext, newObject)
+	bootstrapTrace("initHealMRF")
 	initHealMRF(GlobalContext, newObject)
+	bootstrapTrace("initBackgroundExpiry")
 	initBackgroundExpiry(GlobalContext, newObject)
 
 	if !globalCLIContext.StrictS3Compat {
@@ -658,15 +680,18 @@ func serverMain(ctx *cli.Context) {
 
 	// Initialize users credentials and policies in background right after config has initialized.
 	go func() {
+		bootstrapTrace("globalIAMSys.Init")
 		globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalRefreshIAMInterval)
 
 		// Initialize
 		if globalBrowserEnabled {
+			bootstrapTrace("initConsoleServer")
 			srv, err := initConsoleServer()
 			if err != nil {
 				logger.FatalIf(err, "Unable to initialize console service")
 			}
 
+			bootstrapTrace("setConsoleSrv")
 			setConsoleSrv(srv)
 
 			go func() {
@@ -676,18 +701,22 @@ func serverMain(ctx *cli.Context) {
 
 		// if we see FTP args, start FTP if possible
 		if len(ctx.StringSlice("ftp")) > 0 {
+			bootstrapTrace("startFTPServer")
 			go startFTPServer(ctx)
 		}
 
 		// If we see SFTP args, start SFTP if possible
 		if len(ctx.StringSlice("sftp")) > 0 {
+			bootstrapTrace("startFTPServer")
 			go startSFTPServer(ctx)
 		}
 	}()
 
 	go func() {
 		if !globalDisableFreezeOnBoot {
+			bootstrapTrace("freezeServices")
 			defer unfreezeServices()
+			defer bootstrapTrace("unfreezeServices")
 			t := time.AfterFunc(5*time.Minute, func() {
 				logger.Info(color.Yellow("WARNING: Taking more time to initialize the config subsystem. Please set '_MINIO_DISABLE_API_FREEZE_ON_BOOT=true' to not freeze the APIs"))
 			})
@@ -695,21 +724,27 @@ func serverMain(ctx *cli.Context) {
 		}
 
 		// Initialize data scanner.
+		bootstrapTrace("initDataScanner")
 		initDataScanner(GlobalContext, newObject)
 
 		// Initialize background replication
+		bootstrapTrace("initBackgroundReplication")
 		initBackgroundReplication(GlobalContext, newObject)
 
+		bootstrapTrace("globalTransitionState.Init")
 		globalTransitionState.Init(newObject)
 
 		// Initialize batch job pool.
+		bootstrapTrace("newBatchJobPool")
 		globalBatchJobPool = newBatchJobPool(GlobalContext, newObject, 100)
 
 		// Initialize the license update job
+		bootstrapTrace("initLicenseUpdateJob")
 		initLicenseUpdateJob(GlobalContext, newObject)
 
 		go func() {
 			// Initialize transition tier configuration manager
+			bootstrapTrace("globalTierConfigMgr.Init")
 			err := globalTierConfigMgr.Init(GlobalContext, newObject)
 			if err != nil {
 				logger.LogIf(GlobalContext, err)
@@ -732,29 +767,36 @@ func serverMain(ctx *cli.Context) {
 		}
 
 		// Initialize bucket notification system.
+		bootstrapTrace("initBucketTargets")
 		logger.LogIf(GlobalContext, globalEventNotifier.InitBucketTargets(GlobalContext, newObject))
 
 		// List buckets to initialize bucket metadata sub-sys.
+		bootstrapTrace("listBuckets")
 		buckets, err := newObject.ListBuckets(GlobalContext, BucketOptions{})
 		if err != nil {
 			logger.LogIf(GlobalContext, fmt.Errorf("Unable to list buckets to initialize bucket metadata sub-system: %w", err))
 		}
 
 		// Initialize bucket metadata sub-system.
+		bootstrapTrace("globalBucketMetadataSys.Init")
 		globalBucketMetadataSys.Init(GlobalContext, buckets, newObject)
 
 		// initialize replication resync state.
+		bootstrapTrace("go initResync")
 		go globalReplicationPool.initResync(GlobalContext, buckets, newObject)
 
 		// Initialize site replication manager after bucket metadata
+		bootstrapTrace("globalSiteReplicationSys.Init")
 		globalSiteReplicationSys.Init(GlobalContext, newObject)
 
 		// Initialize quota manager.
+		bootstrapTrace("globalBucketQuotaSys.Init")
 		globalBucketQuotaSys.Init(newObject)
 
 		// Populate existing buckets to the etcd backend
 		if globalDNSConfig != nil {
 			// Background this operation.
+			bootstrapTrace("initFederatorBackend")
 			go initFederatorBackend(buckets, newObject)
 		}
 
@@ -771,6 +813,7 @@ func serverMain(ctx *cli.Context) {
 	if region == "" {
 		region = "us-east-1"
 	}
+	bootstrapTrace("globalMinioClient")
 	globalMinioClient, err = minio.New(globalLocalNodeName, &minio.Options{
 		Creds:     credentials.NewStaticV4(globalActiveCred.AccessKey, globalActiveCred.SecretKey, ""),
 		Secure:    globalIsTLS,