avoid waiting for quorum health while debugging (#19955)

This commit is contained in:
Harshavardhana 2024-06-19 10:12:20 -07:00 committed by GitHub
parent e5335450a4
commit 7a4b250c8b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 13 additions and 19 deletions

View File

@ -2586,29 +2586,17 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
healthy := erasureSetUpCount[poolIdx][setIdx].online >= poolWriteQuorums[poolIdx] healthy := erasureSetUpCount[poolIdx][setIdx].online >= poolWriteQuorums[poolIdx]
if !healthy { if !healthy {
if opts.Startup {
storageLogIf(logger.SetReqInfo(ctx, reqInfo), storageLogIf(logger.SetReqInfo(ctx, reqInfo),
fmt.Errorf("Write quorum was not established on pool: %d, set: %d, expected write quorum: %d", fmt.Errorf("Write quorum could not be established on pool: %d, set: %d, expected write quorum: %d, drives-online: %d",
poolIdx, setIdx, poolWriteQuorums[poolIdx]), logger.FatalKind) poolIdx, setIdx, poolWriteQuorums[poolIdx], erasureSetUpCount[poolIdx][setIdx].online), logger.FatalKind)
} else {
storageLogIf(logger.SetReqInfo(ctx, reqInfo),
fmt.Errorf("Write quorum may be lost on pool: %d, set: %d, expected write quorum: %d",
poolIdx, setIdx, poolWriteQuorums[poolIdx]), logger.FatalKind)
}
} }
result.Healthy = result.Healthy && healthy result.Healthy = result.Healthy && healthy
healthyRead := erasureSetUpCount[poolIdx][setIdx].online >= poolReadQuorums[poolIdx] healthyRead := erasureSetUpCount[poolIdx][setIdx].online >= poolReadQuorums[poolIdx]
if !healthyRead { if !healthyRead {
if opts.Startup {
storageLogIf(logger.SetReqInfo(ctx, reqInfo), storageLogIf(logger.SetReqInfo(ctx, reqInfo),
fmt.Errorf("Read quorum was not established on pool: %d, set: %d, expected read quorum: %d", fmt.Errorf("Read quorum could not be established on pool: %d, set: %d, expected read quorum: %d, drives-online: %d",
poolIdx, setIdx, poolReadQuorums[poolIdx])) poolIdx, setIdx, poolReadQuorums[poolIdx], erasureSetUpCount[poolIdx][setIdx].online))
} else {
storageLogIf(logger.SetReqInfo(ctx, reqInfo),
fmt.Errorf("Read quorum may be lost on pool: %d, set: %d, expected read quorum: %d",
poolIdx, setIdx, poolReadQuorums[poolIdx]))
}
} }
result.HealthyRead = result.HealthyRead && healthyRead result.HealthyRead = result.HealthyRead && healthyRead
} }

View File

@ -189,12 +189,14 @@ func printMinIOVersion(c *cli.Context) {
io.Copy(c.App.Writer, versionBanner(c)) io.Copy(c.App.Writer, versionBanner(c))
} }
var debugNoExit = env.Get("_MINIO_DEBUG_NO_EXIT", "") != ""
// Main main for minio server. // Main main for minio server.
func Main(args []string) { func Main(args []string) {
// Set the minio app name. // Set the minio app name.
appName := filepath.Base(args[0]) appName := filepath.Base(args[0])
if env.Get("_MINIO_DEBUG_NO_EXIT", "") != "" { if debugNoExit {
freeze := func(_ int) { freeze := func(_ int) {
// Infinite blocking op // Infinite blocking op
<-make(chan struct{}) <-make(chan struct{})

View File

@ -923,6 +923,10 @@ func serverMain(ctx *cli.Context) {
bootstrapTrace("waitForQuorum", func() { bootstrapTrace("waitForQuorum", func() {
result := newObject.Health(context.Background(), HealthOptions{Startup: true}) result := newObject.Health(context.Background(), HealthOptions{Startup: true})
for !result.Healthy { for !result.Healthy {
if debugNoExit {
logger.Info("Not waiting for quorum since we are debugging.. possible cause unhealthy sets (%s)", result)
break
}
d := time.Duration(r.Float64() * float64(time.Second)) d := time.Duration(r.Float64() * float64(time.Second))
logger.Info("Waiting for quorum healthcheck to succeed.. possible cause unhealthy sets (%s), retrying in %s", result, d) logger.Info("Waiting for quorum healthcheck to succeed.. possible cause unhealthy sets (%s), retrying in %s", result, d)
time.Sleep(d) time.Sleep(d)