converge clusterRead health into cluster health (#19063)

This commit is contained in:
Harshavardhana
2024-02-15 16:48:36 -08:00
committed by GitHub
parent 68dde2359f
commit 607cafadbc
9 changed files with 69 additions and 149 deletions

View File

@@ -2280,55 +2280,23 @@ type HealthOptions struct {
// was queried
type HealthResult struct {
Healthy bool
HealthyRead bool
HealingDrives int
ESHealth []struct {
Maintenance bool
PoolID, SetID int
Healthy bool
HealthyRead bool
HealthyDrives int
HealingDrives int
ReadQuorum int
WriteQuorum int
}
WriteQuorum int
ReadQuorum int
UsingDefaults bool
}
// ReadHealth returns if the cluster can serve read requests
func (z *erasureServerPools) ReadHealth(ctx context.Context) bool {
erasureSetUpCount := make([][]int, len(z.serverPools))
for i := range z.serverPools {
erasureSetUpCount[i] = make([]int, len(z.serverPools[i].sets))
}
diskIDs := globalNotificationSys.GetLocalDiskIDs(ctx)
diskIDs = append(diskIDs, getLocalDiskIDs(z))
for _, localDiskIDs := range diskIDs {
for _, id := range localDiskIDs {
poolIdx, setIdx, _, err := z.getPoolAndSet(id)
if err != nil {
logger.LogIf(ctx, err)
continue
}
erasureSetUpCount[poolIdx][setIdx]++
}
}
b := z.BackendInfo()
poolReadQuorums := make([]int, len(b.StandardSCData))
copy(poolReadQuorums, b.StandardSCData)
for poolIdx := range erasureSetUpCount {
for setIdx := range erasureSetUpCount[poolIdx] {
if erasureSetUpCount[poolIdx][setIdx] < poolReadQuorums[poolIdx] {
return false
}
}
}
return true
}
// Health - returns current status of the object layer health,
// provides if write access exists across sets, additionally
// can be used to query scenarios if health may be lost
@@ -2397,9 +2365,20 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
}
}
var maximumReadQuorum int
for _, readQuorum := range poolReadQuorums {
if maximumReadQuorum == 0 {
maximumReadQuorum = readQuorum
}
if readQuorum > maximumReadQuorum {
maximumReadQuorum = readQuorum
}
}
result := HealthResult{
Healthy: true,
WriteQuorum: maximumWriteQuorum,
ReadQuorum: maximumReadQuorum,
UsingDefaults: usingDefaults, // indicates if config was not initialized and we are using defaults on this node.
}
@@ -2409,6 +2388,7 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
Maintenance bool
PoolID, SetID int
Healthy bool
HealthyRead bool
HealthyDrives, HealingDrives int
ReadQuorum, WriteQuorum int
}{
@@ -2416,6 +2396,7 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
SetID: setIdx,
PoolID: poolIdx,
Healthy: erasureSetUpCount[poolIdx][setIdx].online >= poolWriteQuorums[poolIdx],
HealthyRead: erasureSetUpCount[poolIdx][setIdx].online >= poolReadQuorums[poolIdx],
HealthyDrives: erasureSetUpCount[poolIdx][setIdx].online,
HealingDrives: erasureSetUpCount[poolIdx][setIdx].healing,
ReadQuorum: poolReadQuorums[poolIdx],
@@ -2428,6 +2409,12 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
fmt.Errorf("Write quorum may be lost on pool: %d, set: %d, expected write quorum: %d",
poolIdx, setIdx, poolWriteQuorums[poolIdx]))
}
result.HealthyRead = erasureSetUpCount[poolIdx][setIdx].online >= poolReadQuorums[poolIdx]
if !result.HealthyRead {
logger.LogIf(logger.SetReqInfo(ctx, reqInfo),
fmt.Errorf("Read quorum may be lost on pool: %d, set: %d, expected read quorum: %d",
poolIdx, setIdx, poolReadQuorums[poolIdx]))
}
}
}