fix: healing deadlocks and ordering (#16643)

This commit is contained in:
Klaus Post 2023-02-17 18:52:43 +01:00 committed by GitHub
parent 98a84d88e2
commit 84bb7d05a9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 15 additions and 18 deletions

View File

@ -407,9 +407,6 @@ type healSequence struct {
// bucket, and object on which heal seq. was initiated
bucket, object string
// A channel of entities with heal result
respCh chan healResult
// Report healing progress
reportProgress bool
@ -472,7 +469,6 @@ func newHealSequence(ctx context.Context, bucket, objPrefix, clientAddr string,
clientToken := mustGetUUID()
return &healSequence{
respCh: make(chan healResult),
bucket: bucket,
object: objPrefix,
reportProgress: true,
@ -719,15 +715,18 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
if serverDebugLog {
logger.Info("Task in the queue: %#v", task)
}
case <-h.ctx.Done():
return nil
default:
// task queue is full, no more workers, we shall move on and heal later.
return nil
}
} else {
// respCh must be set for guaranteed result
task.respCh = h.respCh
// Don't wait for result
return nil
}
// respCh must be set to wait for result.
// We make it size 1, so a result can always be written
// even if we aren't listening.
task.respCh = make(chan healResult, 1)
select {
case globalBackgroundHealRoutine.tasks <- task:
if serverDebugLog {
@ -736,11 +735,10 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
case <-h.ctx.Done():
return nil
}
}
// task queued, now wait for the response.
select {
case res := <-h.respCh:
case res := <-task.respCh:
if !h.reportProgress {
if errors.Is(res.err, errSkipFile) { // this is only sent usually by nopHeal
return nil

View File

@ -48,7 +48,6 @@ func newBgHealSequence() *healSequence {
}
return &healSequence{
respCh: make(chan healResult),
startTime: UTCNow(),
clientToken: bgHealingUUID,
// run-background heal with reserved bucket