fix: prioritized latest buckets for crawler to finish the scans faster (#11115)

crawler should only ListBuckets once not for each serverPool,
buckets are same across all pools, across sets and ListBuckets
always returns an unified view, once list buckets returns
sort it by create time to scan the latest buckets earlier
with the assumption that latest buckets would have lesser
content than older buckets allowing them to be scanned faster
and also to be able to provide more closer to latest view.
This commit is contained in:
Harshavardhana
2020-12-15 17:34:54 -08:00
committed by GitHub
parent d674263eb7
commit c606c76323
6 changed files with 33 additions and 24 deletions

View File

@@ -21,6 +21,7 @@ import (
"encoding/json"
"fmt"
"net/http"
"sort"
"strings"
"sync"
"time"
@@ -873,6 +874,11 @@ func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
return errFnHealFromAPIErr(h.ctx, err)
}
// Heal latest buckets first.
sort.Slice(buckets, func(i, j int) bool {
return buckets[i].Created.After(buckets[j].Created)
})
for _, bucket := range buckets {
if err = h.healBucket(objAPI, bucket.Name, bucketsOnly); err != nil {
return err