mirror of
https://github.com/minio/minio.git
synced 2024-12-23 21:55:53 -05:00
fs: Fix non-progressing scanner (#13218)
Scanner would keep doing the same cycle in FS mode leading to missed updates. Add a few sanity checks and handle errors better.
This commit is contained in:
parent
bf409936e7
commit
f98f115ac2
@ -405,11 +405,11 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
|
||||
err := readDirFn(path.Join(f.root, folder.name), func(entName string, typ os.FileMode) error {
|
||||
// Parse
|
||||
entName = pathClean(path.Join(folder.name, entName))
|
||||
if entName == "" {
|
||||
if entName == "" || entName == folder.name {
|
||||
if f.dataUsageScannerDebug {
|
||||
console.Debugf(scannerLogPrefix+" no bucket (%s,%s)\n", f.root, entName)
|
||||
console.Debugf(scannerLogPrefix+" no entity (%s,%s)\n", f.root, entName)
|
||||
}
|
||||
return errDoneForNow
|
||||
return nil
|
||||
}
|
||||
bucket, prefix := path2BucketObjectWithBasePath(f.root, entName)
|
||||
if bucket == "" {
|
||||
@ -435,7 +435,9 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
|
||||
if typ&os.ModeDir != 0 {
|
||||
h := hashPath(entName)
|
||||
_, exists := f.oldCache.Cache[h.Key()]
|
||||
|
||||
if h == thisHash {
|
||||
return nil
|
||||
}
|
||||
this := cachedFolder{name: entName, parent: &thisHash, objectHealProbDiv: folder.objectHealProbDiv}
|
||||
delete(abandonedChildren, h.Key()) // h.Key() already accounted for.
|
||||
if exists {
|
||||
@ -468,11 +470,14 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
|
||||
item.heal = item.heal && f.healObjectSelect > 0
|
||||
|
||||
sz, err := f.getSize(item)
|
||||
if err == errSkipFile {
|
||||
if err != nil {
|
||||
wait() // wait to proceed to next entry.
|
||||
|
||||
if err != errSkipFile && f.dataUsageScannerDebug {
|
||||
console.Debugf(scannerLogPrefix+" getSize \"%v/%v\" returned err: %v\n", bucket, item.objectPath(), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// successfully read means we have a valid object.
|
||||
foundObjects = true
|
||||
// Remove filename i.e is the meta file to construct object name
|
||||
|
@ -258,6 +258,12 @@ func (fs *FSObjects) NSScanner(ctx context.Context, bf *bloomFilter, updates cha
|
||||
updates <- totalCache.dui(dataUsageRoot, buckets)
|
||||
return nil
|
||||
}
|
||||
for i, b := range buckets {
|
||||
if isReservedOrInvalidBucket(b.Name, false) {
|
||||
// Delete bucket...
|
||||
buckets = append(buckets[:i], buckets[i+1:]...)
|
||||
}
|
||||
}
|
||||
|
||||
totalCache.Info.BloomFilter = bf.bytes()
|
||||
|
||||
@ -282,6 +288,7 @@ func (fs *FSObjects) NSScanner(ctx context.Context, bf *bloomFilter, updates cha
|
||||
bCache.Info.Name = b.Name
|
||||
}
|
||||
bCache.Info.BloomFilter = totalCache.Info.BloomFilter
|
||||
bCache.Info.NextCycle = wantCycle
|
||||
upds := make(chan dataUsageEntry, 1)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
@ -290,7 +297,7 @@ func (fs *FSObjects) NSScanner(ctx context.Context, bf *bloomFilter, updates cha
|
||||
for update := range upds {
|
||||
totalCache.replace(b.Name, dataUsageRoot, update)
|
||||
if intDataUpdateTracker.debug {
|
||||
logger.Info(color.Green("NSScanner:")+" Got update:", len(totalCache.Cache))
|
||||
logger.Info(color.Green("NSScanner:")+" Got update: %v", len(totalCache.Cache))
|
||||
}
|
||||
cloned := totalCache.clone()
|
||||
updates <- cloned.dui(dataUsageRoot, buckets)
|
||||
|
Loading…
Reference in New Issue
Block a user