MyFavMirrors
/
minio
mirror of https://github.com/minio/minio.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: enhance logging in crawler use console.Debug instead of logger.Info (#11179)

This commit is contained in:
Harshavardhana 2020-12-29 01:57:28 -08:00 committed by GitHub
parent ca0d31b09a
commit cc457f1798
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 71 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
cmd/data-crawler.go
View File

@ -173,10 +173,10 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
t := UTCNow() t := UTCNow()
logPrefix := color.Green("data-usage: ") logPrefix := color.Green("data-usage: ")
logSuffix := color.Blue(" - %v + %v", basePath, cache.Info.Name) logSuffix := color.Blue("- %v + %v", basePath, cache.Info.Name)
if intDataUpdateTracker.debug { if intDataUpdateTracker.debug {
defer func() { defer func() {
logger.Info(logPrefix+" Crawl time: %v"+logSuffix, time.Since(t)) console.Debugf(logPrefix+" Crawl time: %v %s\n", time.Since(t), logSuffix)
}() }()
} }
@ -204,7 +204,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
if ok { if ok {
s.disks = objAPI.GetDisksID(cache.Disks...) s.disks = objAPI.GetDisksID(cache.Disks...)
if len(s.disks) != len(cache.Disks) { if len(s.disks) != len(cache.Disks) {
logger.Info(logPrefix+"Missing disks, want %d, found %d. Cannot heal."+logSuffix, len(cache.Disks), len(s.disks)) console.Debugf(logPrefix+"Missing disks, want %d, found %d. Cannot heal. %s\n", len(cache.Disks), len(s.disks), logSuffix)
s.disks = s.disks[:0] s.disks = s.disks[:0]
} }
} }
@ -226,21 +226,21 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
} }
} }
if s.dataUsageCrawlDebug { if s.dataUsageCrawlDebug {
logger.Info(logPrefix+"Start crawling. Bloom filter: %v"+logSuffix, s.withFilter != nil) console.Debugf(logPrefix+"Start crawling. Bloom filter: %v %s\n", s.withFilter != nil, logSuffix)
} }
done := ctx.Done() done := ctx.Done()
var flattenLevels = 2 var flattenLevels = 2
if s.dataUsageCrawlDebug { if s.dataUsageCrawlDebug {
logger.Info(logPrefix+"Cycle: %v, Entries: %v"+logSuffix, cache.Info.NextCycle, len(cache.Cache)) console.Debugf(logPrefix+"Cycle: %v, Entries: %v %s\n", cache.Info.NextCycle, len(cache.Cache), logSuffix)
} }
// Always scan flattenLevels deep. Cache root is level 0. // Always scan flattenLevels deep. Cache root is level 0.
todo := []cachedFolder{{name: cache.Info.Name, objectHealProbDiv: 1}} todo := []cachedFolder{{name: cache.Info.Name, objectHealProbDiv: 1}}
for i := 0; i < flattenLevels; i++ { for i := 0; i < flattenLevels; i++ {
if s.dataUsageCrawlDebug { if s.dataUsageCrawlDebug {
logger.Info(logPrefix+"Level %v, scanning %v directories."+logSuffix, i, len(todo)) console.Debugf(logPrefix+"Level %v, scanning %v directories. %s\n", i, len(todo), logSuffix)
} }
select { select {
case <-done: case <-done:
@ -256,7 +256,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
} }
if s.dataUsageCrawlDebug { if s.dataUsageCrawlDebug {
logger.Info(logPrefix+"New folders: %v"+logSuffix, s.newFolders) console.Debugf(logPrefix+"New folders: %v %s\n", s.newFolders, logSuffix)
} }
// Add new folders first // Add new folders first
@ -272,7 +272,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
continue continue
} }
if du == nil { if du == nil {
logger.Info(logPrefix + "no disk usage provided" + logSuffix) console.Debugln(logPrefix + "no disk usage provided" + logSuffix)
continue continue
} }
@ -285,7 +285,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
} }
if s.dataUsageCrawlDebug { if s.dataUsageCrawlDebug {
logger.Info(logPrefix+"Existing folders: %v"+logSuffix, len(s.existingFolders)) console.Debugf(logPrefix+"Existing folders: %v %s\n", len(s.existingFolders), logSuffix)
} }
// Do selective scanning of existing folders. // Do selective scanning of existing folders.
@ -312,13 +312,13 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
if !s.withFilter.containsDir(folder.name) { if !s.withFilter.containsDir(folder.name) {
if !h.mod(s.oldCache.Info.NextCycle, s.healFolderInclude/folder.objectHealProbDiv) { if !h.mod(s.oldCache.Info.NextCycle, s.healFolderInclude/folder.objectHealProbDiv) {
if s.dataUsageCrawlDebug { if s.dataUsageCrawlDebug {
logger.Info(logPrefix+"Skipping non-updated folder: %v"+logSuffix, folder) console.Debugf(logPrefix+"Skipping non-updated folder: %v %s\n", folder, logSuffix)
} }
s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h.Key()]) s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h.Key()])
continue continue
} else { } else {
if s.dataUsageCrawlDebug { if s.dataUsageCrawlDebug {
logger.Info(logPrefix+"Adding non-updated folder to heal check: %v"+logSuffix, folder.name) console.Debugf(logPrefix+"Adding non-updated folder to heal check: %v %s\n", folder.name, logSuffix)
} }
// Update probability of including objects // Update probability of including objects
folder.objectHealProbDiv = s.healFolderInclude folder.objectHealProbDiv = s.healFolderInclude
@ -340,7 +340,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
s.newCache.replaceHashed(h, folder.parent, *du) s.newCache.replaceHashed(h, folder.parent, *du)
} }
if s.dataUsageCrawlDebug { if s.dataUsageCrawlDebug {
logger.Info(logPrefix+"Finished crawl, %v entries"+logSuffix, len(s.newCache.Cache)) console.Debugf(logPrefix+"Finished crawl, %v entries %s\n", len(s.newCache.Cache), logSuffix)
} }
s.newCache.Info.LastUpdate = UTCNow() s.newCache.Info.LastUpdate = UTCNow()
s.newCache.Info.NextCycle++ s.newCache.Info.NextCycle++
@ -354,6 +354,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFolder, final bool) ([]cachedFolder, error) { func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFolder, final bool) ([]cachedFolder, error) {
var nextFolders []cachedFolder var nextFolders []cachedFolder
done := ctx.Done() done := ctx.Done()
scannerLogPrefix := color.Green("folder-scanner:")
for _, folder := range folders { for _, folder := range folders {
select { select {
case <-done: case <-done:
@ -369,7 +370,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
var activeLifeCycle *lifecycle.Lifecycle var activeLifeCycle *lifecycle.Lifecycle
if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix, true) { if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix, true) {
if f.dataUsageCrawlDebug { if f.dataUsageCrawlDebug {
logger.Info(color.Green("folder-scanner:")+" Prefix %q has active rules", prefix) console.Debugf(scannerLogPrefix+" Prefix %q has active rules\n", prefix)
} }
activeLifeCycle = f.oldCache.Info.lifeCycle activeLifeCycle = f.oldCache.Info.lifeCycle
filter = nil filter = nil
@ -380,12 +381,12 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
if !thisHash.mod(f.oldCache.Info.NextCycle, f.healFolderInclude/folder.objectHealProbDiv) { if !thisHash.mod(f.oldCache.Info.NextCycle, f.healFolderInclude/folder.objectHealProbDiv) {
f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent) f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
if f.dataUsageCrawlDebug { if f.dataUsageCrawlDebug {
logger.Info(color.Green("folder-scanner:")+" Skipping non-updated folder: %v", folder.name) console.Debugf(scannerLogPrefix+" Skipping non-updated folder: %v\n", folder.name)
} }
continue continue
} else { } else {
if f.dataUsageCrawlDebug { if f.dataUsageCrawlDebug {
logger.Info(color.Green("folder-scanner:")+" Adding non-updated folder to heal check: %v", folder.name) console.Debugf(scannerLogPrefix+" Adding non-updated folder to heal check: %v\n", folder.name)
} }
// If probability was already crawlerHealFolderInclude, keep it. // If probability was already crawlerHealFolderInclude, keep it.
folder.objectHealProbDiv = f.healFolderInclude folder.objectHealProbDiv = f.healFolderInclude
@ -402,14 +403,14 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
bucket, prefix := path2BucketObjectWithBasePath(f.root, entName) bucket, prefix := path2BucketObjectWithBasePath(f.root, entName)
if bucket == "" { if bucket == "" {
if f.dataUsageCrawlDebug { if f.dataUsageCrawlDebug {
logger.Info(color.Green("folder-scanner:")+" no bucket (%s,%s)", f.root, entName) console.Debugf(scannerLogPrefix+" no bucket (%s,%s)\n", f.root, entName)
} }
return nil return nil
} }
if isReservedOrInvalidBucket(bucket, false) { if isReservedOrInvalidBucket(bucket, false) {
if f.dataUsageCrawlDebug { if f.dataUsageCrawlDebug {
logger.Info(color.Green("folder-scanner:")+" invalid bucket: %v, entry: %v", bucket, entName) console.Debugf(scannerLogPrefix+" invalid bucket: %v, entry: %v\n", bucket, entName)
} }
return nil return nil
} }
@ -521,10 +522,11 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
bucket: "", bucket: "",
} }
healObjectsPrefix := color.Green("healObjects:")
for k := range existing { for k := range existing {
bucket, prefix := path2BucketObject(k) bucket, prefix := path2BucketObject(k)
if f.dataUsageCrawlDebug { if f.dataUsageCrawlDebug {
logger.Info(color.Green("folder-scanner:")+" checking disappeared folder: %v/%v", bucket, prefix) console.Debugf(scannerLogPrefix+" checking disappeared folder: %v/%v\n", bucket, prefix)
} }
// Dynamic time delay. // Dynamic time delay.
@ -544,13 +546,13 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
// Weird, maybe transient error. // Weird, maybe transient error.
agreed: func(entry metaCacheEntry) { agreed: func(entry metaCacheEntry) {
if f.dataUsageCrawlDebug { if f.dataUsageCrawlDebug {
logger.Info(color.Green("healObjects:")+" got agreement: %v", entry.name) console.Debugf(healObjectsPrefix+" got agreement: %v\n", entry.name)
} }
}, },
// Some disks have data for this. // Some disks have data for this.
partial: func(entries metaCacheEntries, nAgreed int, errs []error) { partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
if f.dataUsageCrawlDebug { if f.dataUsageCrawlDebug {
logger.Info(color.Green("healObjects:")+" got partial, %d agreed, errs: %v", nAgreed, errs) console.Debugf(healObjectsPrefix+" got partial, %d agreed, errs: %v\n", nAgreed, errs)
} }
// agreed value less than expected quorum // agreed value less than expected quorum
@ -572,7 +574,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
} }
if f.dataUsageCrawlDebug { if f.dataUsageCrawlDebug {
logger.Info(color.Green("healObjects:")+" resolved to: %v, dir: %v", entry.name, entry.isDir()) console.Debugf(healObjectsPrefix+" resolved to: %v, dir: %v\n", entry.name, entry.isDir())
} }
if entry.isDir() { if entry.isDir() {
@ -610,20 +612,20 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
// Too many disks failed. // Too many disks failed.
finished: func(errs []error) { finished: func(errs []error) {
if f.dataUsageCrawlDebug { if f.dataUsageCrawlDebug {
logger.Info(color.Green("healObjects:")+" too many errors: %v", errs) console.Debugf(healObjectsPrefix+" too many errors: %v\n", errs)
} }
cancel() cancel()
}, },
}) })
if f.dataUsageCrawlDebug && err != nil && err != errFileNotFound { if f.dataUsageCrawlDebug && err != nil && err != errFileNotFound {
logger.Info(color.Green("healObjects:")+" checking returned value %v (%T)", err, err) console.Debugf(healObjectsPrefix+" checking returned value %v (%T)\n", err, err)
} }
// If we found one or more disks with this folder, delete it. // If we found one or more disks with this folder, delete it.
if err == nil && dangling { if err == nil && dangling {
if f.dataUsageCrawlDebug { if f.dataUsageCrawlDebug {
logger.Info(color.Green("healObjects:")+" deleting dangling directory %s", prefix) console.Debugf(healObjectsPrefix+" deleting dangling directory %s\n", prefix)
} }
objAPI.HealObjects(ctx, bucket, prefix, madmin.HealOpts{ objAPI.HealObjects(ctx, bucket, prefix, madmin.HealOpts{
@ -669,6 +671,7 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
var addDir func(entName string, typ os.FileMode) error var addDir func(entName string, typ os.FileMode) error
var dirStack = []string{f.root, folder.name} var dirStack = []string{f.root, folder.name}
deepScannerLogPrefix := color.Green("deep-scanner:")
addDir = func(entName string, typ os.FileMode) error { addDir = func(entName string, typ os.FileMode) error {
select { select {
case <-done: case <-done:
@ -696,7 +699,7 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
var activeLifeCycle *lifecycle.Lifecycle var activeLifeCycle *lifecycle.Lifecycle
if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix, false) { if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix, false) {
if f.dataUsageCrawlDebug { if f.dataUsageCrawlDebug {
logger.Info(color.Green("folder-scanner:")+" Prefix %q has active rules", prefix) console.Debugf(deepScannerLogPrefix+" Prefix %q has active rules\n", prefix)
} }
activeLifeCycle = f.oldCache.Info.lifeCycle activeLifeCycle = f.oldCache.Info.lifeCycle
} }
@ -713,16 +716,21 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
heal: hashPath(path.Join(prefix, entName)).mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure, heal: hashPath(path.Join(prefix, entName)).mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
}) })
if err == errSkipFile {
// Wait to throttle IO // Wait to throttle IO
wait() wait()
if err == errSkipFile {
return nil return nil
} }
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
cache.addSizes(sizeSummary) cache.addSizes(sizeSummary)
cache.Objects++ cache.Objects++
cache.ObjSizes.add(sizeSummary.totalSize) cache.ObjSizes.add(sizeSummary.totalSize)
// Wait to throttle IO
wait()
return nil return nil
} }
err := readDirFn(path.Join(dirStack...), addDir) err := readDirFn(path.Join(dirStack...), addDir)
@ -784,12 +792,13 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
if i.debug { if i.debug {
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
} }
applyActionsLogPrefix := color.Green("applyActions:")
if i.heal { if i.heal {
if i.debug { if i.debug {
if meta.oi.VersionID != "" { if meta.oi.VersionID != "" {
logger.Info(color.Green("applyActions:")+" heal checking: %v/%v v%s", i.bucket, i.objectPath(), meta.oi.VersionID) console.Debugf(applyActionsLogPrefix+" heal checking: %v/%v v(%s)\n", i.bucket, i.objectPath(), meta.oi.VersionID)
} else { } else {
logger.Info(color.Green("applyActions:")+" heal checking: %v/%v", i.bucket, i.objectPath()) console.Debugf(applyActionsLogPrefix+" heal checking: %v/%v\n", i.bucket, i.objectPath())
} }
} }
healOpts := madmin.HealOpts{Remove: healDeleteDangling} healOpts := madmin.HealOpts{Remove: healDeleteDangling}
@ -808,7 +817,7 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
} }
if i.lifeCycle == nil { if i.lifeCycle == nil {
if i.debug { if i.debug {
logger.Info(color.Green("applyActions:")+" no lifecycle rules to apply: %q", i.objectPath()) console.Debugf(applyActionsLogPrefix+" no lifecycle rules to apply: %q\n", i.objectPath())
} }
return size return size
} }
@ -830,9 +839,9 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
}) })
if i.debug { if i.debug {
if versionID != "" { if versionID != "" {
logger.Info(color.Green("applyActions:")+" lifecycle: %q (version-id=%s), Initial scan: %v", i.objectPath(), versionID, action) console.Debugf(applyActionsLogPrefix+" lifecycle: %q (version-id=%s), Initial scan: %v\n", i.objectPath(), versionID, action)
} else { } else {
logger.Info(color.Green("applyActions:")+" lifecycle: %q Initial scan: %v", i.objectPath(), action) console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), action)
} }
} }
switch action { switch action {
@ -842,7 +851,7 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
default: default:
// No action. // No action.
if i.debug { if i.debug {
logger.Info(color.Green("applyActions:")+" object not expirable: %q", i.objectPath()) console.Debugf(applyActionsLogPrefix+" object not expirable: %q\n", i.objectPath())
} }
return size return size
} }
@ -885,7 +894,7 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
} }
action = i.lifeCycle.ComputeAction(lcOpts) action = i.lifeCycle.ComputeAction(lcOpts)
if i.debug { if i.debug {
logger.Info(color.Green("applyActions:")+" lifecycle: Secondary scan: %v", action) console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", action)
} }
switch action { switch action {
case lifecycle.DeleteAction, lifecycle.DeleteVersionAction: case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
@ -908,9 +917,9 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
if locked { if locked {
if i.debug { if i.debug {
if obj.VersionID != "" { if obj.VersionID != "" {
logger.Info(color.Green("applyActions:")+" lifecycle: %s v%s is locked, not deleting", i.objectPath(), obj.VersionID) console.Debugf(applyActionsLogPrefix+" lifecycle: %s v(%s) is locked, not deleting\n", i.objectPath(), obj.VersionID)
} else { } else {
logger.Info(color.Green("applyActions:")+" lifecycle: %s is locked, not deleting", i.objectPath()) console.Debugf(applyActionsLogPrefix+" lifecycle: %s is locked, not deleting\n", i.objectPath())
} }
} }
return size return size

27
cmd/data-update-tracker.go
View File

@ -34,6 +34,7 @@ import (
"github.com/minio/minio/cmd/config" "github.com/minio/minio/cmd/config"
"github.com/minio/minio/cmd/logger" "github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/color" "github.com/minio/minio/pkg/color"
"github.com/minio/minio/pkg/console"
"github.com/minio/minio/pkg/env" "github.com/minio/minio/pkg/env"
"github.com/willf/bloom" "github.com/willf/bloom"
) )
@ -171,16 +172,17 @@ func (d *dataUpdateTracker) current() uint64 {
// latestWithDir returns the highest index that contains the directory. // latestWithDir returns the highest index that contains the directory.
// This means that any cycle higher than this does NOT contain the entry. // This means that any cycle higher than this does NOT contain the entry.
func (d *dataUpdateTracker) latestWithDir(dir string) uint64 { func (d *dataUpdateTracker) latestWithDir(dir string) uint64 {
dateUpdateTrackerLogPrefix := color.Green("dataUpdateTracker:")
bucket, _ := path2BucketObjectWithBasePath("", dir) bucket, _ := path2BucketObjectWithBasePath("", dir)
if bucket == "" { if bucket == "" {
if d.debug && len(dir) > 0 { if d.debug && len(dir) > 0 {
logger.Info(color.Green("dataUpdateTracker:")+" no bucket (%s)", dir) console.Debugf(dateUpdateTrackerLogPrefix+" no bucket (%s)\n", dir)
} }
return d.current() return d.current()
} }
if isReservedOrInvalidBucket(bucket, false) { if isReservedOrInvalidBucket(bucket, false) {
if d.debug { if d.debug {
logger.Info(color.Green("dataUpdateTracker:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, dir) console.Debugf(dateUpdateTrackerLogPrefix+" isReservedOrInvalidBucket: %v, entry: %v\n", bucket, dir)
} }
return d.current() return d.current()
} }
@ -191,7 +193,7 @@ func (d *dataUpdateTracker) latestWithDir(dir string) uint64 {
return d.Current.idx return d.Current.idx
} }
if d.debug { if d.debug {
logger.Info("current bloom does NOT contains dir %s", dir) console.Debugf(dateUpdateTrackerLogPrefix+" current bloom does NOT contains dir %s\n", dir)
} }
idx := d.Current.idx - 1 idx := d.Current.idx - 1
@ -281,7 +283,7 @@ func (d *dataUpdateTracker) startSaver(ctx context.Context, interval time.Durati
d.Saved = UTCNow() d.Saved = UTCNow()
err := d.serialize(&buf) err := d.serialize(&buf)
if d.debug { if d.debug {
logger.Info(color.Green("dataUpdateTracker:")+" Saving: %v bytes, Current idx: %v", buf.Len(), d.Current.idx) console.Debugf(color.Green("dataUpdateTracker:")+" Saving: %v bytes, Current idx: %v\n", buf.Len(), d.Current.idx)
} }
d.dirty = false d.dirty = false
d.mu.Unlock() d.mu.Unlock()
@ -404,7 +406,7 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
} }
switch tmp[0] { switch tmp[0] {
case 1, 2, 3: case 1, 2, 3:
logger.Info(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.") console.Println(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
return nil return nil
case dataUpdateTrackerVersion: case dataUpdateTrackerVersion:
default: default:
@ -481,14 +483,14 @@ func (d *dataUpdateTracker) startCollector(ctx context.Context) {
bucket, _ := path2BucketObjectWithBasePath("", in) bucket, _ := path2BucketObjectWithBasePath("", in)
if bucket == "" { if bucket == "" {
if d.debug && len(in) > 0 { if d.debug && len(in) > 0 {
logger.Info(color.Green("dataUpdateTracker:")+" no bucket (%s)", in) console.Debugf(color.Green("dataUpdateTracker:")+" no bucket (%s)\n", in)
} }
continue continue
} }
if isReservedOrInvalidBucket(bucket, false) { if isReservedOrInvalidBucket(bucket, false) {
if d.debug { if d.debug {
logger.Info(color.Green("dataUpdateTracker:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, in) console.Debugf(color.Green("dataUpdateTracker:")+" isReservedOrInvalidBucket: %v, entry: %v\n", bucket, in)
} }
continue continue
} }
@ -498,7 +500,7 @@ func (d *dataUpdateTracker) startCollector(ctx context.Context) {
d.mu.Lock() d.mu.Lock()
for i := range split { for i := range split {
if d.debug { if d.debug {
logger.Info(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...))) console.Debugln(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
} }
d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String()) d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
} }
@ -510,16 +512,17 @@ func (d *dataUpdateTracker) startCollector(ctx context.Context) {
// markDirty adds the supplied path to the current bloom filter. // markDirty adds the supplied path to the current bloom filter.
func (d *dataUpdateTracker) markDirty(in string) { func (d *dataUpdateTracker) markDirty(in string) {
bucket, _ := path2BucketObjectWithBasePath("", in) bucket, _ := path2BucketObjectWithBasePath("", in)
dateUpdateTrackerLogPrefix := color.Green("dataUpdateTracker:")
if bucket == "" { if bucket == "" {
if d.debug && len(in) > 0 { if d.debug && len(in) > 0 {
logger.Info(color.Green("dataUpdateTracker:")+" no bucket (%s)", in) console.Debugf(dateUpdateTrackerLogPrefix+" no bucket (%s)\n", in)
} }
return return
} }
if isReservedOrInvalidBucket(bucket, false) { if isReservedOrInvalidBucket(bucket, false) {
if d.debug && false { if d.debug && false {
logger.Info(color.Green("dataUpdateTracker:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, in) console.Debugf(dateUpdateTrackerLogPrefix+" isReservedOrInvalidBucket: %v, entry: %v\n", bucket, in)
} }
return return
} }
@ -529,7 +532,7 @@ func (d *dataUpdateTracker) markDirty(in string) {
d.mu.Lock() d.mu.Lock()
for i := range split { for i := range split {
if d.debug { if d.debug {
logger.Info(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...))) console.Debugln(dateUpdateTrackerLogPrefix + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
} }
d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String()) d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
} }
@ -620,7 +623,7 @@ func (d *dataUpdateTracker) cycleFilter(ctx context.Context, req bloomFilterRequ
if d.Current.idx != current { if d.Current.idx != current {
d.dirty = true d.dirty = true
if d.debug { if d.debug {
logger.Info(color.Green("dataUpdateTracker:")+" cycle bloom filter: %v -> %v", d.Current.idx, current) console.Debugf(color.Green("dataUpdateTracker:")+" cycle bloom filter: %v -> %v\n", d.Current.idx, current)
} }
d.History = append(d.History, d.Current) d.History = append(d.History, d.Current)

11
cmd/metacache-set.go
View File

@ -31,6 +31,7 @@ import (
jsoniter "github.com/json-iterator/go" jsoniter "github.com/json-iterator/go"
"github.com/minio/minio/cmd/logger" "github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/color"
"github.com/minio/minio/pkg/console" "github.com/minio/minio/pkg/console"
"github.com/minio/minio/pkg/hash" "github.com/minio/minio/pkg/hash"
) )
@ -126,7 +127,7 @@ func (o listPathOptions) newMetacache() metacache {
func (o *listPathOptions) debugf(format string, data ...interface{}) { func (o *listPathOptions) debugf(format string, data ...interface{}) {
if serverDebugLog { if serverDebugLog {
console.Debugf(format, data...) console.Debugf(format+"\n", data...)
} }
} }
@ -545,7 +546,7 @@ func (er erasureObjects) SetDriveCount() int {
// Will return io.EOF if continuing would not yield more results. // Will return io.EOF if continuing would not yield more results.
func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions) (entries metaCacheEntriesSorted, err error) { func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions) (entries metaCacheEntriesSorted, err error) {
o.debugf("listPath with options: %#v\n", o) o.debugf(color.Green("listPath:")+" with options: %#v", o)
// See if we have the listing stored. // See if we have the listing stored.
if !o.Create && !o.discardResult { if !o.Create && !o.discardResult {
@ -571,7 +572,7 @@ func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions) (entr
rpc := globalNotificationSys.restClientFromHash(o.Bucket) rpc := globalNotificationSys.restClientFromHash(o.Bucket)
var metaMu sync.Mutex var metaMu sync.Mutex
o.debugln("listPath: scanning bucket:", o.Bucket, "basedir:", o.BaseDir, "prefix:", o.Prefix, "marker:", o.Marker) o.debugln(color.Green("listPath:")+" scanning bucket:", o.Bucket, "basedir:", o.BaseDir, "prefix:", o.Prefix, "marker:", o.Marker)
// Disconnect from call above, but cancel on exit. // Disconnect from call above, but cancel on exit.
ctx, cancel := context.WithCancel(GlobalContext) ctx, cancel := context.WithCancel(GlobalContext)
@ -579,7 +580,7 @@ func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions) (entr
disks := er.getOnlineDisks() disks := er.getOnlineDisks()
defer func() { defer func() {
o.debugln("listPath returning:", entries.len(), "err:", err) o.debugln(color.Green("listPath:")+" returning:", entries.len(), "err:", err)
if err != nil && !errors.Is(err, io.EOF) { if err != nil && !errors.Is(err, io.EOF) {
go func(err string) { go func(err string) {
metaMu.Lock() metaMu.Lock()
@ -674,7 +675,7 @@ func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions) (entr
if len(b.data) == 0 && b.n == 0 && o.Transient { if len(b.data) == 0 && b.n == 0 && o.Transient {
return nil return nil
} }
o.debugln("listPath: saving block", b.n, "to", o.objectPath(b.n)) o.debugln(color.Green("listPath:")+" saving block", b.n, "to", o.objectPath(b.n))
r, err := hash.NewReader(bytes.NewReader(b.data), int64(len(b.data)), "", "", int64(len(b.data)), false) r, err := hash.NewReader(bytes.NewReader(b.data), int64(len(b.data)), "", "", int64(len(b.data)), false)
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
custom := b.headerKV() custom := b.headerKV()

7
cmd/xl-storage.go
View File

@ -47,6 +47,7 @@ import (
"github.com/minio/minio/cmd/logger" "github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/bucket/lifecycle" "github.com/minio/minio/pkg/bucket/lifecycle"
"github.com/minio/minio/pkg/color" "github.com/minio/minio/pkg/color"
"github.com/minio/minio/pkg/console"
"github.com/minio/minio/pkg/disk" "github.com/minio/minio/pkg/disk"
"github.com/minio/minio/pkg/env" "github.com/minio/minio/pkg/env"
xioutil "github.com/minio/minio/pkg/ioutil" xioutil "github.com/minio/minio/pkg/ioutil"
@ -340,7 +341,7 @@ func (s *xlStorage) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCac
if err == nil && lc.HasActiveRules("", true) { if err == nil && lc.HasActiveRules("", true) {
cache.Info.lifeCycle = lc cache.Info.lifeCycle = lc
if intDataUpdateTracker.debug { if intDataUpdateTracker.debug {
logger.Info(color.Green("crawlDisk:") + " lifecycle: Active rules found") console.Debugln(color.Green("crawlDisk:") + " lifecycle: Active rules found")
} }
} }
} }
@ -363,7 +364,7 @@ func (s *xlStorage) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCac
buf, err := ioutil.ReadFile(item.Path) buf, err := ioutil.ReadFile(item.Path)
if err != nil { if err != nil {
if intDataUpdateTracker.debug { if intDataUpdateTracker.debug {
logger.Info(color.Green("crawlBucket:")+" object path missing: %v: %w", item.Path, err) console.Debugf(color.Green("crawlBucket:")+" object path missing: %v: %w\n", item.Path, err)
} }
return sizeSummary{}, errSkipFile return sizeSummary{}, errSkipFile
} }
@ -374,7 +375,7 @@ func (s *xlStorage) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCac
fivs, err := getFileInfoVersions(buf, item.bucket, item.objectPath()) fivs, err := getFileInfoVersions(buf, item.bucket, item.objectPath())
if err != nil { if err != nil {
if intDataUpdateTracker.debug { if intDataUpdateTracker.debug {
logger.Info(color.Green("crawlBucket:")+" reading xl.meta failed: %v: %w", item.Path, err) console.Debugf(color.Green("crawlBucket:")+" reading xl.meta failed: %v: %w\n", item.Path, err)
} }
return sizeSummary{}, errSkipFile return sizeSummary{}, errSkipFile
} }