diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index f7b221446..cef455ee1 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -1835,45 +1835,86 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re return } + send := func(objInfo ObjectInfo) bool { + select { + case <-ctx.Done(): + return false + case results <- objInfo: + return true + } + } + + askDisks := getListQuorum(opts.WalkAskDisks, set.setDriveCount) + var fallbackDisks []StorageAPI + + // Special case: ask all disks if the drive count is 4 + if set.setDriveCount == 4 || askDisks > len(disks) { + askDisks = len(disks) // use all available drives + } + + if askDisks > 0 && len(disks) > askDisks { + fallbackDisks = disks[askDisks:] + disks = disks[:askDisks] + } + + requestedVersions := 0 + if opts.WalkLatestOnly { + requestedVersions = 1 + } loadEntry := func(entry metaCacheEntry) { if entry.isDir() { return } - fivs, err := entry.fileInfoVersions(bucket) - if err != nil { - cancel() - return - } - - versionsSorter(fivs.Versions).reverse() - - for _, version := range fivs.Versions { - send := true - if opts.WalkFilter != nil && !opts.WalkFilter(version) { - send = false - } - - if !send { - continue - } - - versioned := vcfg != nil && vcfg.Versioned(version.Name) - objInfo := version.ToObjectInfo(bucket, version.Name, versioned) - - select { - case <-ctx.Done(): + if opts.WalkLatestOnly { + fi, err := entry.fileInfo(bucket) + if err != nil { + cancel() return - case results <- objInfo: + } + if opts.WalkFilter != nil { + if opts.WalkFilter(fi) { + if !send(fi.ToObjectInfo(bucket, fi.Name, vcfg != nil && vcfg.Versioned(fi.Name))) { + return + } + } + } else { + if !send(fi.ToObjectInfo(bucket, fi.Name, vcfg != nil && vcfg.Versioned(fi.Name))) { + return + } + } + + } else { + fivs, err := entry.fileInfoVersions(bucket) + if err != nil { + cancel() + return + } + + versionsSorter(fivs.Versions).reverse() + + for _, version := range fivs.Versions { + if opts.WalkFilter != nil { + if opts.WalkFilter(version) { + if !send(version.ToObjectInfo(bucket, version.Name, vcfg != nil && vcfg.Versioned(version.Name))) { + return + } + } + } else { + if !send(version.ToObjectInfo(bucket, version.Name, vcfg != nil && vcfg.Versioned(version.Name))) { + return + } + } } } } // How to resolve partial results. resolver := metadataResolutionParams{ - dirQuorum: 1, - objQuorum: 1, - bucket: bucket, + dirQuorum: 1, + objQuorum: 1, + bucket: bucket, + requestedVersions: requestedVersions, } path := baseDirFromPrefix(prefix) @@ -1884,6 +1925,7 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re lopts := listPathRawOptions{ disks: disks, + fallbackDisks: fallbackDisks, bucket: bucket, path: path, filterPrefix: filterPrefix, diff --git a/cmd/metacache-set.go b/cmd/metacache-set.go index f3a1d83b5..7e52793b9 100644 --- a/cmd/metacache-set.go +++ b/cmd/metacache-set.go @@ -594,14 +594,11 @@ func getListQuorum(quorum string, driveCount int) int { return 1 case "reduced": return 2 - case "strict": - return driveCount - } - // Defaults to (driveCount+1)/2 drives per set, defaults to "optimal" value - if driveCount > 0 { + case "optimal": return (driveCount + 1) / 2 - } // "3" otherwise. - return 3 + } + // defaults to 'strict' + return driveCount } // Will return io.EOF if continuing would not yield more results. diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go index ab08775db..2611f719c 100644 --- a/cmd/object-api-interface.go +++ b/cmd/object-api-interface.go @@ -97,6 +97,8 @@ type ObjectOptions struct { WalkFilter func(info FileInfo) bool // return WalkFilter returns 'true/false' WalkMarker string // set to skip until this object + WalkLatestOnly bool // returns only latest versions for all matching objects + WalkAskDisks string // dictates how many disks are being listed PrefixEnabledFn func(prefix string) bool // function which returns true if versioning is enabled on prefix // IndexCB will return any index created but the compression.