MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 12:52:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

ListObjects Metadata Caching (#10648)

Browse Source 
Design: https://gist.github.com/klauspost/025c09b48ed4a1293c917cecfabdf21c

Gist of improvements:

* Cross-server caching and listing will use the same data across servers and requests.
* Lists can be arbitrarily resumed at a constant speed.
* Metadata for all files scanned is stored for streaming retrieval.
* The existing bloom filters controlled by the crawler is used for validating caches.
* Concurrent requests for the same data (or parts of it) will not spawn additional walkers.
* Listing a subdirectory of an existing recursive cache will use the cache.
* All listing operations are fully streamable so the number of objects in a bucket no 
  longer dictates the amount of memory.
* Listings can be handled by any server within the cluster.
* Caches are cleaned up when out of date or superseded by a more recent one.
This commit is contained in:
Klaus Post
2020-10-28 09:18:35 -07:00
committed by GitHub
parent 51222cc664
commit a982baff27
65 changed files with 6328 additions and 742 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
cmd/naughty-disk_test.go
View File

@@ -143,6 +143,13 @@ func (d *naughtyDisk) DeleteVol(ctx context.Context, volume string, forceDelete
return d.disk.DeleteVol(ctx, volume, forceDelete)
}
func (d *naughtyDisk) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writer) error {
if err := d.calcError(); err != nil {
return err
}
return d.disk.WalkDir(ctx, opts, wr)
}
func (d *naughtyDisk) WalkSplunk(ctx context.Context, volume, dirPath, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error) {
if err := d.calcError(); err != nil {
return nil, err
@@ -227,11 +234,11 @@ func (d *naughtyDisk) CheckFile(ctx context.Context, volume string, path string)
return d.disk.CheckFile(ctx, volume, path)
}
func (d *naughtyDisk) DeleteFile(ctx context.Context, volume string, path string) (err error) {
func (d *naughtyDisk) Delete(ctx context.Context, volume string, path string, recursive bool) (err error) {
if err := d.calcError(); err != nil {
return err
}
return d.disk.DeleteFile(ctx, volume, path)
return d.disk.Delete(ctx, volume, path, recursive)
}
func (d *naughtyDisk) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) []error {