MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-20 09:56:07 -05:00
Code Issues Packages Projects Releases Wiki Activity

re-implement data usage crawler to be more efficient (#9075)

Browse Source 
Implementation overview: 

https://gist.github.com/klauspost/1801c858d5e0df391114436fdad6987b
This commit is contained in:
Klaus Post
2020-03-19 00:19:29 +01:00
committed by GitHub
parent 7fdeb44372
commit 8d98662633
61 changed files with 2895 additions and 543 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
cmd/storage-rest-client.go
View File

@@ -17,7 +17,6 @@
package cmd
import (
"bufio"
"bytes"
"context"
"crypto/tls"
@@ -152,26 +151,25 @@ func (client *storageRESTClient) Hostname() string {
return client.endpoint.Host
}
func (client *storageRESTClient) CrawlAndGetDataUsage(endCh <-chan struct{}) (DataUsageInfo, error) {
respBody, err := client.call(storageRESTMethodCrawlAndGetDataUsage, nil, nil, -1)
func (client *storageRESTClient) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache) (dataUsageCache, error) {
b := cache.serialize()
respBody, err := client.call(storageRESTMethodCrawlAndGetDataUsage,
url.Values{},
bytes.NewBuffer(b), int64(len(b)))
defer http.DrainBody(respBody)
if err != nil {
return DataUsageInfo{}, err
return cache, err
}
reader := bufio.NewReader(respBody)
for {
b, err := reader.ReadByte()
if err != nil {
return DataUsageInfo{}, err
}
if b != ' ' {
reader.UnreadByte()
break
}
reader, err := waitForHTTPResponse(respBody)
if err != nil {
return cache, err
}
var usageInfo DataUsageInfo
err = gob.NewDecoder(reader).Decode(&usageInfo)
return usageInfo, err
b, err = ioutil.ReadAll(reader)
if err != nil {
return cache, err
}
var newCache dataUsageCache
return newCache, newCache.deserialize(b)
}
func (client *storageRESTClient) SetDiskID(id string) {
@@ -418,7 +416,7 @@ func (client *storageRESTClient) DeleteFileBulk(volume string, paths []string) (
return nil, err
}
reader, err := clearLeadingSpaces(respBody)
reader, err := waitForHTTPResponse(respBody)
if err != nil {
return nil, err
}
@@ -455,7 +453,7 @@ func (client *storageRESTClient) DeletePrefixes(volume string, paths []string) (
return nil, err
}
reader, err := clearLeadingSpaces(respBody)
reader, err := waitForHTTPResponse(respBody)
if err != nil {
return nil, err
}
@@ -484,22 +482,6 @@ func (client *storageRESTClient) RenameFile(srcVolume, srcPath, dstVolume, dstPa
return err
}
// clearLeadingSpaces removes all the first spaces returned from a reader.
func clearLeadingSpaces(r io.Reader) (io.Reader, error) {
reader := bufio.NewReader(r)
for {
b, err := reader.ReadByte()
if err != nil {
return nil, err
}
if b != ' ' {
reader.UnreadByte()
break
}
}
return reader, nil
}
func (client *storageRESTClient) VerifyFile(volume, path string, size int64, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
@@ -514,7 +496,7 @@ func (client *storageRESTClient) VerifyFile(volume, path string, size int64, alg
if err != nil {
return err
}
reader, err := clearLeadingSpaces(respBody)
reader, err := waitForHTTPResponse(respBody)
if err != nil {
return err
}