Time getSize and use to estimate latency (#8959)

Remove the random sleep. This is running in 4 goroutines, 
so mostly doing nothing.

We use the getSize latency to estimate system load, 
meaning when there is little load on the system and 
we get the result fast we sleep a little.

If it took a long time we have high load and release
ourselves longer.

We are sleeping inside the mutex so this affects all
goroutines doing IO.
This commit is contained in:
Klaus Post 2020-02-06 19:35:55 -08:00 committed by GitHub
parent 6d5d77f62c
commit 2165d45d3f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -20,7 +20,6 @@ import (
"bytes"
"context"
"encoding/json"
"math/rand"
"os"
"path/filepath"
"sync"
@ -160,19 +159,12 @@ func updateUsage(basePath string, doneCh <-chan struct{}, waitForLowActiveIO fun
}
numWorkers := 4
walkInterval := 1 * time.Millisecond
var mutex sync.Mutex // Mutex to update dataUsageInfo
r := rand.New(rand.NewSource(UTCNow().UnixNano()))
fastWalk(basePath, numWorkers, doneCh, func(path string, typ os.FileMode) error {
// Wait for I/O to go down.
waitForLowActiveIO()
// Randomize sleep intervals, to stagger the walk.
defer time.Sleep(time.Duration(r.Float64() * float64(walkInterval)))
bucket, entry := path2BucketObjectWithBasePath(basePath, path)
if bucket == "" {
return nil
@ -197,7 +189,13 @@ func updateUsage(basePath string, doneCh <-chan struct{}, waitForLowActiveIO fun
return nil
}
t := time.Now()
size, err := getSize(Item{path, typ})
// Use the response time of the getSize call to guess system load.
// Sleep equivalent time.
if d := time.Since(t); d > 100*time.Microsecond {
time.Sleep(d)
}
if err != nil {
return errSkipFile
}