re-implement data usage crawler to be more efficient (#9075)

Implementation overview: 

https://gist.github.com/klauspost/1801c858d5e0df391114436fdad6987b
This commit is contained in:
Klaus Post
2020-03-19 00:19:29 +01:00
committed by GitHub
parent 7fdeb44372
commit 8d98662633
61 changed files with 2895 additions and 543 deletions

View File

@@ -101,28 +101,24 @@ type HTTPAPIStats struct {
// Inc increments the api stats counter.
func (stats *HTTPAPIStats) Inc(api string) {
stats.Lock()
defer stats.Unlock()
if stats == nil {
return
}
stats.Lock()
defer stats.Unlock()
if stats.apiStats == nil {
stats.apiStats = make(map[string]int)
}
if _, ok := stats.apiStats[api]; ok {
stats.apiStats[api]++
return
}
stats.apiStats[api] = 1
stats.apiStats[api]++
}
// Dec increments the api stats counter.
func (stats *HTTPAPIStats) Dec(api string) {
stats.Lock()
defer stats.Unlock()
if stats == nil {
return
}
stats.Lock()
defer stats.Unlock()
if val, ok := stats.apiStats[api]; ok && val > 0 {
stats.apiStats[api]--
}