Add crawler delay config + dynamic config values (#11018)

This commit is contained in:
Klaus Post
2020-12-04 09:32:35 -08:00
committed by GitHub
parent e083471ec4
commit a896125490
19 changed files with 440 additions and 89 deletions

View File

@@ -436,7 +436,10 @@ func (o ObjectInfo) GetActualSize() (int64, error) {
// Using compression and encryption together enables room for side channel attacks.
// Eliminate non-compressible objects by extensions/content-types.
func isCompressible(header http.Header, object string) bool {
if crypto.IsRequested(header) || excludeForCompression(header, object, globalCompressConfig) {
globalCompressConfigMu.Lock()
cfg := globalCompressConfig
globalCompressConfigMu.Unlock()
if !cfg.Enabled || crypto.IsRequested(header) || excludeForCompression(header, object, cfg) {
return false
}
return true