re-implement data usage crawler to be more efficient (#9075)

Implementation overview: 

https://gist.github.com/klauspost/1801c858d5e0df391114436fdad6987b
This commit is contained in:
Klaus Post
2020-03-19 00:19:29 +01:00
committed by GitHub
parent 7fdeb44372
commit 8d98662633
61 changed files with 2895 additions and 543 deletions

View File

@@ -274,7 +274,7 @@ func (e BucketSSEConfigNotFound) Error() string {
// BucketNameInvalid - bucketname provided is invalid.
type BucketNameInvalid GenericError
// Return string an error formatted as the given text.
// Error returns string an error formatted as the given text.
func (e BucketNameInvalid) Error() string {
return "Bucket name invalid: " + e.Bucket
}
@@ -290,17 +290,17 @@ type ObjectNameTooLong GenericError
// ObjectNamePrefixAsSlash - object name has a slash as prefix.
type ObjectNamePrefixAsSlash GenericError
// Return string an error formatted as the given text.
// Error returns string an error formatted as the given text.
func (e ObjectNameInvalid) Error() string {
return "Object name invalid: " + e.Bucket + "#" + e.Object
}
// Return string an error formatted as the given text.
// Error returns string an error formatted as the given text.
func (e ObjectNameTooLong) Error() string {
return "Object name too long: " + e.Bucket + "#" + e.Object
}
// Return string an error formatted as the given text.
// Error returns string an error formatted as the given text.
func (e ObjectNamePrefixAsSlash) Error() string {
return "Object name contains forward slash as pefix: " + e.Bucket + "#" + e.Object
}
@@ -308,7 +308,7 @@ func (e ObjectNamePrefixAsSlash) Error() string {
// AllAccessDisabled All access to this object has been disabled
type AllAccessDisabled GenericError
// Return string an error formatted as the given text.
// Error returns string an error formatted as the given text.
func (e AllAccessDisabled) Error() string {
return "All access to this object has been disabled"
}
@@ -316,7 +316,7 @@ func (e AllAccessDisabled) Error() string {
// IncompleteBody You did not provide the number of bytes specified by the Content-Length HTTP header.
type IncompleteBody GenericError
// Return string an error formatted as the given text.
// Error returns string an error formatted as the given text.
func (e IncompleteBody) Error() string {
return e.Bucket + "#" + e.Object + "has incomplete body"
}