fix size accounting for encrypted/compressed objects (#9690)

size calculation in crawler was using the real size
of the object instead of its actual size i.e either
a decrypted or uncompressed size.

this is needed to make sure all other accounting
such as bucket quota and mcs UI to display the
correct values.
This commit is contained in:
Harshavardhana
2020-05-24 11:19:17 -07:00
committed by GitHub
parent bc285cf0dd
commit 0c71ce3398
9 changed files with 70 additions and 126 deletions

View File

@@ -372,17 +372,23 @@ func (o ObjectInfo) IsCompressedOK() (bool, error) {
return true, fmt.Errorf("unknown compression scheme: %s", scheme)
}
// GetActualSize - read the decompressed size from the meta json.
func (o ObjectInfo) GetActualSize() int64 {
metadata := o.UserDefined
sizeStr, ok := metadata[ReservedMetadataPrefix+"actual-size"]
if ok {
size, err := strconv.ParseInt(sizeStr, 10, 64)
if err == nil {
return size
}
// GetActualSize - returns the actual size of the stored object
func (o ObjectInfo) GetActualSize() (int64, error) {
if crypto.IsEncrypted(o.UserDefined) {
return o.DecryptedSize()
}
return -1
if o.IsCompressed() {
sizeStr, ok := o.UserDefined[ReservedMetadataPrefix+"actual-size"]
if !ok {
return -1, errInvalidDecompressedSize
}
size, err := strconv.ParseInt(sizeStr, 10, 64)
if err != nil {
return -1, errInvalidDecompressedSize
}
return size, nil
}
return o.Size, nil
}
// Disabling compression for encrypted enabled requests.
@@ -608,9 +614,9 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions, cl
}
case isCompressed:
// Read the decompressed size from the meta.json.
actualSize := oi.GetActualSize()
if actualSize < 0 {
return nil, 0, 0, errInvalidDecompressedSize
actualSize, err := oi.GetActualSize()
if err != nil {
return nil, 0, 0, err
}
off, length = int64(0), oi.Size
decOff, decLength := int64(0), actualSize