fix size accounting for encrypted/compressed objects (#9690)

size calculation in crawler was using the real size
of the object instead of its actual size i.e either
a decrypted or uncompressed size.

this is needed to make sure all other accounting
such as bucket quota and mcs UI to display the
correct values.

cmd/api-headers.go

@@ -120,20 +120,9 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		w.Header().Set(k, v)
 	}
 
-	var totalObjectSize int64
+	totalObjectSize, err := objInfo.GetActualSize()
-	switch {
+	if err != nil {
-	case crypto.IsEncrypted(objInfo.UserDefined):
+		return err
-		totalObjectSize, err = objInfo.DecryptedSize()
-		if err != nil {
-			return err
-		}
-	case objInfo.IsCompressed():
-		totalObjectSize = objInfo.GetActualSize()
-		if totalObjectSize < 0 {
-			return errInvalidDecompressedSize
-		}
-	default:
-		totalObjectSize = objInfo.Size
 	}
 
 	// for providing ranged content

cmd/bucket-listobjects-handlers.go

@@ -99,24 +99,13 @@ func (api objectAPIHandlers) ListBucketObjectVersionsHandler(w http.ResponseWrit
 	}
 
 	for i := range listObjectsInfo.Objects {
-		var actualSize int64
+		if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
-		if listObjectsInfo.Objects[i].IsCompressed() {
-			// Read the decompressed size from the meta.json.
-			actualSize = listObjectsInfo.Objects[i].GetActualSize()
-			if actualSize < 0 {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDecompressedSize),
-					r.URL, guessIsBrowserReq(r))
-				return
-			}
-			// Set the info.Size to the actualSize.
-			listObjectsInfo.Objects[i].Size = actualSize
-		} else if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
 			listObjectsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsInfo.Objects[i], false)
-			listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].DecryptedSize()
+		}
-			if err != nil {
+		listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].GetActualSize()
-				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		if err != nil {
-				return
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			}
+			return
 		}
 	}
 
@@ -181,23 +170,13 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 	}
 
 	for i := range listObjectsV2Info.Objects {
-		var actualSize int64
+		if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
-		if listObjectsV2Info.Objects[i].IsCompressed() {
-			// Read the decompressed size from the meta.json.
-			actualSize = listObjectsV2Info.Objects[i].GetActualSize()
-			if actualSize < 0 {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDecompressedSize), r.URL, guessIsBrowserReq(r))
-				return
-			}
-			// Set the info.Size to the actualSize.
-			listObjectsV2Info.Objects[i].Size = actualSize
-		} else if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
 			listObjectsV2Info.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsV2Info.Objects[i], false)
-			listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].DecryptedSize()
+		}
-			if err != nil {
+		listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].GetActualSize()
-				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		if err != nil {
-				return
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			}
+			return
 		}
 	}
 
@@ -265,23 +244,13 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	}
 
 	for i := range listObjectsV2Info.Objects {
-		var actualSize int64
+		if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
-		if listObjectsV2Info.Objects[i].IsCompressed() {
-			// Read the decompressed size from the meta.json.
-			actualSize = listObjectsV2Info.Objects[i].GetActualSize()
-			if actualSize < 0 {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDecompressedSize), r.URL, guessIsBrowserReq(r))
-				return
-			}
-			// Set the info.Size to the actualSize.
-			listObjectsV2Info.Objects[i].Size = actualSize
-		} else if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
 			listObjectsV2Info.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsV2Info.Objects[i], false)
-			listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].DecryptedSize()
+		}
-			if err != nil {
+		listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].GetActualSize()
-				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		if err != nil {
-				return
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			}
+			return
 		}
 	}
 
@@ -344,25 +313,16 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	}
 
 	for i := range listObjectsInfo.Objects {
-		var actualSize int64
+		if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
-		if listObjectsInfo.Objects[i].IsCompressed() {
-			// Read the decompressed size from the meta.json.
-			actualSize = listObjectsInfo.Objects[i].GetActualSize()
-			if actualSize < 0 {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDecompressedSize), r.URL, guessIsBrowserReq(r))
-				return
-			}
-			// Set the info.Size to the actualSize.
-			listObjectsInfo.Objects[i].Size = actualSize
-		} else if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
 			listObjectsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsInfo.Objects[i], false)
-			listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].DecryptedSize()
+		}
-			if err != nil {
+		listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].GetActualSize()
-				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		if err != nil {
-				return
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			}
+			return
 		}
 	}
+
 	response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType, maxKeys, listObjectsInfo)
 
 	// Write success response.

cmd/notification.go

@@ -1260,9 +1260,6 @@ func (args eventArgs) ToEvent(escape bool) event.Event {
 	if args.EventName != event.ObjectRemovedDelete {
 		newEvent.S3.Object.ETag = args.Object.ETag
 		newEvent.S3.Object.Size = args.Object.Size
-		if args.Object.IsCompressed() {
-			newEvent.S3.Object.Size = args.Object.GetActualSize()
-		}
 		newEvent.S3.Object.ContentType = args.Object.ContentType
 		newEvent.S3.Object.UserMetadata = args.Object.UserDefined
 	}
@@ -1271,16 +1268,9 @@ func (args eventArgs) ToEvent(escape bool) event.Event {
 }
 
 func sendEvent(args eventArgs) {
-	// remove sensitive encryption entries in metadata.
+	args.Object.Size, _ = args.Object.GetActualSize()
-	switch {
-	case crypto.IsEncrypted(args.Object.UserDefined):
-		if totalObjectSize, err := args.Object.DecryptedSize(); err == nil {
-			args.Object.Size = totalObjectSize
-		}
-	case args.Object.IsCompressed():
-		args.Object.Size = args.Object.GetActualSize()
-	}
 
+	// remove sensitive encryption entries in metadata.
 	crypto.RemoveSensitiveEntries(args.Object.UserDefined)
 	crypto.RemoveInternalEntries(args.Object.UserDefined)
 

cmd/object-api-utils.go

@@ -372,17 +372,23 @@ func (o ObjectInfo) IsCompressedOK() (bool, error) {
 	return true, fmt.Errorf("unknown compression scheme: %s", scheme)
 }
 
-// GetActualSize - read the decompressed size from the meta json.
+// GetActualSize - returns the actual size of the stored object
-func (o ObjectInfo) GetActualSize() int64 {
+func (o ObjectInfo) GetActualSize() (int64, error) {
-	metadata := o.UserDefined
+	if crypto.IsEncrypted(o.UserDefined) {
-	sizeStr, ok := metadata[ReservedMetadataPrefix+"actual-size"]
+		return o.DecryptedSize()
-	if ok {
-		size, err := strconv.ParseInt(sizeStr, 10, 64)
-		if err == nil {
-			return size
-		}
 	}
-	return -1
+	if o.IsCompressed() {
+		sizeStr, ok := o.UserDefined[ReservedMetadataPrefix+"actual-size"]
+		if !ok {
+			return -1, errInvalidDecompressedSize
+		}
+		size, err := strconv.ParseInt(sizeStr, 10, 64)
+		if err != nil {
+			return -1, errInvalidDecompressedSize
+		}
+		return size, nil
+	}
+	return o.Size, nil
 }
 
 // Disabling compression for encrypted enabled requests.
@@ -608,9 +614,9 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions, cl
 		}
 	case isCompressed:
 		// Read the decompressed size from the meta.json.
-		actualSize := oi.GetActualSize()
+		actualSize, err := oi.GetActualSize()
-		if actualSize < 0 {
+		if err != nil {
-			return nil, 0, 0, errInvalidDecompressedSize
+			return nil, 0, 0, err
 		}
 		off, length = int64(0), oi.Size
 		decOff, decLength := int64(0), actualSize

cmd/object-api-utils_test.go

@@ -521,7 +521,7 @@ func TestGetActualSize(t *testing.T) {
 		},
 	}
 	for i, test := range testCases {
-		got := test.objInfo.GetActualSize()
+		got, _ := test.objInfo.GetActualSize()
 		if got != test.result {
 			t.Errorf("Test %d - expected %d but received %d",
 				i+1, test.result, got)

cmd/object-handlers.go

@@ -1188,10 +1188,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	// Write success response.
 	writeSuccessResponseXML(w, encodedSuccessResponse)
 
-	if objInfo.IsCompressed() {
-		objInfo.Size = actualSize
-	}
-
 	// Notify object created event.
 	sendEvent(eventArgs{
 		EventName:    event.ObjectCreatedCopy,
@@ -1508,6 +1504,9 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 
 	writeSuccessResponseHeadersOnly(w)
 
+	// Set the etag sent to the client as part of the event.
+	objInfo.ETag = etag
+
 	// Notify object created event.
 	sendEvent(eventArgs{
 		EventName:    event.ObjectCreatedPut,

cmd/posix.go

@@ -383,7 +383,15 @@ func (s *posix) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache)
 			return 0, nil
 		}
 
-		return meta.Stat.Size, nil
+		// we don't necessarily care about the names
+		// of bucket and object, only interested in size.
+		// so use some dummy names.
+		size, err := meta.ToObjectInfo("dummy", "dummy").GetActualSize()
+		if err != nil {
+			return 0, errSkipFile
+		}
+		return size, nil
+
 	})
 	if err != nil {
 		return dataUsageInfo, err

cmd/web-handlers.go

@@ -529,17 +529,9 @@ func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, r
 			return &json2.Error{Message: err.Error()}
 		}
 		for i := range lo.Objects {
-			if crypto.IsEncrypted(lo.Objects[i].UserDefined) {
+			lo.Objects[i].Size, err = lo.Objects[i].GetActualSize()
-				lo.Objects[i].Size, err = lo.Objects[i].DecryptedSize()
+			if err != nil {
-				if err != nil {
+				return toJSONError(ctx, err)
-					return toJSONError(ctx, err)
-				}
-			} else if lo.Objects[i].IsCompressed() {
-				actualSize := lo.Objects[i].GetActualSize()
-				if actualSize < 0 {
-					return toJSONError(ctx, errInvalidDecompressedSize)
-				}
-				lo.Objects[i].Size = actualSize
 			}
 		}
 
@@ -1505,10 +1497,10 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 			info := gr.ObjInfo
 			// filter object lock metadata if permission does not permit
 			info.UserDefined = objectlock.FilterObjectLockMetadata(info.UserDefined, getRetPerms[i] != ErrNone, legalHoldPerms[i] != ErrNone)
-
+			// For reporting, set the file size to the uncompressed size.
-			if info.IsCompressed() {
+			info.Size, err = info.GetActualSize()
-				// For reporting, set the file size to the uncompressed size.
+			if err != nil {
-				info.Size = info.GetActualSize()
+				return err
 			}
 			header := &zip.FileHeader{
 				Name:     strings.TrimPrefix(objectName, args.Prefix),

go.mod

@@ -106,7 +106,7 @@ require (
 	github.com/stretchr/testify v1.5.1 // indirect
 	github.com/tinylib/msgp v1.1.1
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
-	github.com/ugorji/go/codec v1.1.5-pre // indirect
+	github.com/ugorji/go v1.1.5-pre // indirect
 	github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a
 	github.com/willf/bitset v1.1.10 // indirect
 	github.com/willf/bloom v2.0.3+incompatible