diff --git a/cmd/data-scanner.go b/cmd/data-scanner.go index b03197c64..04501007a 100644 --- a/cmd/data-scanner.go +++ b/cmd/data-scanner.go @@ -62,11 +62,12 @@ var ( globalHealConfig heal.Config // Sleeper values are updated when config is loaded. - scannerSleeper = newDynamicSleeper(2, time.Second, true) // Keep defaults same as config defaults - scannerCycle = uatomic.NewDuration(dataScannerStartDelay) - scannerIdleMode = uatomic.NewInt32(0) // default is throttled when idle - scannerExcessObjectVersions = uatomic.NewInt64(100) - scannerExcessFolders = uatomic.NewInt64(50000) + scannerSleeper = newDynamicSleeper(2, time.Second, true) // Keep defaults same as config defaults + scannerCycle = uatomic.NewDuration(dataScannerStartDelay) + scannerIdleMode = uatomic.NewInt32(0) // default is throttled when idle + scannerExcessObjectVersions = uatomic.NewInt64(100) + scannerExcessObjectVersionsTotalSize = uatomic.NewInt64(1024 * 1024 * 1024 * 1024) // 1 TB + scannerExcessFolders = uatomic.NewInt64(50000) ) // initDataScanner will start the scanner in the background. @@ -1065,7 +1066,7 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi } // Check if we have many versions after applyNewerNoncurrentVersionLimit. - if len(objInfos) > int(scannerExcessObjectVersions.Load()) { + if len(objInfos) >= int(scannerExcessObjectVersions.Load()) { // Notify object accessed via a GET request. sendEvent(eventArgs{ EventName: event.ObjectManyVersions, @@ -1089,6 +1090,39 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi }) } + cumulativeSize := int64(0) + for _, objInfo := range objInfos { + cumulativeSize += objInfo.Size + } + // Check if the cumulative size of all versions of this object is high. + if cumulativeSize >= scannerExcessObjectVersionsTotalSize.Load() { + // Notify object accessed via a GET request. + sendEvent(eventArgs{ + EventName: event.ObjectLargeVersions, + BucketName: i.bucket, + Object: ObjectInfo{ + Name: i.objectPath(), + }, + UserAgent: "Scanner", + Host: globalLocalNodeName, + RespElements: map[string]string{ + "x-minio-versions-count": strconv.Itoa(len(objInfos)), + "x-minio-versions-size": strconv.FormatInt(cumulativeSize, 10), + }, + }) + + auditLogInternal(context.Background(), AuditLogOptions{ + Event: "scanner:largeversions", + APIName: "Scanner", + Bucket: i.bucket, + Object: i.objectPath(), + Tags: map[string]interface{}{ + "x-minio-versions-count": strconv.Itoa(len(objInfos)), + "x-minio-versions-size": strconv.FormatInt(cumulativeSize, 10), + }, + }) + } + return objInfos, nil } diff --git a/internal/event/name.go b/internal/event/name.go index df0096740..1e52621ce 100644 --- a/internal/event/name.go +++ b/internal/event/name.go @@ -61,6 +61,7 @@ const ( ObjectTransitionFailed ObjectTransitionComplete ObjectManyVersions + ObjectLargeVersions PrefixManyFolders objectSingleTypesEnd @@ -124,6 +125,7 @@ func (name Name) Expand() []Name { case ObjectScannerAll: return []Name{ ObjectManyVersions, + ObjectLargeVersions, PrefixManyFolders, } case Everything: @@ -223,6 +225,9 @@ func (name Name) String() string { return "s3:ObjectTransition:Complete" case ObjectManyVersions: return "s3:Scanner:ManyVersions" + case ObjectLargeVersions: + return "s3:Scanner:LargeVersions" + case PrefixManyFolders: return "s3:Scanner:BigPrefix" } @@ -345,6 +350,8 @@ func ParseName(s string) (Name, error) { return ObjectTransitionAll, nil case "s3:Scanner:ManyVersions": return ObjectManyVersions, nil + case "s3:Scanner:LargeVersions": + return ObjectLargeVersions, nil case "s3:Scanner:BigPrefix": return PrefixManyFolders, nil default: