From 0120ff93bc4b9cfaf2865e55850e9b20e5ef703d Mon Sep 17 00:00:00 2001 From: Krishnan Parthasarathi Date: Tue, 18 Jul 2023 10:49:40 -0700 Subject: [PATCH] admin-info: add DeleteMarkers count (#17659) --- cmd/admin-handlers.go | 31 ++++++++++++---------- cmd/data-scanner.go | 1 + cmd/data-usage-cache.go | 19 +++++++++----- cmd/data-usage-cache_gen.go | 51 +++++++++++++++++++++++++++---------- cmd/data-usage-utils.go | 6 ++++- cmd/xl-storage.go | 3 +++ go.mod | 2 +- go.sum | 4 +-- 8 files changed, 80 insertions(+), 37 deletions(-) diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index aa7fecad3..187e21928 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -1806,6 +1806,7 @@ func getPoolsInfo(ctx context.Context, allDisks []madmin.Disk) (map[int]map[int] dataUsageInfo := cache.dui(dataUsageRoot, nil) erasureSet.ObjectsCount = dataUsageInfo.ObjectsTotalCount erasureSet.VersionsCount = dataUsageInfo.VersionsTotalCount + erasureSet.DeleteMarkersCount = dataUsageInfo.DeleteMarkersTotalCount erasureSet.Usage = dataUsageInfo.ObjectsTotalSize } } @@ -1855,6 +1856,7 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request) buckets := madmin.Buckets{} objects := madmin.Objects{} versions := madmin.Versions{} + deleteMarkers := madmin.DeleteMarkers{} usage := madmin.Usage{} objectAPI := newObjectLayerFn() @@ -1867,10 +1869,12 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request) buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount} objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount} versions = madmin.Versions{Count: dataUsageInfo.VersionsTotalCount} + deleteMarkers = madmin.DeleteMarkers{Count: dataUsageInfo.DeleteMarkersTotalCount} usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize} } else { buckets = madmin.Buckets{Error: err.Error()} objects = madmin.Objects{Error: err.Error()} + deleteMarkers = madmin.DeleteMarkers{Error: err.Error()} usage = madmin.Usage{Error: err.Error()} } @@ -1909,19 +1913,20 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request) } return madmin.InfoMessage{ - Mode: string(mode), - Domain: domain, - Region: globalSite.Region, - SQSARN: globalEventNotifier.GetARNList(false), - DeploymentID: globalDeploymentID, - Buckets: buckets, - Objects: objects, - Versions: versions, - Usage: usage, - Services: services, - Backend: backend, - Servers: servers, - Pools: poolsInfo, + Mode: string(mode), + Domain: domain, + Region: globalSite.Region, + SQSARN: globalEventNotifier.GetARNList(false), + DeploymentID: globalDeploymentID, + Buckets: buckets, + Objects: objects, + Versions: versions, + DeleteMarkers: deleteMarkers, + Usage: usage, + Services: services, + Backend: backend, + Servers: servers, + Pools: poolsInfo, } } diff --git a/cmd/data-scanner.go b/cmd/data-scanner.go index 80d00f70f..c3e7c99fe 100644 --- a/cmd/data-scanner.go +++ b/cmd/data-scanner.go @@ -845,6 +845,7 @@ type scannerItem struct { type sizeSummary struct { totalSize int64 versions uint64 + deleteMarkers uint64 replicatedSize int64 pendingSize int64 failedSize int64 diff --git a/cmd/data-usage-cache.go b/cmd/data-usage-cache.go index 45435a22e..a989a74d3 100644 --- a/cmd/data-usage-cache.go +++ b/cmd/data-usage-cache.go @@ -57,6 +57,7 @@ type dataUsageEntry struct { Size int64 `msg:"sz"` Objects uint64 `msg:"os"` Versions uint64 `msg:"vs"` // Versions that are not delete markers. + DeleteMarkers uint64 `msg:"dms"` ObjSizes sizeHistogram `msg:"szs"` ObjVersions versionsHistogram `msg:"vh"` ReplicationStats *replicationAllStats `msg:"rs,omitempty"` @@ -328,6 +329,7 @@ type dataUsageCacheInfo struct { func (e *dataUsageEntry) addSizes(summary sizeSummary) { e.Size += summary.totalSize e.Versions += summary.versions + e.DeleteMarkers += summary.deleteMarkers e.ObjSizes.add(summary.totalSize) e.ObjVersions.add(summary.versions) @@ -366,6 +368,7 @@ func (e *dataUsageEntry) addSizes(summary sizeSummary) { func (e *dataUsageEntry) merge(other dataUsageEntry) { e.Objects += other.Objects e.Versions += other.Versions + e.DeleteMarkers += other.DeleteMarkers e.Size += other.Size if other.ReplicationStats != nil { if e.ReplicationStats == nil { @@ -531,13 +534,14 @@ func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo { } flat := d.flatten(*e) dui := DataUsageInfo{ - LastUpdate: d.Info.LastUpdate, - ObjectsTotalCount: flat.Objects, - VersionsTotalCount: flat.Versions, - ObjectsTotalSize: uint64(flat.Size), - BucketsCount: uint64(len(e.Children)), - BucketsUsage: d.bucketsUsageInfo(buckets), - TierStats: d.tiersUsageInfo(buckets), + LastUpdate: d.Info.LastUpdate, + ObjectsTotalCount: flat.Objects, + VersionsTotalCount: flat.Versions, + DeleteMarkersTotalCount: flat.DeleteMarkers, + ObjectsTotalSize: uint64(flat.Size), + BucketsCount: uint64(len(e.Children)), + BucketsUsage: d.bucketsUsageInfo(buckets), + TierStats: d.tiersUsageInfo(buckets), } return dui } @@ -805,6 +809,7 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke Size: uint64(flat.Size), VersionsCount: flat.Versions, ObjectsCount: flat.Objects, + DeleteMarkersCount: flat.DeleteMarkers, ObjectSizesHistogram: flat.ObjSizes.toMap(), ObjectVersionsHistogram: flat.ObjVersions.toMap(), } diff --git a/cmd/data-usage-cache_gen.go b/cmd/data-usage-cache_gen.go index 3173c438a..4c747046d 100644 --- a/cmd/data-usage-cache_gen.go +++ b/cmd/data-usage-cache_gen.go @@ -1522,6 +1522,12 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.WrapError(err, "Versions") return } + case "dms": + z.DeleteMarkers, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "DeleteMarkers") + return + } case "szs": var zb0002 uint32 zb0002, err = dc.ReadArrayHeader() @@ -1614,16 +1620,16 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) { // omitempty: check for empty values - zb0001Len := uint32(9) - var zb0001Mask uint16 /* 9 bits */ + zb0001Len := uint32(10) + var zb0001Mask uint16 /* 10 bits */ _ = zb0001Mask if z.ReplicationStats == nil { zb0001Len-- - zb0001Mask |= 0x40 + zb0001Mask |= 0x80 } if z.AllTierStats == nil { zb0001Len-- - zb0001Mask |= 0x80 + zb0001Mask |= 0x100 } // variable map header, size zb0001Len err = en.Append(0x80 | uint8(zb0001Len)) @@ -1673,6 +1679,16 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "Versions") return } + // write "dms" + err = en.Append(0xa3, 0x64, 0x6d, 0x73) + if err != nil { + return + } + err = en.WriteUint64(z.DeleteMarkers) + if err != nil { + err = msgp.WrapError(err, "DeleteMarkers") + return + } // write "szs" err = en.Append(0xa3, 0x73, 0x7a, 0x73) if err != nil { @@ -1707,7 +1723,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) { return } } - if (zb0001Mask & 0x40) == 0 { // if not empty + if (zb0001Mask & 0x80) == 0 { // if not empty // write "rs" err = en.Append(0xa2, 0x72, 0x73) if err != nil { @@ -1726,7 +1742,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) { } } } - if (zb0001Mask & 0x80) == 0 { // if not empty + if (zb0001Mask & 0x100) == 0 { // if not empty // write "ats" err = en.Append(0xa3, 0x61, 0x74, 0x73) if err != nil { @@ -1762,16 +1778,16 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) { func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // omitempty: check for empty values - zb0001Len := uint32(9) - var zb0001Mask uint16 /* 9 bits */ + zb0001Len := uint32(10) + var zb0001Mask uint16 /* 10 bits */ _ = zb0001Mask if z.ReplicationStats == nil { zb0001Len-- - zb0001Mask |= 0x40 + zb0001Mask |= 0x80 } if z.AllTierStats == nil { zb0001Len-- - zb0001Mask |= 0x80 + zb0001Mask |= 0x100 } // variable map header, size zb0001Len o = append(o, 0x80|uint8(zb0001Len)) @@ -1794,6 +1810,9 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) { // string "vs" o = append(o, 0xa2, 0x76, 0x73) o = msgp.AppendUint64(o, z.Versions) + // string "dms" + o = append(o, 0xa3, 0x64, 0x6d, 0x73) + o = msgp.AppendUint64(o, z.DeleteMarkers) // string "szs" o = append(o, 0xa3, 0x73, 0x7a, 0x73) o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen)) @@ -1806,7 +1825,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) { for za0002 := range z.ObjVersions { o = msgp.AppendUint64(o, z.ObjVersions[za0002]) } - if (zb0001Mask & 0x40) == 0 { // if not empty + if (zb0001Mask & 0x80) == 0 { // if not empty // string "rs" o = append(o, 0xa2, 0x72, 0x73) if z.ReplicationStats == nil { @@ -1819,7 +1838,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) { } } } - if (zb0001Mask & 0x80) == 0 { // if not empty + if (zb0001Mask & 0x100) == 0 { // if not empty // string "ats" o = append(o, 0xa3, 0x61, 0x74, 0x73) if z.AllTierStats == nil { @@ -1880,6 +1899,12 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "Versions") return } + case "dms": + z.DeleteMarkers, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "DeleteMarkers") + return + } case "szs": var zb0002 uint32 zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) @@ -1970,7 +1995,7 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *dataUsageEntry) Msgsize() (s int) { - s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 3 + s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 3 if z.ReplicationStats == nil { s += msgp.NilSize } else { diff --git a/cmd/data-usage-utils.go b/cmd/data-usage-utils.go index eb8a35556..f1c94d8f3 100644 --- a/cmd/data-usage-utils.go +++ b/cmd/data-usage-utils.go @@ -62,6 +62,7 @@ type BucketUsageInfo struct { ObjectSizesHistogram map[string]uint64 `json:"objectsSizesHistogram"` ObjectVersionsHistogram map[string]uint64 `json:"objectsVersionsHistogram"` VersionsCount uint64 `json:"versionsCount"` + DeleteMarkersCount uint64 `json:"deleteMarkersCount"` ReplicaSize uint64 `json:"objectReplicaTotalSize"` ReplicationInfo map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"` } @@ -75,9 +76,12 @@ type DataUsageInfo struct { // Objects total count across all buckets ObjectsTotalCount uint64 `json:"objectsCount"` - // Objects total count across all buckets + // Versions total count across all buckets VersionsTotalCount uint64 `json:"versionsCount"` + // Delete markers total count across all buckets + DeleteMarkersTotalCount uint64 `json:"deleteMarkersCount"` + // Objects total size across all buckets ObjectsTotalSize uint64 `json:"objectsTotalSize"` ReplicationInfo map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"` diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index d8c97faac..b0550fb7f 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -547,6 +547,9 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates done = globalScannerMetrics.time(scannerMetricApplyVersion) sz := item.applyActions(ctx, objAPI, oi, &sizeS) done() + if oi.DeleteMarker { + sizeS.deleteMarkers++ + } if oi.VersionID != "" && sz == oi.Size { sizeS.versions++ } diff --git a/go.mod b/go.mod index f585cd83b..e031cbd02 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/minio/dperf v0.5.0 github.com/minio/highwayhash v1.0.2 github.com/minio/kes-go v0.1.0 - github.com/minio/madmin-go/v3 v3.0.7 + github.com/minio/madmin-go/v3 v3.0.8-0.20230717144230-b697fb223a25 github.com/minio/minio-go/v7 v7.0.61 github.com/minio/mux v1.9.0 github.com/minio/pkg v1.7.5 diff --git a/go.sum b/go.sum index 45626efc9..6455f7439 100644 --- a/go.sum +++ b/go.sum @@ -484,8 +484,8 @@ github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/kes-go v0.1.0 h1:h201DyOYP5sTqajkxFGxmXz/kPbT8HQNX1uh3Yx2PFc= github.com/minio/kes-go v0.1.0/go.mod h1:VorHLaIYis9/MxAHAtXN4d8PUMNKhIxTIlvFt0hBOEo= -github.com/minio/madmin-go/v3 v3.0.7 h1:nuRwrqarFrkzbUiA36H/HKAcuNr8J9TjKlWRlua7lNo= -github.com/minio/madmin-go/v3 v3.0.7/go.mod h1:lPrMoc1aeiIWmmrxBthkDqzMPQwC/Lu9ByuyM2wenJk= +github.com/minio/madmin-go/v3 v3.0.8-0.20230717144230-b697fb223a25 h1:4zQOJq6tbY6MDXbdPw5tuajM9rQroJ7eIRGs5mxr40w= +github.com/minio/madmin-go/v3 v3.0.8-0.20230717144230-b697fb223a25/go.mod h1:lPrMoc1aeiIWmmrxBthkDqzMPQwC/Lu9ByuyM2wenJk= github.com/minio/mc v0.0.0-20230713161043-15ea1125e377 h1:jcIt177m3jJCrDEAZFhgUcq+PdMnpfsF8QrFNSbG0nU= github.com/minio/mc v0.0.0-20230713161043-15ea1125e377/go.mod h1:eXUhMFdo6O9d0axon9FVNaEtCDPMJdZW7K7jyYQRR5Q= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=