MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-01-26 06:03:17 -05:00
Code Issues Packages Projects Releases Wiki Activity

admin-info: add DeleteMarkers count (#17659)

Browse Source
This commit is contained in:
Krishnan Parthasarathi 2023-07-18 10:49:40 -07:00 committed by GitHub
parent 49638fa533
commit 0120ff93bc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 80 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
cmd/admin-handlers.go
View File

@ -1806,6 +1806,7 @@ func getPoolsInfo(ctx context.Context, allDisks []madmin.Disk) (map[int]map[int]
dataUsageInfo := cache.dui(dataUsageRoot, nil) dataUsageInfo := cache.dui(dataUsageRoot, nil)
erasureSet.ObjectsCount = dataUsageInfo.ObjectsTotalCount erasureSet.ObjectsCount = dataUsageInfo.ObjectsTotalCount
erasureSet.VersionsCount = dataUsageInfo.VersionsTotalCount erasureSet.VersionsCount = dataUsageInfo.VersionsTotalCount
erasureSet.DeleteMarkersCount = dataUsageInfo.DeleteMarkersTotalCount
erasureSet.Usage = dataUsageInfo.ObjectsTotalSize erasureSet.Usage = dataUsageInfo.ObjectsTotalSize
} }
} }
@ -1855,6 +1856,7 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
buckets := madmin.Buckets{} buckets := madmin.Buckets{}
objects := madmin.Objects{} objects := madmin.Objects{}
versions := madmin.Versions{} versions := madmin.Versions{}
deleteMarkers := madmin.DeleteMarkers{}
usage := madmin.Usage{} usage := madmin.Usage{}
objectAPI := newObjectLayerFn() objectAPI := newObjectLayerFn()
@ -1867,10 +1869,12 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount} buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount}
objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount} objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount}
versions = madmin.Versions{Count: dataUsageInfo.VersionsTotalCount} versions = madmin.Versions{Count: dataUsageInfo.VersionsTotalCount}
deleteMarkers = madmin.DeleteMarkers{Count: dataUsageInfo.DeleteMarkersTotalCount}
usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize} usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize}
} else { } else {
buckets = madmin.Buckets{Error: err.Error()} buckets = madmin.Buckets{Error: err.Error()}
objects = madmin.Objects{Error: err.Error()} objects = madmin.Objects{Error: err.Error()}
deleteMarkers = madmin.DeleteMarkers{Error: err.Error()}
usage = madmin.Usage{Error: err.Error()} usage = madmin.Usage{Error: err.Error()}
} }
@ -1909,19 +1913,20 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
} }
return madmin.InfoMessage{ return madmin.InfoMessage{
Mode: string(mode), Mode: string(mode),
Domain: domain, Domain: domain,
Region: globalSite.Region, Region: globalSite.Region,
SQSARN: globalEventNotifier.GetARNList(false), SQSARN: globalEventNotifier.GetARNList(false),
DeploymentID: globalDeploymentID, DeploymentID: globalDeploymentID,
Buckets: buckets, Buckets: buckets,
Objects: objects, Objects: objects,
Versions: versions, Versions: versions,
Usage: usage, DeleteMarkers: deleteMarkers,
Services: services, Usage: usage,
Backend: backend, Services: services,
Servers: servers, Backend: backend,
Pools: poolsInfo, Servers: servers,
Pools: poolsInfo,
} }
} }

1
cmd/data-scanner.go
View File

@ -845,6 +845,7 @@ type scannerItem struct {
type sizeSummary struct { type sizeSummary struct {
totalSize int64 totalSize int64
versions uint64 versions uint64
deleteMarkers uint64
replicatedSize int64 replicatedSize int64
pendingSize int64 pendingSize int64
failedSize int64 failedSize int64

19
cmd/data-usage-cache.go
View File

@ -57,6 +57,7 @@ type dataUsageEntry struct {
Size int64 `msg:"sz"` Size int64 `msg:"sz"`
Objects uint64 `msg:"os"` Objects uint64 `msg:"os"`
Versions uint64 `msg:"vs"` // Versions that are not delete markers. Versions uint64 `msg:"vs"` // Versions that are not delete markers.
DeleteMarkers uint64 `msg:"dms"`
ObjSizes sizeHistogram `msg:"szs"` ObjSizes sizeHistogram `msg:"szs"`
ObjVersions versionsHistogram `msg:"vh"` ObjVersions versionsHistogram `msg:"vh"`
ReplicationStats *replicationAllStats `msg:"rs,omitempty"` ReplicationStats *replicationAllStats `msg:"rs,omitempty"`
@ -328,6 +329,7 @@ type dataUsageCacheInfo struct {
func (e *dataUsageEntry) addSizes(summary sizeSummary) { func (e *dataUsageEntry) addSizes(summary sizeSummary) {
e.Size += summary.totalSize e.Size += summary.totalSize
e.Versions += summary.versions e.Versions += summary.versions
e.DeleteMarkers += summary.deleteMarkers
e.ObjSizes.add(summary.totalSize) e.ObjSizes.add(summary.totalSize)
e.ObjVersions.add(summary.versions) e.ObjVersions.add(summary.versions)
@ -366,6 +368,7 @@ func (e *dataUsageEntry) addSizes(summary sizeSummary) {
func (e *dataUsageEntry) merge(other dataUsageEntry) { func (e *dataUsageEntry) merge(other dataUsageEntry) {
e.Objects += other.Objects e.Objects += other.Objects
e.Versions += other.Versions e.Versions += other.Versions
e.DeleteMarkers += other.DeleteMarkers
e.Size += other.Size e.Size += other.Size
if other.ReplicationStats != nil { if other.ReplicationStats != nil {
if e.ReplicationStats == nil { if e.ReplicationStats == nil {
@ -531,13 +534,14 @@ func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
} }
flat := d.flatten(*e) flat := d.flatten(*e)
dui := DataUsageInfo{ dui := DataUsageInfo{
LastUpdate: d.Info.LastUpdate, LastUpdate: d.Info.LastUpdate,
ObjectsTotalCount: flat.Objects, ObjectsTotalCount: flat.Objects,
VersionsTotalCount: flat.Versions, VersionsTotalCount: flat.Versions,
ObjectsTotalSize: uint64(flat.Size), DeleteMarkersTotalCount: flat.DeleteMarkers,
BucketsCount: uint64(len(e.Children)), ObjectsTotalSize: uint64(flat.Size),
BucketsUsage: d.bucketsUsageInfo(buckets), BucketsCount: uint64(len(e.Children)),
TierStats: d.tiersUsageInfo(buckets), BucketsUsage: d.bucketsUsageInfo(buckets),
TierStats: d.tiersUsageInfo(buckets),
} }
return dui return dui
} }
@ -805,6 +809,7 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
Size: uint64(flat.Size), Size: uint64(flat.Size),
VersionsCount: flat.Versions, VersionsCount: flat.Versions,
ObjectsCount: flat.Objects, ObjectsCount: flat.Objects,
DeleteMarkersCount: flat.DeleteMarkers,
ObjectSizesHistogram: flat.ObjSizes.toMap(), ObjectSizesHistogram: flat.ObjSizes.toMap(),
ObjectVersionsHistogram: flat.ObjVersions.toMap(), ObjectVersionsHistogram: flat.ObjVersions.toMap(),
} }

51
cmd/data-usage-cache_gen.go
View File

@ -1522,6 +1522,12 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
err = msgp.WrapError(err, "Versions") err = msgp.WrapError(err, "Versions")
return return
} }
case "dms":
z.DeleteMarkers, err = dc.ReadUint64()
if err != nil {
err = msgp.WrapError(err, "DeleteMarkers")
return
}
case "szs": case "szs":
var zb0002 uint32 var zb0002 uint32
zb0002, err = dc.ReadArrayHeader() zb0002, err = dc.ReadArrayHeader()
@ -1614,16 +1620,16 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
// EncodeMsg implements msgp.Encodable // EncodeMsg implements msgp.Encodable
func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) { func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
// omitempty: check for empty values // omitempty: check for empty values
zb0001Len := uint32(9) zb0001Len := uint32(10)
var zb0001Mask uint16 /* 9 bits */ var zb0001Mask uint16 /* 10 bits */
_ = zb0001Mask _ = zb0001Mask
if z.ReplicationStats == nil { if z.ReplicationStats == nil {
zb0001Len-- zb0001Len--
zb0001Mask |= 0x40 zb0001Mask |= 0x80
} }
if z.AllTierStats == nil { if z.AllTierStats == nil {
zb0001Len-- zb0001Len--
zb0001Mask |= 0x80 zb0001Mask |= 0x100
} }
// variable map header, size zb0001Len // variable map header, size zb0001Len
err = en.Append(0x80 | uint8(zb0001Len)) err = en.Append(0x80 | uint8(zb0001Len))
@ -1673,6 +1679,16 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
err = msgp.WrapError(err, "Versions") err = msgp.WrapError(err, "Versions")
return return
} }
// write "dms"
err = en.Append(0xa3, 0x64, 0x6d, 0x73)
if err != nil {
return
}
err = en.WriteUint64(z.DeleteMarkers)
if err != nil {
err = msgp.WrapError(err, "DeleteMarkers")
return
}
// write "szs" // write "szs"
err = en.Append(0xa3, 0x73, 0x7a, 0x73) err = en.Append(0xa3, 0x73, 0x7a, 0x73)
if err != nil { if err != nil {
@ -1707,7 +1723,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
return return
} }
} }
if (zb0001Mask & 0x40) == 0 { // if not empty if (zb0001Mask & 0x80) == 0 { // if not empty
// write "rs" // write "rs"
err = en.Append(0xa2, 0x72, 0x73) err = en.Append(0xa2, 0x72, 0x73)
if err != nil { if err != nil {
@ -1726,7 +1742,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
} }
} }
} }
if (zb0001Mask & 0x80) == 0 { // if not empty if (zb0001Mask & 0x100) == 0 { // if not empty
// write "ats" // write "ats"
err = en.Append(0xa3, 0x61, 0x74, 0x73) err = en.Append(0xa3, 0x61, 0x74, 0x73)
if err != nil { if err != nil {
@ -1762,16 +1778,16 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) { func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize()) o = msgp.Require(b, z.Msgsize())
// omitempty: check for empty values // omitempty: check for empty values
zb0001Len := uint32(9) zb0001Len := uint32(10)
var zb0001Mask uint16 /* 9 bits */ var zb0001Mask uint16 /* 10 bits */
_ = zb0001Mask _ = zb0001Mask
if z.ReplicationStats == nil { if z.ReplicationStats == nil {
zb0001Len-- zb0001Len--
zb0001Mask |= 0x40 zb0001Mask |= 0x80
} }
if z.AllTierStats == nil { if z.AllTierStats == nil {
zb0001Len-- zb0001Len--
zb0001Mask |= 0x80 zb0001Mask |= 0x100
} }
// variable map header, size zb0001Len // variable map header, size zb0001Len
o = append(o, 0x80|uint8(zb0001Len)) o = append(o, 0x80|uint8(zb0001Len))
@ -1794,6 +1810,9 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
// string "vs" // string "vs"
o = append(o, 0xa2, 0x76, 0x73) o = append(o, 0xa2, 0x76, 0x73)
o = msgp.AppendUint64(o, z.Versions) o = msgp.AppendUint64(o, z.Versions)
// string "dms"
o = append(o, 0xa3, 0x64, 0x6d, 0x73)
o = msgp.AppendUint64(o, z.DeleteMarkers)
// string "szs" // string "szs"
o = append(o, 0xa3, 0x73, 0x7a, 0x73) o = append(o, 0xa3, 0x73, 0x7a, 0x73)
o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen)) o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen))
@ -1806,7 +1825,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
for za0002 := range z.ObjVersions { for za0002 := range z.ObjVersions {
o = msgp.AppendUint64(o, z.ObjVersions[za0002]) o = msgp.AppendUint64(o, z.ObjVersions[za0002])
} }
if (zb0001Mask & 0x40) == 0 { // if not empty if (zb0001Mask & 0x80) == 0 { // if not empty
// string "rs" // string "rs"
o = append(o, 0xa2, 0x72, 0x73) o = append(o, 0xa2, 0x72, 0x73)
if z.ReplicationStats == nil { if z.ReplicationStats == nil {
@ -1819,7 +1838,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
} }
} }
} }
if (zb0001Mask & 0x80) == 0 { // if not empty if (zb0001Mask & 0x100) == 0 { // if not empty
// string "ats" // string "ats"
o = append(o, 0xa3, 0x61, 0x74, 0x73) o = append(o, 0xa3, 0x61, 0x74, 0x73)
if z.AllTierStats == nil { if z.AllTierStats == nil {
@ -1880,6 +1899,12 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
err = msgp.WrapError(err, "Versions") err = msgp.WrapError(err, "Versions")
return return
} }
case "dms":
z.DeleteMarkers, bts, err = msgp.ReadUint64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "DeleteMarkers")
return
}
case "szs": case "szs":
var zb0002 uint32 var zb0002 uint32
zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
@ -1970,7 +1995,7 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *dataUsageEntry) Msgsize() (s int) { func (z *dataUsageEntry) Msgsize() (s int) {
s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 3 s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 3
if z.ReplicationStats == nil { if z.ReplicationStats == nil {
s += msgp.NilSize s += msgp.NilSize
} else { } else {

6
cmd/data-usage-utils.go
View File

@ -62,6 +62,7 @@ type BucketUsageInfo struct {
ObjectSizesHistogram map[string]uint64 `json:"objectsSizesHistogram"` ObjectSizesHistogram map[string]uint64 `json:"objectsSizesHistogram"`
ObjectVersionsHistogram map[string]uint64 `json:"objectsVersionsHistogram"` ObjectVersionsHistogram map[string]uint64 `json:"objectsVersionsHistogram"`
VersionsCount uint64 `json:"versionsCount"` VersionsCount uint64 `json:"versionsCount"`
DeleteMarkersCount uint64 `json:"deleteMarkersCount"`
ReplicaSize uint64 `json:"objectReplicaTotalSize"` ReplicaSize uint64 `json:"objectReplicaTotalSize"`
ReplicationInfo map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"` ReplicationInfo map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
} }
@ -75,9 +76,12 @@ type DataUsageInfo struct {
// Objects total count across all buckets // Objects total count across all buckets
ObjectsTotalCount uint64 `json:"objectsCount"` ObjectsTotalCount uint64 `json:"objectsCount"`
// Objects total count across all buckets // Versions total count across all buckets
VersionsTotalCount uint64 `json:"versionsCount"` VersionsTotalCount uint64 `json:"versionsCount"`
// Delete markers total count across all buckets
DeleteMarkersTotalCount uint64 `json:"deleteMarkersCount"`
// Objects total size across all buckets // Objects total size across all buckets
ObjectsTotalSize uint64 `json:"objectsTotalSize"` ObjectsTotalSize uint64 `json:"objectsTotalSize"`
ReplicationInfo map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"` ReplicationInfo map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`

3
cmd/xl-storage.go
View File

@ -547,6 +547,9 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
done = globalScannerMetrics.time(scannerMetricApplyVersion) done = globalScannerMetrics.time(scannerMetricApplyVersion)
sz := item.applyActions(ctx, objAPI, oi, &sizeS) sz := item.applyActions(ctx, objAPI, oi, &sizeS)
done() done()
if oi.DeleteMarker {
sizeS.deleteMarkers++
}
if oi.VersionID != "" && sz == oi.Size { if oi.VersionID != "" && sz == oi.Size {
sizeS.versions++ sizeS.versions++
} }

2
go.mod
View File

@ -49,7 +49,7 @@ require (
github.com/minio/dperf v0.5.0 github.com/minio/dperf v0.5.0
github.com/minio/highwayhash v1.0.2 github.com/minio/highwayhash v1.0.2
github.com/minio/kes-go v0.1.0 github.com/minio/kes-go v0.1.0
github.com/minio/madmin-go/v3 v3.0.7 github.com/minio/madmin-go/v3 v3.0.8-0.20230717144230-b697fb223a25
github.com/minio/minio-go/v7 v7.0.61 github.com/minio/minio-go/v7 v7.0.61
github.com/minio/mux v1.9.0 github.com/minio/mux v1.9.0
github.com/minio/pkg v1.7.5 github.com/minio/pkg v1.7.5

4
go.sum
View File

@ -484,8 +484,8 @@ github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA
github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
github.com/minio/kes-go v0.1.0 h1:h201DyOYP5sTqajkxFGxmXz/kPbT8HQNX1uh3Yx2PFc= github.com/minio/kes-go v0.1.0 h1:h201DyOYP5sTqajkxFGxmXz/kPbT8HQNX1uh3Yx2PFc=
github.com/minio/kes-go v0.1.0/go.mod h1:VorHLaIYis9/MxAHAtXN4d8PUMNKhIxTIlvFt0hBOEo= github.com/minio/kes-go v0.1.0/go.mod h1:VorHLaIYis9/MxAHAtXN4d8PUMNKhIxTIlvFt0hBOEo=
github.com/minio/madmin-go/v3 v3.0.7 h1:nuRwrqarFrkzbUiA36H/HKAcuNr8J9TjKlWRlua7lNo= github.com/minio/madmin-go/v3 v3.0.8-0.20230717144230-b697fb223a25 h1:4zQOJq6tbY6MDXbdPw5tuajM9rQroJ7eIRGs5mxr40w=
github.com/minio/madmin-go/v3 v3.0.7/go.mod h1:lPrMoc1aeiIWmmrxBthkDqzMPQwC/Lu9ByuyM2wenJk= github.com/minio/madmin-go/v3 v3.0.8-0.20230717144230-b697fb223a25/go.mod h1:lPrMoc1aeiIWmmrxBthkDqzMPQwC/Lu9ByuyM2wenJk=
github.com/minio/mc v0.0.0-20230713161043-15ea1125e377 h1:jcIt177m3jJCrDEAZFhgUcq+PdMnpfsF8QrFNSbG0nU= github.com/minio/mc v0.0.0-20230713161043-15ea1125e377 h1:jcIt177m3jJCrDEAZFhgUcq+PdMnpfsF8QrFNSbG0nU=
github.com/minio/mc v0.0.0-20230713161043-15ea1125e377/go.mod h1:eXUhMFdo6O9d0axon9FVNaEtCDPMJdZW7K7jyYQRR5Q= github.com/minio/mc v0.0.0-20230713161043-15ea1125e377/go.mod h1:eXUhMFdo6O9d0axon9FVNaEtCDPMJdZW7K7jyYQRR5Q=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=