mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
Add Object Version count histogram (#16739)
This commit is contained in:
parent
9800760cb3
commit
d85da9236e
@ -1332,11 +1332,12 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
// Fetch the data usage of the current bucket
|
||||
var size uint64
|
||||
var objectsCount uint64
|
||||
var objectsHist map[string]uint64
|
||||
var objectsHist, versionsHist map[string]uint64
|
||||
if !dataUsageInfo.LastUpdate.IsZero() {
|
||||
size = dataUsageInfo.BucketsUsage[bucket.Name].Size
|
||||
objectsCount = dataUsageInfo.BucketsUsage[bucket.Name].ObjectsCount
|
||||
objectsHist = dataUsageInfo.BucketsUsage[bucket.Name].ObjectSizesHistogram
|
||||
versionsHist = dataUsageInfo.BucketsUsage[bucket.Name].ObjectVersionsHistogram
|
||||
}
|
||||
// Fetch the prefix usage of the current bucket
|
||||
var prefixUsage map[string]uint64
|
||||
@ -1350,12 +1351,13 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
tcfg, _, _ := globalBucketMetadataSys.GetTaggingConfig(bucket.Name)
|
||||
|
||||
acctInfo.Buckets = append(acctInfo.Buckets, madmin.BucketAccessInfo{
|
||||
Name: bucket.Name,
|
||||
Created: bucket.Created,
|
||||
Size: size,
|
||||
Objects: objectsCount,
|
||||
ObjectSizesHistogram: objectsHist,
|
||||
PrefixUsage: prefixUsage,
|
||||
Name: bucket.Name,
|
||||
Created: bucket.Created,
|
||||
Size: size,
|
||||
Objects: objectsCount,
|
||||
ObjectSizesHistogram: objectsHist,
|
||||
ObjectVersionsHistogram: versionsHist,
|
||||
PrefixUsage: prefixUsage,
|
||||
Details: &madmin.BucketDetails{
|
||||
Versioning: globalBucketVersioningSys.Enabled(bucket.Name),
|
||||
VersioningSuspended: globalBucketVersioningSys.Suspended(bucket.Name),
|
||||
|
@ -48,6 +48,9 @@ type dataUsageHash string
|
||||
// sizeHistogram is a size histogram.
|
||||
type sizeHistogram [dataUsageBucketLen]uint64
|
||||
|
||||
// versionsHistogram is a histogram of number of versions in an object.
|
||||
type versionsHistogram [dataUsageVersionLen]uint64
|
||||
|
||||
type dataUsageEntry struct {
|
||||
Children dataUsageHashMap `msg:"ch"`
|
||||
// These fields do no include any children.
|
||||
@ -55,6 +58,7 @@ type dataUsageEntry struct {
|
||||
Objects uint64 `msg:"os"`
|
||||
Versions uint64 `msg:"vs"` // Versions that are not delete markers.
|
||||
ObjSizes sizeHistogram `msg:"szs"`
|
||||
ObjVersions versionsHistogram `msg:"vh"`
|
||||
ReplicationStats *replicationAllStats `msg:"rs,omitempty"`
|
||||
AllTierStats *allTierStats `msg:"ats,omitempty"`
|
||||
Compacted bool `msg:"c"`
|
||||
@ -292,6 +296,7 @@ func (e *dataUsageEntry) addSizes(summary sizeSummary) {
|
||||
e.Size += summary.totalSize
|
||||
e.Versions += summary.versions
|
||||
e.ObjSizes.add(summary.totalSize)
|
||||
e.ObjVersions.add(summary.versions)
|
||||
|
||||
if e.ReplicationStats == nil {
|
||||
e.ReplicationStats = &replicationAllStats{
|
||||
@ -352,6 +357,10 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
|
||||
e.ObjSizes[i] += v
|
||||
}
|
||||
|
||||
for i, v := range other.ObjVersions[:] {
|
||||
e.ObjVersions[i] += v
|
||||
}
|
||||
|
||||
if other.AllTierStats != nil {
|
||||
if e.AllTierStats == nil {
|
||||
e.AllTierStats = newAllTierStats()
|
||||
@ -695,7 +704,7 @@ func (d *dataUsageCache) flatten(root dataUsageEntry) dataUsageEntry {
|
||||
func (h *sizeHistogram) add(size int64) {
|
||||
// Fetch the histogram interval corresponding
|
||||
// to the passed object size.
|
||||
for i, interval := range ObjectsHistogramIntervals {
|
||||
for i, interval := range ObjectsHistogramIntervals[:] {
|
||||
if size >= interval.start && size <= interval.end {
|
||||
h[i]++
|
||||
break
|
||||
@ -712,6 +721,27 @@ func (h *sizeHistogram) toMap() map[string]uint64 {
|
||||
return res
|
||||
}
|
||||
|
||||
// add a version count to the histogram.
|
||||
func (h *versionsHistogram) add(versions uint64) {
|
||||
// Fetch the histogram interval corresponding
|
||||
// to the passed object size.
|
||||
for i, interval := range ObjectsVersionCountIntervals[:] {
|
||||
if versions >= uint64(interval.start) && versions <= uint64(interval.end) {
|
||||
h[i]++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// toMap returns the map to a map[string]uint64.
|
||||
func (h *versionsHistogram) toMap() map[string]uint64 {
|
||||
res := make(map[string]uint64, dataUsageVersionLen)
|
||||
for i, count := range h {
|
||||
res[ObjectsVersionCountIntervals[i].name] = count
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (d *dataUsageCache) tiersUsageInfo(buckets []BucketInfo) *allTierStats {
|
||||
dst := newAllTierStats()
|
||||
for _, bucket := range buckets {
|
||||
@ -742,10 +772,11 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
|
||||
}
|
||||
flat := d.flatten(*e)
|
||||
bui := BucketUsageInfo{
|
||||
Size: uint64(flat.Size),
|
||||
VersionsCount: flat.Versions,
|
||||
ObjectsCount: flat.Objects,
|
||||
ObjectSizesHistogram: flat.ObjSizes.toMap(),
|
||||
Size: uint64(flat.Size),
|
||||
VersionsCount: flat.Versions,
|
||||
ObjectsCount: flat.Objects,
|
||||
ObjectSizesHistogram: flat.ObjSizes.toMap(),
|
||||
ObjectVersionsHistogram: flat.ObjVersions.toMap(),
|
||||
}
|
||||
if flat.ReplicationStats != nil {
|
||||
bui.ReplicaSize = flat.ReplicationStats.ReplicaSize
|
||||
|
@ -1540,6 +1540,24 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
return
|
||||
}
|
||||
}
|
||||
case "vh":
|
||||
var zb0003 uint32
|
||||
zb0003, err = dc.ReadArrayHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjVersions")
|
||||
return
|
||||
}
|
||||
if zb0003 != uint32(dataUsageVersionLen) {
|
||||
err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003}
|
||||
return
|
||||
}
|
||||
for za0002 := range z.ObjVersions {
|
||||
z.ObjVersions[za0002], err = dc.ReadUint64()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjVersions", za0002)
|
||||
return
|
||||
}
|
||||
}
|
||||
case "rs":
|
||||
if dc.IsNil() {
|
||||
err = dc.ReadNil()
|
||||
@ -1596,16 +1614,16 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// omitempty: check for empty values
|
||||
zb0001Len := uint32(8)
|
||||
var zb0001Mask uint8 /* 8 bits */
|
||||
zb0001Len := uint32(9)
|
||||
var zb0001Mask uint16 /* 9 bits */
|
||||
_ = zb0001Mask
|
||||
if z.ReplicationStats == nil {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x20
|
||||
zb0001Mask |= 0x40
|
||||
}
|
||||
if z.AllTierStats == nil {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x40
|
||||
zb0001Mask |= 0x80
|
||||
}
|
||||
// variable map header, size zb0001Len
|
||||
err = en.Append(0x80 | uint8(zb0001Len))
|
||||
@ -1672,7 +1690,24 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if (zb0001Mask & 0x20) == 0 { // if not empty
|
||||
// write "vh"
|
||||
err = en.Append(0xa2, 0x76, 0x68)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteArrayHeader(uint32(dataUsageVersionLen))
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjVersions")
|
||||
return
|
||||
}
|
||||
for za0002 := range z.ObjVersions {
|
||||
err = en.WriteUint64(z.ObjVersions[za0002])
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjVersions", za0002)
|
||||
return
|
||||
}
|
||||
}
|
||||
if (zb0001Mask & 0x40) == 0 { // if not empty
|
||||
// write "rs"
|
||||
err = en.Append(0xa2, 0x72, 0x73)
|
||||
if err != nil {
|
||||
@ -1691,7 +1726,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
if (zb0001Mask & 0x40) == 0 { // if not empty
|
||||
if (zb0001Mask & 0x80) == 0 { // if not empty
|
||||
// write "ats"
|
||||
err = en.Append(0xa3, 0x61, 0x74, 0x73)
|
||||
if err != nil {
|
||||
@ -1727,16 +1762,16 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// omitempty: check for empty values
|
||||
zb0001Len := uint32(8)
|
||||
var zb0001Mask uint8 /* 8 bits */
|
||||
zb0001Len := uint32(9)
|
||||
var zb0001Mask uint16 /* 9 bits */
|
||||
_ = zb0001Mask
|
||||
if z.ReplicationStats == nil {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x20
|
||||
zb0001Mask |= 0x40
|
||||
}
|
||||
if z.AllTierStats == nil {
|
||||
zb0001Len--
|
||||
zb0001Mask |= 0x40
|
||||
zb0001Mask |= 0x80
|
||||
}
|
||||
// variable map header, size zb0001Len
|
||||
o = append(o, 0x80|uint8(zb0001Len))
|
||||
@ -1765,7 +1800,13 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
for za0001 := range z.ObjSizes {
|
||||
o = msgp.AppendUint64(o, z.ObjSizes[za0001])
|
||||
}
|
||||
if (zb0001Mask & 0x20) == 0 { // if not empty
|
||||
// string "vh"
|
||||
o = append(o, 0xa2, 0x76, 0x68)
|
||||
o = msgp.AppendArrayHeader(o, uint32(dataUsageVersionLen))
|
||||
for za0002 := range z.ObjVersions {
|
||||
o = msgp.AppendUint64(o, z.ObjVersions[za0002])
|
||||
}
|
||||
if (zb0001Mask & 0x40) == 0 { // if not empty
|
||||
// string "rs"
|
||||
o = append(o, 0xa2, 0x72, 0x73)
|
||||
if z.ReplicationStats == nil {
|
||||
@ -1778,7 +1819,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
if (zb0001Mask & 0x40) == 0 { // if not empty
|
||||
if (zb0001Mask & 0x80) == 0 { // if not empty
|
||||
// string "ats"
|
||||
o = append(o, 0xa3, 0x61, 0x74, 0x73)
|
||||
if z.AllTierStats == nil {
|
||||
@ -1857,6 +1898,24 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
return
|
||||
}
|
||||
}
|
||||
case "vh":
|
||||
var zb0003 uint32
|
||||
zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjVersions")
|
||||
return
|
||||
}
|
||||
if zb0003 != uint32(dataUsageVersionLen) {
|
||||
err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003}
|
||||
return
|
||||
}
|
||||
for za0002 := range z.ObjVersions {
|
||||
z.ObjVersions[za0002], bts, err = msgp.ReadUint64Bytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjVersions", za0002)
|
||||
return
|
||||
}
|
||||
}
|
||||
case "rs":
|
||||
if msgp.IsNil(bts) {
|
||||
bts, err = msgp.ReadNilBytes(bts)
|
||||
@ -1911,7 +1970,7 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *dataUsageEntry) Msgsize() (s int) {
|
||||
s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3
|
||||
s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 3
|
||||
if z.ReplicationStats == nil {
|
||||
s += msgp.NilSize
|
||||
} else {
|
||||
@ -3704,3 +3763,81 @@ func (z tierStats) Msgsize() (s int) {
|
||||
s = 1 + 3 + msgp.Uint64Size + 3 + msgp.IntSize + 3 + msgp.IntSize
|
||||
return
|
||||
}
|
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *versionsHistogram) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
var zb0001 uint32
|
||||
zb0001, err = dc.ReadArrayHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
if zb0001 != uint32(dataUsageVersionLen) {
|
||||
err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0001}
|
||||
return
|
||||
}
|
||||
for za0001 := range z {
|
||||
z[za0001], err = dc.ReadUint64()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *versionsHistogram) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
err = en.WriteArrayHeader(uint32(dataUsageVersionLen))
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
for za0001 := range z {
|
||||
err = en.WriteUint64(z[za0001])
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *versionsHistogram) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
o = msgp.AppendArrayHeader(o, uint32(dataUsageVersionLen))
|
||||
for za0001 := range z {
|
||||
o = msgp.AppendUint64(o, z[za0001])
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// UnmarshalMsg implements msgp.Unmarshaler
|
||||
func (z *versionsHistogram) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
var zb0001 uint32
|
||||
zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
if zb0001 != uint32(dataUsageVersionLen) {
|
||||
err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0001}
|
||||
return
|
||||
}
|
||||
for za0001 := range z {
|
||||
z[za0001], bts, err = msgp.ReadUint64Bytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
o = bts
|
||||
return
|
||||
}
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *versionsHistogram) Msgsize() (s int) {
|
||||
s = msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size))
|
||||
return
|
||||
}
|
||||
|
@ -1196,3 +1196,116 @@ func BenchmarkDecodetierStats(b *testing.B) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalversionsHistogram(t *testing.T) {
|
||||
v := versionsHistogram{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgversionsHistogram(b *testing.B) {
|
||||
v := versionsHistogram{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgversionsHistogram(b *testing.B) {
|
||||
v := versionsHistogram{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalversionsHistogram(b *testing.B) {
|
||||
v := versionsHistogram{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeversionsHistogram(t *testing.T) {
|
||||
v := versionsHistogram{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeversionsHistogram Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := versionsHistogram{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeversionsHistogram(b *testing.B) {
|
||||
v := versionsHistogram{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeversionsHistogram(b *testing.B) {
|
||||
v := versionsHistogram{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -58,11 +58,12 @@ type BucketUsageInfo struct {
|
||||
// Total number of objects that failed replication
|
||||
ReplicationFailedCountV1 uint64 `json:"objectsFailedReplicationCount"`
|
||||
|
||||
ObjectsCount uint64 `json:"objectsCount"`
|
||||
ObjectSizesHistogram map[string]uint64 `json:"objectsSizesHistogram"`
|
||||
VersionsCount uint64 `json:"versionsCount"`
|
||||
ReplicaSize uint64 `json:"objectReplicaTotalSize"`
|
||||
ReplicationInfo map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
|
||||
ObjectsCount uint64 `json:"objectsCount"`
|
||||
ObjectSizesHistogram map[string]uint64 `json:"objectsSizesHistogram"`
|
||||
ObjectVersionsHistogram map[string]uint64 `json:"objectsVersionsHistogram"`
|
||||
VersionsCount uint64 `json:"versionsCount"`
|
||||
ReplicaSize uint64 `json:"objectReplicaTotalSize"`
|
||||
ReplicationInfo map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
|
||||
}
|
||||
|
||||
// DataUsageInfo represents data usage stats of the underlying Object API
|
||||
|
@ -183,8 +183,9 @@ const (
|
||||
usageInfo MetricName = "usage_info"
|
||||
versionInfo MetricName = "version_info"
|
||||
|
||||
sizeDistribution = "size_distribution"
|
||||
ttfbDistribution = "ttfb_seconds_distribution"
|
||||
sizeDistribution = "size_distribution"
|
||||
versionDistribution = "version_distribution"
|
||||
ttfbDistribution = "ttfb_seconds_distribution"
|
||||
|
||||
lastActivityTime = "last_activity_nano_seconds"
|
||||
startTime = "starttime_seconds"
|
||||
@ -565,6 +566,16 @@ func getBucketObjectDistributionMD() MetricDescription {
|
||||
}
|
||||
}
|
||||
|
||||
func getBucketObjectVersionsMD() MetricDescription {
|
||||
return MetricDescription{
|
||||
Namespace: bucketMetricNamespace,
|
||||
Subsystem: objectsSubsystem,
|
||||
Name: versionDistribution,
|
||||
Help: "Distribution of object sizes in the bucket, includes label for the bucket name",
|
||||
Type: histogramMetric,
|
||||
}
|
||||
}
|
||||
|
||||
func getInternodeFailedRequests() MetricDescription {
|
||||
return MetricDescription{
|
||||
Namespace: interNodeMetricNamespace,
|
||||
@ -1999,6 +2010,12 @@ func getBucketUsageMetrics() *MetricsGroup {
|
||||
HistogramBucketLabel: "range",
|
||||
VariableLabels: map[string]string{"bucket": bucket},
|
||||
})
|
||||
metrics = append(metrics, Metric{
|
||||
Description: getBucketObjectVersionsMD(),
|
||||
Histogram: usage.ObjectVersionsHistogram,
|
||||
HistogramBucketLabel: "range",
|
||||
VariableLabels: map[string]string{"bucket": bucket},
|
||||
})
|
||||
}
|
||||
return
|
||||
})
|
||||
|
@ -441,6 +441,18 @@ func bucketUsageMetricsPrometheus(ch chan<- prometheus.Metric) {
|
||||
k,
|
||||
)
|
||||
}
|
||||
for k, v := range usageInfo.ObjectVersionsHistogram {
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName(bucketNamespace, "objects", "histogram"),
|
||||
"Total number of versions of objects in a bucket",
|
||||
[]string{"bucket", "object_versions"}, nil),
|
||||
prometheus.GaugeValue,
|
||||
float64(v),
|
||||
bucket,
|
||||
k,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -54,12 +54,13 @@ type objectHistogramInterval struct {
|
||||
|
||||
const (
|
||||
// dataUsageBucketLen must be length of ObjectsHistogramIntervals
|
||||
dataUsageBucketLen = 7
|
||||
dataUsageBucketLen = 7
|
||||
dataUsageVersionLen = 7
|
||||
)
|
||||
|
||||
// ObjectsHistogramIntervals is the list of all intervals
|
||||
// of object sizes to be included in objects histogram.
|
||||
var ObjectsHistogramIntervals = []objectHistogramInterval{
|
||||
var ObjectsHistogramIntervals = [dataUsageBucketLen]objectHistogramInterval{
|
||||
{"LESS_THAN_1024_B", 0, humanize.KiByte - 1},
|
||||
{"BETWEEN_1024_B_AND_1_MB", humanize.KiByte, humanize.MiByte - 1},
|
||||
{"BETWEEN_1_MB_AND_10_MB", humanize.MiByte, humanize.MiByte*10 - 1},
|
||||
@ -69,6 +70,18 @@ var ObjectsHistogramIntervals = []objectHistogramInterval{
|
||||
{"GREATER_THAN_512_MB", humanize.MiByte * 512, math.MaxInt64},
|
||||
}
|
||||
|
||||
// ObjectsVersionCountIntervals is the list of all intervals
|
||||
// of object version count to be included in objects histogram.
|
||||
var ObjectsVersionCountIntervals = [dataUsageVersionLen]objectHistogramInterval{
|
||||
{"UNVERSIONED", 0, 0},
|
||||
{"SINGLE_VERSION", 1, 1},
|
||||
{"BETWEEN_2_AND_10", 2, 9},
|
||||
{"BETWEEN_10_AND_100", 10, 99},
|
||||
{"BETWEEN_100_AND_1000", 100, 999},
|
||||
{"BETWEEN_1000_AND_10000", 1000, 9999},
|
||||
{"GREATER_THAN_10000", 10000, math.MaxInt64},
|
||||
}
|
||||
|
||||
// BucketInfo - represents bucket metadata.
|
||||
type BucketInfo struct {
|
||||
// Name of the bucket.
|
||||
|
2
go.mod
2
go.mod
@ -47,7 +47,7 @@ require (
|
||||
github.com/minio/dperf v0.4.2
|
||||
github.com/minio/highwayhash v1.0.2
|
||||
github.com/minio/kes-go v0.1.0
|
||||
github.com/minio/madmin-go/v2 v2.0.15
|
||||
github.com/minio/madmin-go/v2 v2.0.16-0.20230302223330-683c505848dc
|
||||
github.com/minio/minio-go/v7 v7.0.49
|
||||
github.com/minio/mux v1.9.0
|
||||
github.com/minio/pkg v1.6.3
|
||||
|
4
go.sum
4
go.sum
@ -771,8 +771,8 @@ github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLT
|
||||
github.com/minio/kes-go v0.1.0 h1:h201DyOYP5sTqajkxFGxmXz/kPbT8HQNX1uh3Yx2PFc=
|
||||
github.com/minio/kes-go v0.1.0/go.mod h1:VorHLaIYis9/MxAHAtXN4d8PUMNKhIxTIlvFt0hBOEo=
|
||||
github.com/minio/madmin-go v1.6.6/go.mod h1:ATvkBOLiP3av4D++2v1UEHC/QzsGtgXD5kYvvRYzdKs=
|
||||
github.com/minio/madmin-go/v2 v2.0.15 h1:da/wvVryJXDpkE/gck+tsHfmd9XInaWvJmDKp/sBwuk=
|
||||
github.com/minio/madmin-go/v2 v2.0.15/go.mod h1:8bL1RMNkblIENFSgGYjeHrzUx9PxROb7OqfNuMU9ivE=
|
||||
github.com/minio/madmin-go/v2 v2.0.16-0.20230302223330-683c505848dc h1:o5QY2XRWb0RIi3/J4NY3dx1eHfWWj3RTpQ1TlLfNjD8=
|
||||
github.com/minio/madmin-go/v2 v2.0.16-0.20230302223330-683c505848dc/go.mod h1:8bL1RMNkblIENFSgGYjeHrzUx9PxROb7OqfNuMU9ivE=
|
||||
github.com/minio/mc v0.0.0-20230228001259-5fbe8c26bab5 h1:4QYXUlzVc2cG0yyTwCnV/RSr14bDvUBzc8X115BM2yo=
|
||||
github.com/minio/mc v0.0.0-20230228001259-5fbe8c26bab5/go.mod h1:CiT/i2btLPcn0HA6kAHe8DKjkQEnpESsW1HVU4EFMpI=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
|
Loading…
Reference in New Issue
Block a user