mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
Add prometheus metrics for ILM tasks (#12933)
This commit is contained in:
parent
17e0de1e87
commit
cf8abd8888
@ -26,6 +26,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
@ -83,6 +84,11 @@ type expiryState struct {
|
|||||||
expiryCh chan expiryTask
|
expiryCh chan expiryTask
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PendingTasks returns the number of pending ILM expiry tasks.
|
||||||
|
func (es *expiryState) PendingTasks() int {
|
||||||
|
return len(es.expiryCh)
|
||||||
|
}
|
||||||
|
|
||||||
func (es *expiryState) queueExpiryTask(oi ObjectInfo, rmVersion bool) {
|
func (es *expiryState) queueExpiryTask(oi ObjectInfo, rmVersion bool) {
|
||||||
select {
|
select {
|
||||||
case <-GlobalContext.Done():
|
case <-GlobalContext.Done():
|
||||||
@ -122,6 +128,8 @@ type transitionState struct {
|
|||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
numWorkers int
|
numWorkers int
|
||||||
killCh chan struct{}
|
killCh chan struct{}
|
||||||
|
|
||||||
|
activeTasks int32
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *transitionState) queueTransitionTask(oi ObjectInfo) {
|
func (t *transitionState) queueTransitionTask(oi ObjectInfo) {
|
||||||
@ -148,6 +156,17 @@ func newTransitionState(ctx context.Context, objAPI ObjectLayer) *transitionStat
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PendingTasks returns the number of ILM transition tasks waiting for a worker
|
||||||
|
// goroutine.
|
||||||
|
func (t *transitionState) PendingTasks() int {
|
||||||
|
return len(globalTransitionState.transitionCh)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ActiveTasks returns the number of active (ongoing) ILM transition tasks.
|
||||||
|
func (t *transitionState) ActiveTasks() int {
|
||||||
|
return int(atomic.LoadInt32(&t.activeTasks))
|
||||||
|
}
|
||||||
|
|
||||||
// worker waits for transition tasks
|
// worker waits for transition tasks
|
||||||
func (t *transitionState) worker(ctx context.Context, objectAPI ObjectLayer) {
|
func (t *transitionState) worker(ctx context.Context, objectAPI ObjectLayer) {
|
||||||
for {
|
for {
|
||||||
@ -160,10 +179,11 @@ func (t *transitionState) worker(ctx context.Context, objectAPI ObjectLayer) {
|
|||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
atomic.AddInt32(&t.activeTasks, 1)
|
||||||
if err := transitionObject(ctx, objectAPI, oi); err != nil {
|
if err := transitionObject(ctx, objectAPI, oi); err != nil {
|
||||||
logger.LogIf(ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w", oi.Bucket, oi.Name, oi.VersionID, err))
|
logger.LogIf(ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w", oi.Bucket, oi.Name, oi.VersionID, err))
|
||||||
}
|
}
|
||||||
|
atomic.AddInt32(&t.activeTasks, -1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -68,6 +68,7 @@ const (
|
|||||||
softwareSubsystem MetricSubsystem = "software"
|
softwareSubsystem MetricSubsystem = "software"
|
||||||
sysCallSubsystem MetricSubsystem = "syscall"
|
sysCallSubsystem MetricSubsystem = "syscall"
|
||||||
usageSubsystem MetricSubsystem = "usage"
|
usageSubsystem MetricSubsystem = "usage"
|
||||||
|
ilmSubsystem MetricSubsystem = "ilm"
|
||||||
)
|
)
|
||||||
|
|
||||||
// MetricName are the individual names for the metric.
|
// MetricName are the individual names for the metric.
|
||||||
@ -121,6 +122,10 @@ const (
|
|||||||
upTime = "uptime_seconds"
|
upTime = "uptime_seconds"
|
||||||
memory = "resident_memory_bytes"
|
memory = "resident_memory_bytes"
|
||||||
cpu = "cpu_total_seconds"
|
cpu = "cpu_total_seconds"
|
||||||
|
|
||||||
|
expiryPendingTasks MetricName = "expiry_pending_tasks"
|
||||||
|
transitionPendingTasks MetricName = "transition_pending_tasks"
|
||||||
|
transitionActiveTasks MetricName = "transition_active_tasks"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -249,6 +254,7 @@ func GetGeneratorsForPeer() []MetricsGenerator {
|
|||||||
getMinioVersionMetrics,
|
getMinioVersionMetrics,
|
||||||
getNetworkMetrics,
|
getNetworkMetrics,
|
||||||
getS3TTFBMetric,
|
getS3TTFBMetric,
|
||||||
|
getILMNodeMetrics,
|
||||||
}
|
}
|
||||||
return g
|
return g
|
||||||
}
|
}
|
||||||
@ -1000,6 +1006,66 @@ func getS3TTFBMetric() MetricsGroup {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getTransitionPendingTasksMD() MetricDescription {
|
||||||
|
return MetricDescription{
|
||||||
|
Namespace: nodeMetricNamespace,
|
||||||
|
Subsystem: ilmSubsystem,
|
||||||
|
Name: transitionPendingTasks,
|
||||||
|
Help: "Number of pending ILM transition tasks in the queue.",
|
||||||
|
Type: gaugeMetric,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func getTransitionActiveTasksMD() MetricDescription {
|
||||||
|
return MetricDescription{
|
||||||
|
Namespace: nodeMetricNamespace,
|
||||||
|
Subsystem: ilmSubsystem,
|
||||||
|
Name: transitionActiveTasks,
|
||||||
|
Help: "Number of active ILM transition tasks.",
|
||||||
|
Type: gaugeMetric,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func getExpiryPendingTasksMD() MetricDescription {
|
||||||
|
return MetricDescription{
|
||||||
|
Namespace: nodeMetricNamespace,
|
||||||
|
Subsystem: ilmSubsystem,
|
||||||
|
Name: expiryPendingTasks,
|
||||||
|
Help: "Number of pending ILM expiry tasks in the queue.",
|
||||||
|
Type: gaugeMetric,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func getILMNodeMetrics() MetricsGroup {
|
||||||
|
return MetricsGroup{
|
||||||
|
id: "ILMNodeMetrics",
|
||||||
|
cachedRead: cachedRead,
|
||||||
|
read: func(_ context.Context) []Metric {
|
||||||
|
expPendingTasks := Metric{
|
||||||
|
Description: getExpiryPendingTasksMD(),
|
||||||
|
}
|
||||||
|
trPendingTasks := Metric{
|
||||||
|
Description: getTransitionPendingTasksMD(),
|
||||||
|
}
|
||||||
|
trActiveTasks := Metric{
|
||||||
|
Description: getTransitionActiveTasksMD(),
|
||||||
|
}
|
||||||
|
if globalExpiryState != nil {
|
||||||
|
expPendingTasks.Value = float64(globalExpiryState.PendingTasks())
|
||||||
|
}
|
||||||
|
if globalTransitionState != nil {
|
||||||
|
trPendingTasks.Value = float64(globalTransitionState.PendingTasks())
|
||||||
|
trActiveTasks.Value = float64(globalTransitionState.ActiveTasks())
|
||||||
|
}
|
||||||
|
return []Metric{
|
||||||
|
expPendingTasks,
|
||||||
|
trPendingTasks,
|
||||||
|
trActiveTasks,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func getMinioVersionMetrics() MetricsGroup {
|
func getMinioVersionMetrics() MetricsGroup {
|
||||||
return MetricsGroup{
|
return MetricsGroup{
|
||||||
id: "MinioVersionMetrics",
|
id: "MinioVersionMetrics",
|
||||||
|
Loading…
Reference in New Issue
Block a user