Improve expiration of tiered objects (#18926)

- Use a shared worker pool for all ILM expiry tasks
- Free version cleanup executes in a separate goroutine
- Add a free version only if removing the remote object fails
- Add ILM expiry metrics to the node namespace
- Move tier journal tasks to expiryState
- Remove unused on-disk journal for tiered objects pending deletion
- Distribute expiry tasks across workers such that the expiry of versions of
  the same object serialized
- Ability to resize worker pool without server restart
- Make scaling down of expiryState workers' concurrency safe; Thanks
  @klauspost
- Add error logs when expiryState and transition state are not
  initialized (yet)
* metrics: Add missed tier journal entry tasks
* Initialize the ILM worker pool after the object layer

internal/config/api/api.go

@@ -41,6 +41,7 @@ const (
 	apiReplicationMaxWorkers   = "replication_max_workers"
 
 	apiTransitionWorkers           = "transition_workers"
+	apiExpiryWorkers               = "expiry_workers"
 	apiStaleUploadsCleanupInterval = "stale_uploads_cleanup_interval"
 	apiStaleUploadsExpiry          = "stale_uploads_expiry"
 	apiDeleteCleanupInterval       = "delete_cleanup_interval"
@@ -56,6 +57,7 @@ const (
 	EnvAPICorsAllowOrigin             = "MINIO_API_CORS_ALLOW_ORIGIN"
 	EnvAPIRemoteTransportDeadline     = "MINIO_API_REMOTE_TRANSPORT_DEADLINE"
 	EnvAPITransitionWorkers           = "MINIO_API_TRANSITION_WORKERS"
+	EnvAPIExpiryWorkers               = "MINIO_API_EXPIRY_WORKERS"
 	EnvAPIListQuorum                  = "MINIO_API_LIST_QUORUM"
 	EnvAPISecureCiphers               = "MINIO_API_SECURE_CIPHERS" // default config.EnableOn
 	EnvAPIReplicationPriority         = "MINIO_API_REPLICATION_PRIORITY"
@@ -117,6 +119,10 @@ var (
 			Key:   apiTransitionWorkers,
 			Value: "100",
 		},
+		config.KV{
+			Key:   apiExpiryWorkers,
+			Value: "100",
+		},
 		config.KV{
 			Key:   apiStaleUploadsCleanupInterval,
 			Value: "6h",
@@ -164,6 +170,7 @@ type Config struct {
 	ReplicationPriority         string        `json:"replication_priority"`
 	ReplicationMaxWorkers       int           `json:"replication_max_workers"`
 	TransitionWorkers           int           `json:"transition_workers"`
+	ExpiryWorkers               int           `json:"expiry_workers"`
 	StaleUploadsCleanupInterval time.Duration `json:"stale_uploads_cleanup_interval"`
 	StaleUploadsExpiry          time.Duration `json:"stale_uploads_expiry"`
 	DeleteCleanupInterval       time.Duration `json:"delete_cleanup_interval"`
@@ -281,6 +288,15 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 	}
 	cfg.TransitionWorkers = transitionWorkers
 
+	expiryWorkers, err := strconv.Atoi(env.Get(EnvAPIExpiryWorkers, kvs.GetWithDefault(apiExpiryWorkers, DefaultKVS)))
+	if err != nil {
+		return cfg, err
+	}
+	if expiryWorkers <= 0 || expiryWorkers > 500 {
+		return cfg, config.ErrInvalidExpiryWorkersValue(nil).Msg("Number of expiry workers should be between 1 and 500")
+	}
+	cfg.ExpiryWorkers = expiryWorkers
+
 	v := env.Get(EnvAPIDeleteCleanupInterval, kvs.Get(apiDeleteCleanupInterval))
 	if v == "" {
 		v = env.Get(EnvDeleteCleanupInterval, kvs.GetWithDefault(apiDeleteCleanupInterval, DefaultKVS))

internal/config/api/help.go

@@ -19,12 +19,12 @@ package api
 
 import "github.com/minio/minio/internal/config"
 
-// Help template for storageclass feature.
 var (
 	defaultHelpPostfix = func(key string) string {
 		return config.DefaultHelpPostfix(DefaultKVS, key)
 	}
 
+	// Help holds configuration keys and their default values for api subsystem.
 	Help = config.HelpKVS{
 		config.HelpKV{
 			Key:         apiRequestsMax,
@@ -80,6 +80,12 @@ var (
 			Optional:    true,
 			Type:        "number",
 		},
+		config.HelpKV{
+			Key:         apiExpiryWorkers,
+			Description: `set the number of expiry workers` + defaultHelpPostfix(apiExpiryWorkers),
+			Optional:    true,
+			Type:        "number",
+		},
 		config.HelpKV{
 			Key:         apiStaleUploadsExpiry,
 			Description: `set to expire stale multipart uploads older than this values` + defaultHelpPostfix(apiStaleUploadsExpiry),

internal/config/config.go

@@ -120,6 +120,7 @@ const (
 	DriveSubSys          = madmin.DriveSubSys
 	BatchSubSys          = madmin.BatchSubSys
 	BrowserSubSys        = madmin.BrowserSubSys
+	ILMSubSys            = madmin.ILMSubsys
 
 	// Add new constants here (similar to above) if you add new fields to config.
 )
@@ -188,6 +189,7 @@ var SubSystemsDynamic = set.CreateStringSet(
 	AuditKafkaSubSys,
 	StorageClassSubSys,
 	CacheSubSys,
+	ILMSubSys,
 	BatchSubSys,
 	BrowserSubSys,
 )
@@ -211,6 +213,7 @@ var SubSystemsSingleTargets = set.CreateStringSet(
 	SubnetSubSys,
 	CallhomeSubSys,
 	DriveSubSys,
+	ILMSubSys,
 	BatchSubSys,
 	BrowserSubSys,
 )

internal/config/errors.go

@@ -224,6 +224,11 @@ Examples:
 		"",
 		"MINIO_API_TRANSITION_WORKERS: should be >= GOMAXPROCS/2",
 	)
+	ErrInvalidExpiryWorkersValue = newErrFn(
+		"Invalid value for expiry workers",
+		"",
+		"MINIO_API_EXPIRY_WORKERS: should be between 1 and 500",
+	)
 	ErrInvalidBatchKeyRotationWorkersWait = newErrFn(
 		"Invalid value for batch key rotation workers wait",
 		"Please input a non-negative duration",

internal/config/ilm/help.go

@@ -0,0 +1,52 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ilm
+
+import "github.com/minio/minio/internal/config"
+
+const (
+	transitionWorkers = "transition_workers"
+	expirationWorkers = "expiration_workers"
+	// EnvILMTransitionWorkers env variable to configure number of transition workers
+	EnvILMTransitionWorkers = "MINIO_ILM_TRANSITION_WORKERS"
+	// EnvILMExpirationWorkers env variable to configure number of expiration workers
+	EnvILMExpirationWorkers = "MINIO_ILM_EXPIRATION_WORKERS"
+)
+
+var (
+	defaultHelpPostfix = func(key string) string {
+		return config.DefaultHelpPostfix(DefaultKVS, key)
+	}
+
+	// HelpILM holds configuration keys and their default values for the ILM
+	// subsystem
+	HelpILM = config.HelpKVS{
+		config.HelpKV{
+			Key:         transitionWorkers,
+			Type:        "number",
+			Description: `set the number of transition workers` + defaultHelpPostfix(transitionWorkers),
+			Optional:    true,
+		},
+		config.HelpKV{
+			Key:         expirationWorkers,
+			Type:        "number",
+			Description: `set the number of expiration workers` + defaultHelpPostfix(expirationWorkers),
+			Optional:    true,
+		},
+	}
+)

internal/config/ilm/ilm.go

@@ -0,0 +1,60 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ilm
+
+import (
+	"strconv"
+
+	"github.com/minio/minio/internal/config"
+	"github.com/minio/pkg/v2/env"
+)
+
+// DefaultKVS default configuration values for ILM subsystem
+var DefaultKVS = config.KVS{
+	config.KV{
+		Key:   transitionWorkers,
+		Value: "100",
+	},
+	config.KV{
+		Key:   expirationWorkers,
+		Value: "100",
+	},
+}
+
+// Config represents the different configuration values for ILM subsystem
+type Config struct {
+	TransitionWorkers int
+	ExpirationWorkers int
+}
+
+// LookupConfig - lookup ilm config and override with valid environment settings if any.
+func LookupConfig(kvs config.KVS) (cfg Config, err error) {
+	tw, err := strconv.Atoi(env.Get(EnvILMTransitionWorkers, kvs.GetWithDefault(transitionWorkers, DefaultKVS)))
+	if err != nil {
+		return cfg, err
+	}
+
+	ew, err := strconv.Atoi(env.Get(EnvILMExpirationWorkers, kvs.GetWithDefault(expirationWorkers, DefaultKVS)))
+	if err != nil {
+		return cfg, err
+	}
+
+	cfg.TransitionWorkers = tw
+	cfg.ExpirationWorkers = ew
+	return cfg, nil
+}