MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 12:52:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

Improve expiration of tiered objects (#18926)

Browse Source 
- Use a shared worker pool for all ILM expiry tasks
- Free version cleanup executes in a separate goroutine
- Add a free version only if removing the remote object fails
- Add ILM expiry metrics to the node namespace
- Move tier journal tasks to expiryState
- Remove unused on-disk journal for tiered objects pending deletion
- Distribute expiry tasks across workers such that the expiry of versions of
  the same object serialized
- Ability to resize worker pool without server restart
- Make scaling down of expiryState workers' concurrency safe; Thanks
  @klauspost
- Add error logs when expiryState and transition state are not
  initialized (yet)
* metrics: Add missed tier journal entry tasks
* Initialize the ILM worker pool after the object layer
This commit is contained in:
Krishnan Parthasarathi
2024-03-01 21:11:03 -08:00
committed by GitHub
parent 325fd80687
commit a7577da768
28 changed files with 611 additions and 1131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
internal/config/api/api.go
View File

@@ -41,6 +41,7 @@ const (
apiReplicationMaxWorkers = "replication_max_workers"
apiTransitionWorkers = "transition_workers"
apiExpiryWorkers = "expiry_workers"
apiStaleUploadsCleanupInterval = "stale_uploads_cleanup_interval"
apiStaleUploadsExpiry = "stale_uploads_expiry"
apiDeleteCleanupInterval = "delete_cleanup_interval"
@@ -56,6 +57,7 @@ const (
EnvAPICorsAllowOrigin = "MINIO_API_CORS_ALLOW_ORIGIN"
EnvAPIRemoteTransportDeadline = "MINIO_API_REMOTE_TRANSPORT_DEADLINE"
EnvAPITransitionWorkers = "MINIO_API_TRANSITION_WORKERS"
EnvAPIExpiryWorkers = "MINIO_API_EXPIRY_WORKERS"
EnvAPIListQuorum = "MINIO_API_LIST_QUORUM"
EnvAPISecureCiphers = "MINIO_API_SECURE_CIPHERS" // default config.EnableOn
EnvAPIReplicationPriority = "MINIO_API_REPLICATION_PRIORITY"
@@ -117,6 +119,10 @@ var (
Key: apiTransitionWorkers,
Value: "100",
},
config.KV{
Key: apiExpiryWorkers,
Value: "100",
},
config.KV{
Key: apiStaleUploadsCleanupInterval,
Value: "6h",
@@ -164,6 +170,7 @@ type Config struct {
ReplicationPriority string `json:"replication_priority"`
ReplicationMaxWorkers int `json:"replication_max_workers"`
TransitionWorkers int `json:"transition_workers"`
ExpiryWorkers int `json:"expiry_workers"`
StaleUploadsCleanupInterval time.Duration `json:"stale_uploads_cleanup_interval"`
StaleUploadsExpiry time.Duration `json:"stale_uploads_expiry"`
DeleteCleanupInterval time.Duration `json:"delete_cleanup_interval"`
@@ -281,6 +288,15 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
}
cfg.TransitionWorkers = transitionWorkers
expiryWorkers, err := strconv.Atoi(env.Get(EnvAPIExpiryWorkers, kvs.GetWithDefault(apiExpiryWorkers, DefaultKVS)))
if err != nil {
return cfg, err
}
if expiryWorkers <= 0 || expiryWorkers > 500 {
return cfg, config.ErrInvalidExpiryWorkersValue(nil).Msg("Number of expiry workers should be between 1 and 500")
}
cfg.ExpiryWorkers = expiryWorkers
v := env.Get(EnvAPIDeleteCleanupInterval, kvs.Get(apiDeleteCleanupInterval))
if v == "" {
v = env.Get(EnvDeleteCleanupInterval, kvs.GetWithDefault(apiDeleteCleanupInterval, DefaultKVS))

8
internal/config/api/help.go
View File

@@ -19,12 +19,12 @@ package api
import "github.com/minio/minio/internal/config"
// Help template for storageclass feature.
var (
defaultHelpPostfix = func(key string) string {
return config.DefaultHelpPostfix(DefaultKVS, key)
}
// Help holds configuration keys and their default values for api subsystem.
Help = config.HelpKVS{
config.HelpKV{
Key: apiRequestsMax,
@@ -80,6 +80,12 @@ var (
Optional: true,
Type: "number",
},
config.HelpKV{
Key: apiExpiryWorkers,
Description: `set the number of expiry workers` + defaultHelpPostfix(apiExpiryWorkers),
Optional: true,
Type: "number",
},
config.HelpKV{
Key: apiStaleUploadsExpiry,
Description: `set to expire stale multipart uploads older than this values` + defaultHelpPostfix(apiStaleUploadsExpiry),

3
internal/config/config.go
View File

@@ -120,6 +120,7 @@ const (
DriveSubSys = madmin.DriveSubSys
BatchSubSys = madmin.BatchSubSys
BrowserSubSys = madmin.BrowserSubSys
ILMSubSys = madmin.ILMSubsys
// Add new constants here (similar to above) if you add new fields to config.
)
@@ -188,6 +189,7 @@ var SubSystemsDynamic = set.CreateStringSet(
AuditKafkaSubSys,
StorageClassSubSys,
CacheSubSys,
ILMSubSys,
BatchSubSys,
BrowserSubSys,
)
@@ -211,6 +213,7 @@ var SubSystemsSingleTargets = set.CreateStringSet(
SubnetSubSys,
CallhomeSubSys,
DriveSubSys,
ILMSubSys,
BatchSubSys,
BrowserSubSys,
)

5
internal/config/errors.go
View File

@@ -224,6 +224,11 @@ Examples:
"",
"MINIO_API_TRANSITION_WORKERS: should be >= GOMAXPROCS/2",
)
ErrInvalidExpiryWorkersValue = newErrFn(
"Invalid value for expiry workers",
"",
"MINIO_API_EXPIRY_WORKERS: should be between 1 and 500",
)
ErrInvalidBatchKeyRotationWorkersWait = newErrFn(
"Invalid value for batch key rotation workers wait",
"Please input a non-negative duration",

52
internal/config/ilm/help.go Normal file
View File

@@ -0,0 +1,52 @@
// Copyright (c) 2015-2024 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ilm
import "github.com/minio/minio/internal/config"
const (
transitionWorkers = "transition_workers"
expirationWorkers = "expiration_workers"
// EnvILMTransitionWorkers env variable to configure number of transition workers
EnvILMTransitionWorkers = "MINIO_ILM_TRANSITION_WORKERS"
// EnvILMExpirationWorkers env variable to configure number of expiration workers
EnvILMExpirationWorkers = "MINIO_ILM_EXPIRATION_WORKERS"
)
var (
defaultHelpPostfix = func(key string) string {
return config.DefaultHelpPostfix(DefaultKVS, key)
}
// HelpILM holds configuration keys and their default values for the ILM
// subsystem
HelpILM = config.HelpKVS{
config.HelpKV{
Key: transitionWorkers,
Type: "number",
Description: `set the number of transition workers` + defaultHelpPostfix(transitionWorkers),
Optional: true,
},
config.HelpKV{
Key: expirationWorkers,
Type: "number",
Description: `set the number of expiration workers` + defaultHelpPostfix(expirationWorkers),
Optional: true,
},
}
)

60
internal/config/ilm/ilm.go Normal file
View File

@@ -0,0 +1,60 @@
// Copyright (c) 2015-2024 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ilm
import (
"strconv"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v2/env"
)
// DefaultKVS default configuration values for ILM subsystem
var DefaultKVS = config.KVS{
config.KV{
Key: transitionWorkers,
Value: "100",
},
config.KV{
Key: expirationWorkers,
Value: "100",
},
}
// Config represents the different configuration values for ILM subsystem
type Config struct {
TransitionWorkers int
ExpirationWorkers int
}
// LookupConfig - lookup ilm config and override with valid environment settings if any.
func LookupConfig(kvs config.KVS) (cfg Config, err error) {
tw, err := strconv.Atoi(env.Get(EnvILMTransitionWorkers, kvs.GetWithDefault(transitionWorkers, DefaultKVS)))
if err != nil {
return cfg, err
}
ew, err := strconv.Atoi(env.Get(EnvILMExpirationWorkers, kvs.GetWithDefault(expirationWorkers, DefaultKVS)))
if err != nil {
return cfg, err
}
cfg.TransitionWorkers = tw
cfg.ExpirationWorkers = ew
return cfg, nil
}