From 0a56dbde2f9b5bfe021c2231e08805617b888cbd Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 26 Mar 2024 15:06:19 -0700 Subject: [PATCH] allow configuring inline shard size value (#19336) --- cmd/erasure-object.go | 14 +++-- internal/config/storageclass/help.go | 4 +- internal/config/storageclass/storage-class.go | 54 +++++++++++++++++-- 3 files changed, 63 insertions(+), 9 deletions(-) diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 3ad9e03c8..e87313d13 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -31,6 +31,7 @@ import ( "sync" "time" + "github.com/dustin/go-humanize" "github.com/klauspost/readahead" "github.com/minio/madmin-go/v3" "github.com/minio/minio-go/v7/pkg/tags" @@ -1396,20 +1397,25 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st defer er.deleteAll(context.Background(), minioMetaTmpBucket, tempObj) shardFileSize := erasure.ShardFileSize(data.Size()) + inlineBlock := globalStorageClass.InlineBlock() + if inlineBlock <= 0 { + inlineBlock = 128 * humanize.KiByte + } + writers := make([]io.Writer, len(onlineDisks)) var inlineBuffers []*bytes.Buffer if shardFileSize >= 0 { - if !opts.Versioned && shardFileSize < smallFileThreshold { + if !opts.Versioned && shardFileSize < inlineBlock { inlineBuffers = make([]*bytes.Buffer, len(onlineDisks)) - } else if shardFileSize < smallFileThreshold/8 { + } else if shardFileSize < inlineBlock/8 { inlineBuffers = make([]*bytes.Buffer, len(onlineDisks)) } } else { // If compressed, use actual size to determine. if sz := erasure.ShardFileSize(data.ActualSize()); sz > 0 { - if !opts.Versioned && sz < smallFileThreshold { + if !opts.Versioned && sz < inlineBlock { inlineBuffers = make([]*bytes.Buffer, len(onlineDisks)) - } else if sz < smallFileThreshold/8 { + } else if sz < inlineBlock/8 { inlineBuffers = make([]*bytes.Buffer, len(onlineDisks)) } } diff --git a/internal/config/storageclass/help.go b/internal/config/storageclass/help.go index a6af8b21c..0b57085e1 100644 --- a/internal/config/storageclass/help.go +++ b/internal/config/storageclass/help.go @@ -39,8 +39,8 @@ var ( Type: "string", }, config.HelpKV{ - Key: ClassOptimize, - Description: `optimize parity calculation for standard storage class, set 'capacity' for capacity optimized (no additional parity)` + defaultHelpPostfix(ClassOptimize), + Key: Optimize, + Description: `optimize parity calculation for standard storage class, set 'capacity' for capacity optimized (no additional parity)` + defaultHelpPostfix(Optimize), Optional: true, Type: "string", }, diff --git a/internal/config/storageclass/storage-class.go b/internal/config/storageclass/storage-class.go index f0a430987..4b537967f 100644 --- a/internal/config/storageclass/storage-class.go +++ b/internal/config/storageclass/storage-class.go @@ -18,13 +18,16 @@ package storageclass import ( + "context" "encoding/json" "fmt" "strconv" "strings" "sync" + "github.com/dustin/go-humanize" "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/logger" "github.com/minio/pkg/v2/env" ) @@ -40,7 +43,8 @@ const ( const ( ClassStandard = "standard" ClassRRS = "rrs" - ClassOptimize = "optimize" + Optimize = "optimize" + InlineBlock = "inline_block" // Reduced redundancy storage class environment variable RRSEnv = "MINIO_STORAGE_CLASS_RRS" @@ -48,6 +52,14 @@ const ( StandardEnv = "MINIO_STORAGE_CLASS_STANDARD" // Optimize storage class environment variable OptimizeEnv = "MINIO_STORAGE_CLASS_OPTIMIZE" + // Inline block indicates the size of the shard + // that is considered for inlining, remember this + // shard value is the value per drive shard it + // will vary based on the parity that is configured + // for the STANDARD storage_class. + // inlining means data and metadata are written + // together in a single file i.e xl.meta + InlineBlockEnv = "MINIO_STORAGE_CLASS_INLINE_BLOCK" // Supported storage class scheme is EC schemePrefix = "EC" @@ -71,9 +83,14 @@ var ( Value: "EC:1", }, config.KV{ - Key: ClassOptimize, + Key: Optimize, Value: "availability", }, + config.KV{ + Key: InlineBlock, + Value: "", + HiddenIfEmpty: true, + }, } ) @@ -90,6 +107,8 @@ type Config struct { Standard StorageClass `json:"standard"` RRS StorageClass `json:"rrs"` Optimize string `json:"optimize"` + inlineBlock int64 + initialized bool } @@ -253,6 +272,19 @@ func (sCfg *Config) GetParityForSC(sc string) (parity int) { } } +// InlineBlock indicates the size of the block which will be used to inline +// an erasure shard and written along with xl.meta on the drive, on a versioned +// bucket this value is automatically chosen to 1/8th of the this value, make +// sure to put this into consideration when choosing this value. +func (sCfg *Config) InlineBlock() int64 { + ConfigLock.RLock() + defer ConfigLock.RUnlock() + if !sCfg.initialized { + return 128 * humanize.KiByte + } + return sCfg.inlineBlock +} + // CapacityOptimized - returns true if the storage-class is capacity optimized // meaning we will not use additional parities when drives are offline. // @@ -355,7 +387,23 @@ func LookupConfig(kvs config.KVS, setDriveCount int) (cfg Config, err error) { return Config{}, err } + cfg.Optimize = env.Get(OptimizeEnv, kvs.Get(Optimize)) + + inlineBlockStr := env.Get(InlineBlockEnv, kvs.Get(InlineBlock)) + if inlineBlockStr != "" { + inlineBlock, err := humanize.ParseBytes(inlineBlockStr) + if err != nil { + return cfg, err + } + if inlineBlock > 128*humanize.KiByte { + logger.LogOnceIf(context.Background(), fmt.Errorf("inline block value bigger than recommended max of 128KiB -> %s, performance may degrade for PUT please benchmark the changes", inlineBlockStr), inlineBlockStr) + } + cfg.inlineBlock = int64(inlineBlock) + } else { + cfg.inlineBlock = 128 * humanize.KiByte + } + cfg.initialized = true - cfg.Optimize = env.Get(OptimizeEnv, kvs.Get(ClassOptimize)) + return cfg, nil }