MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-07-16 04:11:51 -04:00
Code Issues Packages Projects Releases Wiki Activity

move erasure blockSize to 1MiB

Browse Source
This commit is contained in:
Harshavardhana 2021-03-18 21:53:56 -07:00
parent 29e7058ebf
commit ef1ea96044
6 changed files with 22 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
cmd/erasure-sets.go
View File

@ -29,6 +29,7 @@ import (
"time"
"github.com/dchest/siphash"
"github.com/dustin/go-humanize"
"github.com/google/uuid"
"github.com/minio/minio-go/v7/pkg/tags"
"github.com/minio/minio/cmd/config"
@ -375,14 +376,12 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
mutex := newNSLock(globalIsDistErasure)
// Number of buffers, max 2GB.
n := setCount * setDriveCount
if n > 100 {
n = 100
}
// Number of buffers, max 2GB
n := (2 * humanize.GiByte) / (blockSizeV2 * 2)
// Initialize byte pool once for all sets, bpool size is set to
// setCount * setDriveCount with each memory upto blockSizeV1.
bp := bpool.NewBytePoolCap(n, blockSizeV1, blockSizeV1*2)
bp := bpool.NewBytePoolCap(n, blockSizeV2, blockSizeV2*2)
for i := 0; i < setCount; i++ {
s.erasureDisks[i] = make([]StorageAPI, setDriveCount)

8
cmd/handler-api.go
View File

@ -47,13 +47,13 @@ func (t *apiConfig) init(cfg api.Config, setDriveCount int) {
stats, err := sys.GetStats()
if err != nil {
logger.LogIf(GlobalContext, err)
// Default to 16 GiB, not critical.
stats.TotalRAM = 16 << 30
// Default to 8 GiB, not critical.
stats.TotalRAM = 8 << 30
}
// max requests per node is calculated as
// total_ram / ram_per_request
// ram_per_request is 4MiB * setDriveCount + 2 * 10MiB (default erasure block size)
apiRequestsMaxPerNode = int(stats.TotalRAM / uint64(setDriveCount*readBlockSize+blockSizeV1*2))
// ram_per_request is 4MiB * setDriveCount + (2 * 10MiB (v1 erasure block size) + 2 * 1MiB (v2 erasure block size)
apiRequestsMaxPerNode = int(stats.TotalRAM / uint64(setDriveCount*readBlockSize+int(blockSizeV1*2+blockSizeV2*2)))
} else {
apiRequestsMaxPerNode = cfg.RequestsMax
if len(globalEndpoints.Hostnames()) > 0 {

3
cmd/object-api-common.go
View File

@ -31,6 +31,9 @@ const (
// Block size used for all internal operations version 1.
blockSizeV1 = 10 * humanize.MiByte
// Block size used for all internal operations version 2
blockSizeV2 = 1 * humanize.MiByte
// Staging buffer read size for all internal operations version 1.
readSizeV1 = 1 * humanize.MiByte

2
cmd/storage-datatypes.go
View File

@ -109,7 +109,7 @@ func newFileInfo(object string, dataBlocks, parityBlocks int) (fi FileInfo) {
Algorithm: erasureAlgorithm,
DataBlocks: dataBlocks,
ParityBlocks: parityBlocks,
BlockSize: blockSizeV1,
BlockSize: blockSizeV2,
Distribution: hashOrder(object, dataBlocks+parityBlocks),
}
return fi

8
cmd/utils.go
View File

@ -470,6 +470,8 @@ func newInternodeHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration)
Proxy: http.ProxyFromEnvironment,
DialContext: xhttp.DialContextWithDNSCache(globalDNSCache, xhttp.NewInternodeDialContext(dialTimeout)),
MaxIdleConnsPerHost: 1024,
WriteBufferSize: 32 << 10, // 32KiB moving up from 4KiB default
ReadBufferSize: 32 << 10, // 32KiB moving up from 4KiB default
IdleConnTimeout: 15 * time.Second,
ResponseHeaderTimeout: 3 * time.Minute, // Set conservative timeouts for MinIO internode.
TLSHandshakeTimeout: 15 * time.Second,
@ -515,6 +517,8 @@ func newCustomHTTPProxyTransport(tlsConfig *tls.Config, dialTimeout time.Duratio
Proxy: http.ProxyFromEnvironment,
DialContext: xhttp.DialContextWithDNSCache(globalDNSCache, xhttp.NewInternodeDialContext(dialTimeout)),
MaxIdleConnsPerHost: 1024,
WriteBufferSize: 16 << 10, // 16KiB moving up from 4KiB default
ReadBufferSize: 16 << 10, // 16KiB moving up from 4KiB default
IdleConnTimeout: 15 * time.Second,
ResponseHeaderTimeout: 30 * time.Minute, // Set larger timeouts for proxied requests.
TLSHandshakeTimeout: 10 * time.Second,
@ -538,6 +542,8 @@ func newCustomHTTPTransportWithHTTP2(tlsConfig *tls.Config, dialTimeout time.Dur
Proxy: http.ProxyFromEnvironment,
DialContext: xhttp.DialContextWithDNSCache(globalDNSCache, xhttp.NewInternodeDialContext(dialTimeout)),
MaxIdleConnsPerHost: 1024,
WriteBufferSize: 16 << 10, // 16KiB moving up from 4KiB default
ReadBufferSize: 16 << 10, // 16KiB moving up from 4KiB default
IdleConnTimeout: 15 * time.Second,
ResponseHeaderTimeout: 3 * time.Minute, // Set conservative timeouts for MinIO internode.
TLSHandshakeTimeout: 10 * time.Second,
@ -568,6 +574,8 @@ func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration) fu
Proxy: http.ProxyFromEnvironment,
DialContext: xhttp.DialContextWithDNSCache(globalDNSCache, xhttp.NewInternodeDialContext(dialTimeout)),
MaxIdleConnsPerHost: 1024,
WriteBufferSize: 16 << 10, // 16KiB moving up from 4KiB default
ReadBufferSize: 16 << 10, // 16KiB moving up from 4KiB default
IdleConnTimeout: 15 * time.Second,
ResponseHeaderTimeout: 3 * time.Minute, // Set conservative timeouts for MinIO internode.
TLSHandshakeTimeout: 10 * time.Second,

2
cmd/xl-storage.go
View File

@ -53,7 +53,7 @@ import (
const (
nullVersionID = "null"
diskMinTotalSpace = 900 * humanize.MiByte // Min 900MiB total space.
readBlockSize = 4 * humanize.MiByte // Default read block size 4MiB.
readBlockSize = 2 * humanize.MiByte // Default read block size 2MiB.
// On regular files bigger than this;
readAheadSize = 16 << 20