// Copyright (c) 2015-2022 MinIO, Inc. // // This file is part of MinIO Object Storage stack // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . package cmd import ( "context" "errors" "fmt" "io" "math" "net/url" "strings" "time" "github.com/minio/madmin-go/v3" minio "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) type warmBackendMinIO struct { warmBackendS3 } var _ WarmBackend = (*warmBackendMinIO)(nil) const ( maxMultipartPutObjectSize = 1024 * 1024 * 1024 * 1024 * 5 maxPartsCount = 10000 maxPartSize = 1024 * 1024 * 1024 * 5 minPartSize = 1024 * 1024 * 64 // chosen by us to be optimal for HDDs ) // optimalPartInfo - calculate the optimal part info for a given // object size. // // NOTE: Assumption here is that for any object to be uploaded to any S3 compatible // object storage it will have the following parameters as constants. // // maxPartsCount - 10000 // maxMultipartPutObjectSize - 5TiB func optimalPartSize(objectSize int64) (partSize int64, err error) { // object size is '-1' set it to 5TiB. if objectSize == -1 { objectSize = maxMultipartPutObjectSize } // object size is larger than supported maximum. if objectSize > maxMultipartPutObjectSize { err = errors.New("entity too large") return } configuredPartSize := minPartSize // Use floats for part size for all calculations to avoid // overflows during float64 to int64 conversions. partSizeFlt := float64(objectSize / maxPartsCount) partSizeFlt = math.Ceil(partSizeFlt/float64(configuredPartSize)) * float64(configuredPartSize) // Part size. partSize = int64(partSizeFlt) if partSize == 0 { return minPartSize, nil } return partSize, nil } func (m *warmBackendMinIO) Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) { partSize, err := optimalPartSize(length) if err != nil { return remoteVersionID(""), err } res, err := m.client.PutObject(ctx, m.Bucket, m.getDest(object), r, length, minio.PutObjectOptions{ StorageClass: m.StorageClass, PartSize: uint64(partSize), DisableContentSha256: true, }) return remoteVersionID(res.VersionID), m.ToObjectError(err, object) } func newWarmBackendMinIO(conf madmin.TierMinIO, tier string) (*warmBackendMinIO, error) { // Validation of credentials if conf.AccessKey == "" || conf.SecretKey == "" { return nil, errors.New("both access and secret keys are required") } if conf.Bucket == "" { return nil, errors.New("no bucket name was provided") } u, err := url.Parse(conf.Endpoint) if err != nil { return nil, err } creds := credentials.NewStaticV4(conf.AccessKey, conf.SecretKey, "") getRemoteTierTargetInstanceTransportOnce.Do(func() { getRemoteTierTargetInstanceTransport = NewHTTPTransportWithTimeout(10 * time.Minute) }) opts := &minio.Options{ Creds: creds, Secure: u.Scheme == "https", Transport: getRemoteTierTargetInstanceTransport, TrailingHeaders: true, } client, err := minio.New(u.Host, opts) if err != nil { return nil, err } client.SetAppInfo(fmt.Sprintf("minio-tier-%s", tier), ReleaseTag) core := &minio.Core{Client: client} return &warmBackendMinIO{ warmBackendS3{ client: client, core: core, Bucket: conf.Bucket, Prefix: strings.TrimSuffix(conf.Prefix, slashSeparator), }, }, nil }