From d0a0eb97383d23cb12ac5d5b32906c31f1561def Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 24 May 2023 22:51:07 -0700 Subject: [PATCH] support fan-out objects via PostUpload() (#17233) --- cmd/bucket-handlers.go | 243 ++++++++++++++++++++++++------ cmd/post-policy-fan-out.go | 114 ++++++++++++++ docs/extensions/fan-out/README.md | 18 +++ go.mod | 16 +- go.sum | 30 ++-- internal/hash/reader.go | 15 ++ 6 files changed, 367 insertions(+), 69 deletions(-) create mode 100644 cmd/post-policy-fan-out.go create mode 100644 docs/extensions/fan-out/README.md diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index 4dba722f2..59772d131 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -21,6 +21,7 @@ import ( "bytes" "context" "encoding/base64" + "encoding/json" "encoding/xml" "errors" "fmt" @@ -30,6 +31,7 @@ import ( "net/textproto" "net/url" "path" + "runtime" "sort" "strconv" "strings" @@ -37,8 +39,10 @@ import ( "github.com/google/uuid" "github.com/minio/mux" + "github.com/valyala/bytebufferpool" "github.com/minio/madmin-go/v2" + "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/set" "github.com/minio/minio-go/v7/pkg/tags" "github.com/minio/minio/internal/auth" @@ -51,6 +55,7 @@ import ( "github.com/minio/minio/internal/handlers" "github.com/minio/minio/internal/hash" xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/ioutil" "github.com/minio/minio/internal/kms" "github.com/minio/minio/internal/logger" "github.com/minio/pkg/bucket/policy" @@ -889,14 +894,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h } bucket := mux.Vars(r)["bucket"] - - // Require Content-Length to be set in the request - size := r.ContentLength - if size < 0 { - writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL) - return - } - resource, err := getResource(r.URL.Path, r.Host, globalDomainNames) if err != nil { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) @@ -911,7 +908,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h // Here the parameter is the size of the form data that should // be loaded in memory, the remaining being put in temporary files. - reader, err := r.MultipartReader() + mp, err := r.MultipartReader() if err != nil { apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest) apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err) @@ -919,16 +916,20 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h return } + const mapEntryOverhead = 200 + var ( - fileBody io.ReadCloser - fileName string + reader io.Reader + fileSize int64 = -1 + fileName string + fanOutEntries = make([]minio.PutObjectFanOutEntry, 0, 100) ) maxParts := 1000 // Canonicalize the form values into http.Header. formValues := make(http.Header) for { - part, err := reader.NextRawPart() + part, err := mp.NextRawPart() if errors.Is(err, io.EOF) { break } @@ -956,7 +957,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h // Multiple values for the same key (one map entry, longer slice) are cheaper // than the same number of values for different keys (many map entries), but // using a consistent per-value cost for overhead is simpler. - const mapEntryOverhead = 200 maxMemoryBytes := 2 * int64(10<<20) maxMemoryBytes -= int64(len(name)) maxMemoryBytes -= mapEntryOverhead @@ -971,9 +971,29 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h var b bytes.Buffer if fileName == "" { + if http.CanonicalHeaderKey(name) == http.CanonicalHeaderKey("x-minio-fanout-list") { + dec := json.NewDecoder(part) + + // while the array contains values + for dec.More() { + var m minio.PutObjectFanOutEntry + if err := dec.Decode(&m); err != nil { + part.Close() + apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest) + apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, multipart.ErrMessageTooLarge) + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + fanOutEntries = append(fanOutEntries, m) + } + part.Close() + continue + } + // value, store as string in memory n, err := io.CopyN(&b, part, maxMemoryBytes+1) part.Close() + if err != nil && err != io.EOF { apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest) apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err) @@ -1001,8 +1021,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h // The file or text content. // The file or text content must be the last field in the form. // You cannot upload more than one file at a time. - fileBody = part // we have found the File part of the request breakout - defer part.Close() + reader = part + // we have found the File part of the request we are done processing multipart-form break } @@ -1012,6 +1032,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h writeErrorResponse(ctx, w, apiErr, r.URL) return } + if fileName == "" { apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest) apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, errors.New("The file or text content is missing")) @@ -1059,20 +1080,38 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h return } - // Once signature is validated, check if the user has - // explicit permissions for the user. - if !globalIAMSys.IsAllowed(iampolicy.Args{ - AccountName: cred.AccessKey, - Groups: cred.Groups, - Action: iampolicy.PutObjectAction, - ConditionValues: getConditionValues(r, "", cred), - BucketName: bucket, - ObjectName: object, - IsOwner: globalActiveCred.AccessKey == cred.AccessKey, - Claims: cred.Claims, - }) { - writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) - return + if len(fanOutEntries) > 0 { + // Once signature is validated, check if the user has + // explicit permissions for the user. + if !globalIAMSys.IsAllowed(iampolicy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: iampolicy.PutObjectFanOutAction, + ConditionValues: getConditionValues(r, "", cred), + BucketName: bucket, + ObjectName: object, + IsOwner: globalActiveCred.AccessKey == cred.AccessKey, + Claims: cred.Claims, + }) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + } else { + // Once signature is validated, check if the user has + // explicit permissions for the user. + if !globalIAMSys.IsAllowed(iampolicy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: iampolicy.PutObjectAction, + ConditionValues: getConditionValues(r, "", cred), + BucketName: bucket, + ObjectName: object, + IsOwner: globalActiveCred.AccessKey == cred.AccessKey, + Claims: cred.Claims, + }) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } } policyBytes, err := base64.StdEncoding.DecodeString(formValues.Get("Policy")) @@ -1081,15 +1120,14 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h return } - hashReader, err := hash.NewReader(fileBody, -1, "", "", -1) + hashReader, err := hash.NewReader(reader, fileSize, "", "", fileSize) if err != nil { logger.LogIf(ctx, err) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } if checksum != nil && checksum.Valid() { - err = hashReader.AddChecksum(r, false) - if err != nil { + if err = hashReader.AddChecksumNoTrailer(formValues, false); err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } @@ -1134,7 +1172,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h // Check if bucket encryption is enabled sseConfig, _ := globalBucketSSEConfigSys.Get(bucket) - sseConfig.Apply(r.Header, sse.ApplyOptions{ + sseConfig.Apply(formValues, sse.ApplyOptions{ AutoEncrypt: globalAutoEncryption, }) @@ -1145,6 +1183,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h return } + fanOutOpts := fanOutOptions{Checksum: checksum} + if crypto.Requested(formValues) { if crypto.SSECopy.IsRequested(r.Header) { writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL) @@ -1177,29 +1217,137 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h return } } - reader, objectEncryptionKey, err = newEncryptReader(ctx, hashReader, kind, keyID, key, bucket, object, metadata, kmsCtx) - if err != nil { - writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) - return - } - // do not try to verify encrypted content/ - hashReader, err = hash.NewReader(reader, -1, "", "", -1) - if err != nil { - writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) - return - } - if checksum != nil && checksum.Valid() { - err = hashReader.AddChecksum(r, true) + + if len(fanOutEntries) == 0 { + reader, objectEncryptionKey, err = newEncryptReader(ctx, hashReader, kind, keyID, key, bucket, object, metadata, kmsCtx) if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } + // do not try to verify encrypted content/ + hashReader, err = hash.NewReader(reader, -1, "", "", -1) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if checksum != nil && checksum.Valid() { + if err = hashReader.AddChecksumNoTrailer(formValues, true); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } else { + fanOutOpts = fanOutOptions{ + Key: key, + Kind: kind, + KeyID: keyID, + KmsCtx: kmsCtx, + Checksum: checksum, + } } - pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey) + } + + if len(fanOutEntries) > 0 { + // Fan-out requires no copying, and must be carried from original source + // https://en.wikipedia.org/wiki/Copy_protection so the incoming stream + // is always going to be in-memory as we cannot re-read from what we + // wrote to disk - since that amounts to "copying" from a "copy" + // instead of "copying" from source, we need the stream to be seekable + // to ensure that we can make fan-out calls concurrently. + buf := bytebufferpool.Get() + defer bytebufferpool.Put(buf) + + // Maximum allowed fan-out object size. + const maxFanOutSize = 16 << 20 + + n, err := io.Copy(buf, ioutil.HardLimitReader(pReader, maxFanOutSize)) if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } + + concurrentSize := 100 + if runtime.GOMAXPROCS(0) < concurrentSize { + concurrentSize = runtime.GOMAXPROCS(0) + } + + fanOutResp := make([]minio.PutObjectFanOutResponse, 0, len(fanOutEntries)) + eventArgsList := make([]eventArgs, 0, len(fanOutEntries)) + for { + var objInfos []ObjectInfo + var errs []error + + var done bool + if len(fanOutEntries) < concurrentSize { + objInfos, errs = fanOutPutObject(ctx, bucket, objectAPI, fanOutEntries, buf.Bytes()[:n], fanOutOpts) + done = true + } else { + objInfos, errs = fanOutPutObject(ctx, bucket, objectAPI, fanOutEntries[:concurrentSize], buf.Bytes()[:n], fanOutOpts) + fanOutEntries = fanOutEntries[concurrentSize:] + } + + for i, objInfo := range objInfos { + if errs[i] != nil { + fanOutResp = append(fanOutResp, minio.PutObjectFanOutResponse{ + Key: objInfo.Name, + Error: errs[i], + }) + continue + } + + fanOutResp = append(fanOutResp, minio.PutObjectFanOutResponse{ + Key: objInfo.Name, + ETag: getDecryptedETag(formValues, objInfo, false), + VersionID: objInfo.VersionID, + LastModified: &objInfo.ModTime, + }) + + eventArgsList = append(eventArgsList, eventArgs{ + EventName: event.ObjectCreatedPost, + BucketName: objInfo.Bucket, + Object: objInfo, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent() + " " + "MinIO-Fan-Out", + Host: handlers.GetSourceIP(r), + }) + } + + if done { + break + } + } + + enc := json.NewEncoder(w) + for i, fanOutResp := range fanOutResp { + if err = enc.Encode(&fanOutResp); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Notify object created events. + sendEvent(eventArgsList[i]) + + if eventArgsList[i].Object.NumVersions > dataScannerExcessiveVersionsThreshold { + // Send events for excessive versions. + sendEvent(eventArgs{ + EventName: event.ObjectManyVersions, + BucketName: eventArgsList[i].Object.Bucket, + Object: eventArgsList[i].Object, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent() + " " + "MinIO-Fan-Out", + Host: handlers.GetSourceIP(r), + }) + } + } + + return } objInfo, err := objectAPI.PutObject(ctx, bucket, object, pReader, opts) @@ -1232,6 +1380,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h UserAgent: r.UserAgent(), Host: handlers.GetSourceIP(r), }) + if objInfo.NumVersions > dataScannerExcessiveVersionsThreshold { defer sendEvent(eventArgs{ EventName: event.ObjectManyVersions, diff --git a/cmd/post-policy-fan-out.go b/cmd/post-policy-fan-out.go new file mode 100644 index 000000000..c915a35ee --- /dev/null +++ b/cmd/post-policy-fan-out.go @@ -0,0 +1,114 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "sync" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/s3utils" + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/kms" +) + +type fanOutOptions struct { + Kind crypto.Type + KeyID string + Key []byte + KmsCtx kms.Context + Checksum *hash.Checksum +} + +// fanOutPutObject takes an input source reader and fans out multiple PUT operations +// based on the incoming fan-out request, a context cancelation by the caller +// would ensure all fan-out operations are canceled. +func fanOutPutObject(ctx context.Context, bucket string, objectAPI ObjectLayer, fanOutEntries []minio.PutObjectFanOutEntry, fanOutBuf []byte, opts fanOutOptions) ([]ObjectInfo, []error) { + errs := make([]error, len(fanOutEntries)) + objInfos := make([]ObjectInfo, len(fanOutEntries)) + + var wg sync.WaitGroup + for i, req := range fanOutEntries { + wg.Add(1) + go func(idx int, req minio.PutObjectFanOutEntry) { + defer wg.Done() + + objInfos[idx] = ObjectInfo{Name: req.Key} + + hr, err := hash.NewReader(bytes.NewReader(fanOutBuf), int64(len(fanOutBuf)), "", "", -1) + if err != nil { + errs[idx] = err + return + } + + reader := NewPutObjReader(hr) + defer func() { + if err := reader.Close(); err != nil { + errs[idx] = err + } + if err := hr.Close(); err != nil { + errs[idx] = err + } + }() + + userDefined := make(map[string]string, len(req.UserMetadata)) + for k, v := range req.UserMetadata { + userDefined[k] = v + } + userDefined[xhttp.AmzObjectTagging] = s3utils.TagEncode(req.UserTags) + + if opts.Kind != nil { + encrd, objectEncryptionKey, err := newEncryptReader(ctx, hr, opts.Kind, opts.KeyID, opts.Key, bucket, req.Key, userDefined, opts.KmsCtx) + if err != nil { + errs[idx] = err + return + } + + // do not try to verify encrypted content/ + hr, err = hash.NewReader(encrd, -1, "", "", -1) + if err != nil { + errs[idx] = err + return + } + + reader, err = reader.WithEncryption(hr, &objectEncryptionKey) + if err != nil { + errs[idx] = err + return + } + } + + objInfo, err := objectAPI.PutObject(ctx, bucket, req.Key, reader, ObjectOptions{ + Versioned: globalBucketVersioningSys.PrefixEnabled(bucket, req.Key), + VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, req.Key), + UserDefined: userDefined, + }) + if err != nil { + errs[idx] = err + return + } + objInfos[idx] = objInfo + }(i, req) + } + wg.Wait() + + return objInfos, errs +} diff --git a/docs/extensions/fan-out/README.md b/docs/extensions/fan-out/README.md new file mode 100644 index 000000000..2f828e7db --- /dev/null +++ b/docs/extensions/fan-out/README.md @@ -0,0 +1,18 @@ +# Fan-Out Uploads [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) + +## Overview + +MinIO implements an S3 extension to perform multiple concurrent fan-out upload operations. A perfect use case scenario for performing fan-out operations of incoming TSB (Time Shift Buffer's). TSBs are a method of facilitating time-shifted playback of television signaling, and media content. + +MinIO implements an S3 extension to the [PostUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPOST.html) where in a special fan-out list is sent along with the TSB's for MinIO make multiple uploads from a single source stream. Optionally supports custom metadata, tags and other retention settings. All objects are also readable independently once upload is completed via the regular S3 [GetObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html) API. + +## How to enable Fan-Out Uploads ? + +Fan-Out uploads are automatically enabled if `x-minio-fanout-list` form-field is provided with the PostUpload API, to keep things simple higher level APIs are provided in our SDKs for example in `minio-go` SDK: + +``` +PutObjectFanOut(ctx context.Context, bucket string, fanOutContent io.Reader, fanOutReq minio.PutObjectFanOutRequest) ([]minio.PutObjectFanOutResponse, error) +``` + + + diff --git a/go.mod b/go.mod index d50226c24..7012708fb 100644 --- a/go.mod +++ b/go.mod @@ -49,11 +49,11 @@ require ( github.com/minio/highwayhash v1.0.2 github.com/minio/kes-go v0.1.0 github.com/minio/madmin-go/v2 v2.1.3 - github.com/minio/minio-go/v7 v7.0.52 + github.com/minio/minio-go/v7 v7.0.54 github.com/minio/mux v1.9.0 - github.com/minio/pkg v1.7.1 + github.com/minio/pkg v1.7.2 github.com/minio/selfupdate v0.6.0 - github.com/minio/sha256-simd v1.0.0 + github.com/minio/sha256-simd v1.0.1 github.com/minio/simdjson-go v0.4.5 github.com/minio/sio v0.3.1 github.com/minio/xxml v0.0.3 @@ -87,9 +87,9 @@ require ( go.uber.org/atomic v1.10.0 go.uber.org/zap v1.24.0 goftp.io/server/v2 v2.0.0 - golang.org/x/crypto v0.8.0 + golang.org/x/crypto v0.9.0 golang.org/x/oauth2 v0.7.0 - golang.org/x/sys v0.7.0 + golang.org/x/sys v0.8.0 golang.org/x/time v0.3.0 google.golang.org/api v0.117.0 gopkg.in/yaml.v2 v2.4.0 @@ -209,7 +209,7 @@ require ( github.com/rjeczalik/notify v0.9.3 // indirect github.com/rs/xid v1.5.0 // indirect github.com/shoenig/go-m1cpu v0.1.5 // indirect - github.com/sirupsen/logrus v1.9.0 // indirect + github.com/sirupsen/logrus v1.9.2 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect @@ -222,9 +222,9 @@ require ( go.opencensus.io v0.24.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.10.0 // indirect - golang.org/x/net v0.9.0 // indirect + golang.org/x/net v0.10.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/term v0.7.0 // indirect + golang.org/x/term v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/tools v0.8.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect diff --git a/go.sum b/go.sum index 2689f2d51..d922f0e07 100644 --- a/go.sum +++ b/go.sum @@ -790,18 +790,19 @@ github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= github.com/minio/minio-go/v7 v7.0.41/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw= -github.com/minio/minio-go/v7 v7.0.52 h1:8XhG36F6oKQUDDSuz6dY3rioMzovKjW40W6ANuN0Dps= -github.com/minio/minio-go/v7 v7.0.52/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU= +github.com/minio/minio-go/v7 v7.0.54 h1:1tS2v8nhylHEn307qUwXBNioCDHLDcgOMTA4Te4wFVc= +github.com/minio/minio-go/v7 v7.0.54/go.mod h1:NUDy4A4oXPq1l2yK6LTSvCEzAMeIcoz9lcj5dbzSrRE= github.com/minio/mux v1.9.0 h1:dWafQFyEfGhJvK6AwLOt83bIG5bxKxKJnKMCi0XAaoA= github.com/minio/mux v1.9.0/go.mod h1:1pAare17ZRL5GpmNL+9YmqHoWnLmMZF9C/ioUCfy0BQ= github.com/minio/pkg v1.5.4/go.mod h1:2MOaRFdmFKULD+uOLc3qHLGTQTuxCNPKNPfLBTxC8CA= -github.com/minio/pkg v1.7.1 h1:Nu5EJ64PHmJtbq1zOxQv2lNM6drjbhO8QOCdntHSjVE= -github.com/minio/pkg v1.7.1/go.mod h1:0iX1IuJGSCnMvIvrEJauk1GgQSX9JdU6Kh0P3EQRGkI= +github.com/minio/pkg v1.7.2 h1:MdRCuuZIo6e1LdLXiOiH22hIBpJijlBLLtye4J4Zsjk= +github.com/minio/pkg v1.7.2/go.mod h1:0iX1IuJGSCnMvIvrEJauk1GgQSX9JdU6Kh0P3EQRGkI= github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU= github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/minio/simdjson-go v0.4.5 h1:r4IQwjRGmWCQ2VeMc7fGiilu1z5du0gJ/I/FsKwgo5A= github.com/minio/simdjson-go v0.4.5/go.mod h1:eoNz0DcLQRyEDeaPr4Ru6JpjlZPzbA0IodxVJk8lO8E= github.com/minio/sio v0.3.1 h1:d59r5RTHb1OsQaSl1EaTWurzMMDRLA5fgNmjzD4eVu4= @@ -1004,8 +1005,9 @@ github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= +github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= @@ -1163,8 +1165,8 @@ golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20221012134737-56aed061732a/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= -golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1265,8 +1267,8 @@ golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1416,16 +1418,16 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/internal/hash/reader.go b/internal/hash/reader.go index 1dff95a62..649f3782c 100644 --- a/internal/hash/reader.go +++ b/internal/hash/reader.go @@ -184,6 +184,21 @@ func (r *Reader) AddChecksum(req *http.Request, ignoreValue bool) error { return r.AddNonTrailingChecksum(cs, ignoreValue) } +// AddChecksumNoTrailer will add checksum checks as specified in +// https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html +// Returns ErrInvalidChecksum if a problem with the checksum is found. +func (r *Reader) AddChecksumNoTrailer(headers http.Header, ignoreValue bool) error { + cs, err := GetContentChecksum(headers) + if err != nil { + return ErrInvalidChecksum + } + if cs == nil { + return nil + } + r.contentHash = *cs + return r.AddNonTrailingChecksum(cs, ignoreValue) +} + // AddNonTrailingChecksum will add a checksum to the reader. // The checksum cannot be trailing. func (r *Reader) AddNonTrailingChecksum(cs *Checksum, ignoreValue bool) error {