mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
support fan-out objects via PostUpload() (#17233)
This commit is contained in:
Harshavardhana
GitHub
parent
66156b8230
commit
d0a0eb9738
@ -21,6 +21,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
@ -30,6 +31,7 @@ import (
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"path"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@ -37,8 +39,10 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/minio/mux"
|
||||
"github.com/valyala/bytebufferpool"
|
||||
|
||||
"github.com/minio/madmin-go/v2"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
@ -51,6 +55,7 @@ import (
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
@ -889,14 +894,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
}
|
||||
|
||||
bucket := mux.Vars(r)["bucket"]
|
||||
|
||||
// Require Content-Length to be set in the request
|
||||
size := r.ContentLength
|
||||
if size < 0 {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
resource, err := getResource(r.URL.Path, r.Host, globalDomainNames)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
||||
@ -911,7 +908,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
|
||||
// Here the parameter is the size of the form data that should
|
||||
// be loaded in memory, the remaining being put in temporary files.
|
||||
reader, err := r.MultipartReader()
|
||||
mp, err := r.MultipartReader()
|
||||
if err != nil {
|
||||
apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
|
||||
apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err)
|
||||
@ -919,16 +916,20 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
|
||||
const mapEntryOverhead = 200
|
||||
|
||||
var (
|
||||
fileBody io.ReadCloser
|
||||
fileName string
|
||||
reader io.Reader
|
||||
fileSize int64 = -1
|
||||
fileName string
|
||||
fanOutEntries = make([]minio.PutObjectFanOutEntry, 0, 100)
|
||||
)
|
||||
|
||||
maxParts := 1000
|
||||
// Canonicalize the form values into http.Header.
|
||||
formValues := make(http.Header)
|
||||
for {
|
||||
part, err := reader.NextRawPart()
|
||||
part, err := mp.NextRawPart()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
@ -956,7 +957,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
// Multiple values for the same key (one map entry, longer slice) are cheaper
|
||||
// than the same number of values for different keys (many map entries), but
|
||||
// using a consistent per-value cost for overhead is simpler.
|
||||
const mapEntryOverhead = 200
|
||||
maxMemoryBytes := 2 * int64(10<<20)
|
||||
maxMemoryBytes -= int64(len(name))
|
||||
maxMemoryBytes -= mapEntryOverhead
|
||||
@ -971,9 +971,29 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
|
||||
var b bytes.Buffer
|
||||
if fileName == "" {
|
||||
if http.CanonicalHeaderKey(name) == http.CanonicalHeaderKey("x-minio-fanout-list") {
|
||||
dec := json.NewDecoder(part)
|
||||
|
||||
// while the array contains values
|
||||
for dec.More() {
|
||||
var m minio.PutObjectFanOutEntry
|
||||
if err := dec.Decode(&m); err != nil {
|
||||
part.Close()
|
||||
apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
|
||||
apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, multipart.ErrMessageTooLarge)
|
||||
writeErrorResponse(ctx, w, apiErr, r.URL)
|
||||
return
|
||||
}
|
||||
fanOutEntries = append(fanOutEntries, m)
|
||||
}
|
||||
part.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
// value, store as string in memory
|
||||
n, err := io.CopyN(&b, part, maxMemoryBytes+1)
|
||||
part.Close()
|
||||
|
||||
if err != nil && err != io.EOF {
|
||||
apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
|
||||
apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err)
|
||||
@ -1001,8 +1021,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
// The file or text content.
|
||||
// The file or text content must be the last field in the form.
|
||||
// You cannot upload more than one file at a time.
|
||||
fileBody = part // we have found the File part of the request breakout
|
||||
defer part.Close()
|
||||
reader = part
|
||||
// we have found the File part of the request we are done processing multipart-form
|
||||
break
|
||||
}
|
||||
|
||||
@ -1012,6 +1032,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
writeErrorResponse(ctx, w, apiErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if fileName == "" {
|
||||
apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
|
||||
apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, errors.New("The file or text content is missing"))
|
||||
@ -1059,20 +1080,38 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
|
||||
// Once signature is validated, check if the user has
|
||||
// explicit permissions for the user.
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.PutObjectAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
BucketName: bucket,
|
||||
ObjectName: object,
|
||||
IsOwner: globalActiveCred.AccessKey == cred.AccessKey,
|
||||
Claims: cred.Claims,
|
||||
}) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
if len(fanOutEntries) > 0 {
|
||||
// Once signature is validated, check if the user has
|
||||
// explicit permissions for the user.
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.PutObjectFanOutAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
BucketName: bucket,
|
||||
ObjectName: object,
|
||||
IsOwner: globalActiveCred.AccessKey == cred.AccessKey,
|
||||
Claims: cred.Claims,
|
||||
}) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
// Once signature is validated, check if the user has
|
||||
// explicit permissions for the user.
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.PutObjectAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
BucketName: bucket,
|
||||
ObjectName: object,
|
||||
IsOwner: globalActiveCred.AccessKey == cred.AccessKey,
|
||||
Claims: cred.Claims,
|
||||
}) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
policyBytes, err := base64.StdEncoding.DecodeString(formValues.Get("Policy"))
|
||||
@ -1081,15 +1120,14 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
|
||||
hashReader, err := hash.NewReader(fileBody, -1, "", "", -1)
|
||||
hashReader, err := hash.NewReader(reader, fileSize, "", "", fileSize)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if checksum != nil && checksum.Valid() {
|
||||
err = hashReader.AddChecksum(r, false)
|
||||
if err != nil {
|
||||
if err = hashReader.AddChecksumNoTrailer(formValues, false); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
@ -1134,7 +1172,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
|
||||
// Check if bucket encryption is enabled
|
||||
sseConfig, _ := globalBucketSSEConfigSys.Get(bucket)
|
||||
sseConfig.Apply(r.Header, sse.ApplyOptions{
|
||||
sseConfig.Apply(formValues, sse.ApplyOptions{
|
||||
AutoEncrypt: globalAutoEncryption,
|
||||
})
|
||||
|
||||
@ -1145,6 +1183,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
|
||||
fanOutOpts := fanOutOptions{Checksum: checksum}
|
||||
|
||||
if crypto.Requested(formValues) {
|
||||
if crypto.SSECopy.IsRequested(r.Header) {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL)
|
||||
@ -1177,29 +1217,137 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
}
|
||||
reader, objectEncryptionKey, err = newEncryptReader(ctx, hashReader, kind, keyID, key, bucket, object, metadata, kmsCtx)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
// do not try to verify encrypted content/
|
||||
hashReader, err = hash.NewReader(reader, -1, "", "", -1)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if checksum != nil && checksum.Valid() {
|
||||
err = hashReader.AddChecksum(r, true)
|
||||
|
||||
if len(fanOutEntries) == 0 {
|
||||
reader, objectEncryptionKey, err = newEncryptReader(ctx, hashReader, kind, keyID, key, bucket, object, metadata, kmsCtx)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
// do not try to verify encrypted content/
|
||||
hashReader, err = hash.NewReader(reader, -1, "", "", -1)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if checksum != nil && checksum.Valid() {
|
||||
if err = hashReader.AddChecksumNoTrailer(formValues, true); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
fanOutOpts = fanOutOptions{
|
||||
Key: key,
|
||||
Kind: kind,
|
||||
KeyID: keyID,
|
||||
KmsCtx: kmsCtx,
|
||||
Checksum: checksum,
|
||||
}
|
||||
}
|
||||
pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
|
||||
}
|
||||
|
||||
if len(fanOutEntries) > 0 {
|
||||
// Fan-out requires no copying, and must be carried from original source
|
||||
// https://en.wikipedia.org/wiki/Copy_protection so the incoming stream
|
||||
// is always going to be in-memory as we cannot re-read from what we
|
||||
// wrote to disk - since that amounts to "copying" from a "copy"
|
||||
// instead of "copying" from source, we need the stream to be seekable
|
||||
// to ensure that we can make fan-out calls concurrently.
|
||||
buf := bytebufferpool.Get()
|
||||
defer bytebufferpool.Put(buf)
|
||||
|
||||
// Maximum allowed fan-out object size.
|
||||
const maxFanOutSize = 16 << 20
|
||||
|
||||
n, err := io.Copy(buf, ioutil.HardLimitReader(pReader, maxFanOutSize))
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
concurrentSize := 100
|
||||
if runtime.GOMAXPROCS(0) < concurrentSize {
|
||||
concurrentSize = runtime.GOMAXPROCS(0)
|
||||
}
|
||||
|
||||
fanOutResp := make([]minio.PutObjectFanOutResponse, 0, len(fanOutEntries))
|
||||
eventArgsList := make([]eventArgs, 0, len(fanOutEntries))
|
||||
for {
|
||||
var objInfos []ObjectInfo
|
||||
var errs []error
|
||||
|
||||
var done bool
|
||||
if len(fanOutEntries) < concurrentSize {
|
||||
objInfos, errs = fanOutPutObject(ctx, bucket, objectAPI, fanOutEntries, buf.Bytes()[:n], fanOutOpts)
|
||||
done = true
|
||||
} else {
|
||||
objInfos, errs = fanOutPutObject(ctx, bucket, objectAPI, fanOutEntries[:concurrentSize], buf.Bytes()[:n], fanOutOpts)
|
||||
fanOutEntries = fanOutEntries[concurrentSize:]
|
||||
}
|
||||
|
||||
for i, objInfo := range objInfos {
|
||||
if errs[i] != nil {
|
||||
fanOutResp = append(fanOutResp, minio.PutObjectFanOutResponse{
|
||||
Key: objInfo.Name,
|
||||
Error: errs[i],
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
fanOutResp = append(fanOutResp, minio.PutObjectFanOutResponse{
|
||||
Key: objInfo.Name,
|
||||
ETag: getDecryptedETag(formValues, objInfo, false),
|
||||
VersionID: objInfo.VersionID,
|
||||
LastModified: &objInfo.ModTime,
|
||||
})
|
||||
|
||||
eventArgsList = append(eventArgsList, eventArgs{
|
||||
EventName: event.ObjectCreatedPost,
|
||||
BucketName: objInfo.Bucket,
|
||||
Object: objInfo,
|
||||
ReqParams: extractReqParams(r),
|
||||
RespElements: extractRespElements(w),
|
||||
UserAgent: r.UserAgent() + " " + "MinIO-Fan-Out",
|
||||
Host: handlers.GetSourceIP(r),
|
||||
})
|
||||
}
|
||||
|
||||
if done {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
enc := json.NewEncoder(w)
|
||||
for i, fanOutResp := range fanOutResp {
|
||||
if err = enc.Encode(&fanOutResp); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Notify object created events.
|
||||
sendEvent(eventArgsList[i])
|
||||
|
||||
if eventArgsList[i].Object.NumVersions > dataScannerExcessiveVersionsThreshold {
|
||||
// Send events for excessive versions.
|
||||
sendEvent(eventArgs{
|
||||
EventName: event.ObjectManyVersions,
|
||||
BucketName: eventArgsList[i].Object.Bucket,
|
||||
Object: eventArgsList[i].Object,
|
||||
ReqParams: extractReqParams(r),
|
||||
RespElements: extractRespElements(w),
|
||||
UserAgent: r.UserAgent() + " " + "MinIO-Fan-Out",
|
||||
Host: handlers.GetSourceIP(r),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
objInfo, err := objectAPI.PutObject(ctx, bucket, object, pReader, opts)
|
||||
@ -1232,6 +1380,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
UserAgent: r.UserAgent(),
|
||||
Host: handlers.GetSourceIP(r),
|
||||
})
|
||||
|
||||
if objInfo.NumVersions > dataScannerExcessiveVersionsThreshold {
|
||||
defer sendEvent(eventArgs{
|
||||
EventName: event.ObjectManyVersions,
|
||||
|
||||
114
cmd/post-policy-fan-out.go
Normal file
114
cmd/post-policy-fan-out.go
Normal file
@ -0,0 +1,114 @@
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/s3utils"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
)
|
||||
|
||||
type fanOutOptions struct {
|
||||
Kind crypto.Type
|
||||
KeyID string
|
||||
Key []byte
|
||||
KmsCtx kms.Context
|
||||
Checksum *hash.Checksum
|
||||
}
|
||||
|
||||
// fanOutPutObject takes an input source reader and fans out multiple PUT operations
|
||||
// based on the incoming fan-out request, a context cancelation by the caller
|
||||
// would ensure all fan-out operations are canceled.
|
||||
func fanOutPutObject(ctx context.Context, bucket string, objectAPI ObjectLayer, fanOutEntries []minio.PutObjectFanOutEntry, fanOutBuf []byte, opts fanOutOptions) ([]ObjectInfo, []error) {
|
||||
errs := make([]error, len(fanOutEntries))
|
||||
objInfos := make([]ObjectInfo, len(fanOutEntries))
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i, req := range fanOutEntries {
|
||||
wg.Add(1)
|
||||
go func(idx int, req minio.PutObjectFanOutEntry) {
|
||||
defer wg.Done()
|
||||
|
||||
objInfos[idx] = ObjectInfo{Name: req.Key}
|
||||
|
||||
hr, err := hash.NewReader(bytes.NewReader(fanOutBuf), int64(len(fanOutBuf)), "", "", -1)
|
||||
if err != nil {
|
||||
errs[idx] = err
|
||||
return
|
||||
}
|
||||
|
||||
reader := NewPutObjReader(hr)
|
||||
defer func() {
|
||||
if err := reader.Close(); err != nil {
|
||||
errs[idx] = err
|
||||
}
|
||||
if err := hr.Close(); err != nil {
|
||||
errs[idx] = err
|
||||
}
|
||||
}()
|
||||
|
||||
userDefined := make(map[string]string, len(req.UserMetadata))
|
||||
for k, v := range req.UserMetadata {
|
||||
userDefined[k] = v
|
||||
}
|
||||
userDefined[xhttp.AmzObjectTagging] = s3utils.TagEncode(req.UserTags)
|
||||
|
||||
if opts.Kind != nil {
|
||||
encrd, objectEncryptionKey, err := newEncryptReader(ctx, hr, opts.Kind, opts.KeyID, opts.Key, bucket, req.Key, userDefined, opts.KmsCtx)
|
||||
if err != nil {
|
||||
errs[idx] = err
|
||||
return
|
||||
}
|
||||
|
||||
// do not try to verify encrypted content/
|
||||
hr, err = hash.NewReader(encrd, -1, "", "", -1)
|
||||
if err != nil {
|
||||
errs[idx] = err
|
||||
return
|
||||
}
|
||||
|
||||
reader, err = reader.WithEncryption(hr, &objectEncryptionKey)
|
||||
if err != nil {
|
||||
errs[idx] = err
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
objInfo, err := objectAPI.PutObject(ctx, bucket, req.Key, reader, ObjectOptions{
|
||||
Versioned: globalBucketVersioningSys.PrefixEnabled(bucket, req.Key),
|
||||
VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, req.Key),
|
||||
UserDefined: userDefined,
|
||||
})
|
||||
if err != nil {
|
||||
errs[idx] = err
|
||||
return
|
||||
}
|
||||
objInfos[idx] = objInfo
|
||||
}(i, req)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
return objInfos, errs
|
||||
}
|
||||
18
docs/extensions/fan-out/README.md
Normal file
18
docs/extensions/fan-out/README.md
Normal file
@ -0,0 +1,18 @@
|
||||
# Fan-Out Uploads [](https://slack.min.io) [](https://hub.docker.com/r/minio/minio/)
|
||||
|
||||
## Overview
|
||||
|
||||
MinIO implements an S3 extension to perform multiple concurrent fan-out upload operations. A perfect use case scenario for performing fan-out operations of incoming TSB (Time Shift Buffer's). TSBs are a method of facilitating time-shifted playback of television signaling, and media content.
|
||||
|
||||
MinIO implements an S3 extension to the [PostUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPOST.html) where in a special fan-out list is sent along with the TSB's for MinIO make multiple uploads from a single source stream. Optionally supports custom metadata, tags and other retention settings. All objects are also readable independently once upload is completed via the regular S3 [GetObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html) API.
|
||||
|
||||
## How to enable Fan-Out Uploads ?
|
||||
|
||||
Fan-Out uploads are automatically enabled if `x-minio-fanout-list` form-field is provided with the PostUpload API, to keep things simple higher level APIs are provided in our SDKs for example in `minio-go` SDK:
|
||||
|
||||
```
|
||||
PutObjectFanOut(ctx context.Context, bucket string, fanOutContent io.Reader, fanOutReq minio.PutObjectFanOutRequest) ([]minio.PutObjectFanOutResponse, error)
|
||||
```
|
||||
|
||||
|
||||
|
||||
16
go.mod
16
go.mod
@ -49,11 +49,11 @@ require (
|
||||
github.com/minio/highwayhash v1.0.2
|
||||
github.com/minio/kes-go v0.1.0
|
||||
github.com/minio/madmin-go/v2 v2.1.3
|
||||
github.com/minio/minio-go/v7 v7.0.52
|
||||
github.com/minio/minio-go/v7 v7.0.54
|
||||
github.com/minio/mux v1.9.0
|
||||
github.com/minio/pkg v1.7.1
|
||||
github.com/minio/pkg v1.7.2
|
||||
github.com/minio/selfupdate v0.6.0
|
||||
github.com/minio/sha256-simd v1.0.0
|
||||
github.com/minio/sha256-simd v1.0.1
|
||||
github.com/minio/simdjson-go v0.4.5
|
||||
github.com/minio/sio v0.3.1
|
||||
github.com/minio/xxml v0.0.3
|
||||
@ -87,9 +87,9 @@ require (
|
||||
go.uber.org/atomic v1.10.0
|
||||
go.uber.org/zap v1.24.0
|
||||
goftp.io/server/v2 v2.0.0
|
||||
golang.org/x/crypto v0.8.0
|
||||
golang.org/x/crypto v0.9.0
|
||||
golang.org/x/oauth2 v0.7.0
|
||||
golang.org/x/sys v0.7.0
|
||||
golang.org/x/sys v0.8.0
|
||||
golang.org/x/time v0.3.0
|
||||
google.golang.org/api v0.117.0
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
@ -209,7 +209,7 @@ require (
|
||||
github.com/rjeczalik/notify v0.9.3 // indirect
|
||||
github.com/rs/xid v1.5.0 // indirect
|
||||
github.com/shoenig/go-m1cpu v0.1.5 // indirect
|
||||
github.com/sirupsen/logrus v1.9.0 // indirect
|
||||
github.com/sirupsen/logrus v1.9.2 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.1 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.11 // indirect
|
||||
@ -222,9 +222,9 @@ require (
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/mod v0.10.0 // indirect
|
||||
golang.org/x/net v0.9.0 // indirect
|
||||
golang.org/x/net v0.10.0 // indirect
|
||||
golang.org/x/sync v0.1.0 // indirect
|
||||
golang.org/x/term v0.7.0 // indirect
|
||||
golang.org/x/term v0.8.0 // indirect
|
||||
golang.org/x/text v0.9.0 // indirect
|
||||
golang.org/x/tools v0.8.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
|
||||
30
go.sum
30
go.sum
@ -790,18 +790,19 @@ github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
|
||||
github.com/minio/minio-go/v7 v7.0.41/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw=
|
||||
github.com/minio/minio-go/v7 v7.0.52 h1:8XhG36F6oKQUDDSuz6dY3rioMzovKjW40W6ANuN0Dps=
|
||||
github.com/minio/minio-go/v7 v7.0.52/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU=
|
||||
github.com/minio/minio-go/v7 v7.0.54 h1:1tS2v8nhylHEn307qUwXBNioCDHLDcgOMTA4Te4wFVc=
|
||||
github.com/minio/minio-go/v7 v7.0.54/go.mod h1:NUDy4A4oXPq1l2yK6LTSvCEzAMeIcoz9lcj5dbzSrRE=
|
||||
github.com/minio/mux v1.9.0 h1:dWafQFyEfGhJvK6AwLOt83bIG5bxKxKJnKMCi0XAaoA=
|
||||
github.com/minio/mux v1.9.0/go.mod h1:1pAare17ZRL5GpmNL+9YmqHoWnLmMZF9C/ioUCfy0BQ=
|
||||
github.com/minio/pkg v1.5.4/go.mod h1:2MOaRFdmFKULD+uOLc3qHLGTQTuxCNPKNPfLBTxC8CA=
|
||||
github.com/minio/pkg v1.7.1 h1:Nu5EJ64PHmJtbq1zOxQv2lNM6drjbhO8QOCdntHSjVE=
|
||||
github.com/minio/pkg v1.7.1/go.mod h1:0iX1IuJGSCnMvIvrEJauk1GgQSX9JdU6Kh0P3EQRGkI=
|
||||
github.com/minio/pkg v1.7.2 h1:MdRCuuZIo6e1LdLXiOiH22hIBpJijlBLLtye4J4Zsjk=
|
||||
github.com/minio/pkg v1.7.2/go.mod h1:0iX1IuJGSCnMvIvrEJauk1GgQSX9JdU6Kh0P3EQRGkI=
|
||||
github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU=
|
||||
github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM=
|
||||
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
|
||||
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
|
||||
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
|
||||
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
|
||||
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
|
||||
github.com/minio/simdjson-go v0.4.5 h1:r4IQwjRGmWCQ2VeMc7fGiilu1z5du0gJ/I/FsKwgo5A=
|
||||
github.com/minio/simdjson-go v0.4.5/go.mod h1:eoNz0DcLQRyEDeaPr4Ru6JpjlZPzbA0IodxVJk8lO8E=
|
||||
github.com/minio/sio v0.3.1 h1:d59r5RTHb1OsQaSl1EaTWurzMMDRLA5fgNmjzD4eVu4=
|
||||
@ -1004,8 +1005,9 @@ github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
|
||||
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
|
||||
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
|
||||
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y=
|
||||
github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
@ -1163,8 +1165,8 @@ golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0
|
||||
golang.org/x/crypto v0.0.0-20221012134737-56aed061732a/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
|
||||
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
|
||||
golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ=
|
||||
golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE=
|
||||
golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
|
||||
golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
@ -1265,8 +1267,8 @@ golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
|
||||
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@ -1416,16 +1418,16 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
|
||||
golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=
|
||||
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
|
||||
golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
||||
@ -184,6 +184,21 @@ func (r *Reader) AddChecksum(req *http.Request, ignoreValue bool) error {
|
||||
return r.AddNonTrailingChecksum(cs, ignoreValue)
|
||||
}
|
||||
|
||||
// AddChecksumNoTrailer will add checksum checks as specified in
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
|
||||
// Returns ErrInvalidChecksum if a problem with the checksum is found.
|
||||
func (r *Reader) AddChecksumNoTrailer(headers http.Header, ignoreValue bool) error {
|
||||
cs, err := GetContentChecksum(headers)
|
||||
if err != nil {
|
||||
return ErrInvalidChecksum
|
||||
}
|
||||
if cs == nil {
|
||||
return nil
|
||||
}
|
||||
r.contentHash = *cs
|
||||
return r.AddNonTrailingChecksum(cs, ignoreValue)
|
||||
}
|
||||
|
||||
// AddNonTrailingChecksum will add a checksum to the reader.
|
||||
// The checksum cannot be trailing.
|
||||
func (r *Reader) AddNonTrailingChecksum(cs *Checksum, ignoreValue bool) error {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user