MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-24 06:05:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

Simplify PutObjReader for plain-text reader usage (#11470)

Browse Source 
This change moves away from a unified constructor for plaintext and encrypted
usage. NewPutObjReader is simplified for the plain-text reader use. For
encrypted reader use, WithEncryption should be called on an initialized PutObjReader.

Plaintext:
func NewPutObjReader(rawReader *hash.Reader) *PutObjReader

The hash.Reader is used to provide payload size and md5sum to the downstream
consumers. This is different from the previous version in that there is no need
to pass nil values for unused parameters.

Encrypted:
func WithEncryption(encReader *hash.Reader,
key *crypto.ObjectKey) (*PutObjReader, error)

This method sets up encrypted reader along with the key to seal the md5sum
produced by the plain-text reader (already setup when NewPutObjReader was
called).

Usage:
```
  pReader := NewPutObjReader(rawReader)
  // ... other object handler code goes here

  // Prepare the encrypted hashed reader
  pReader, err = pReader.WithEncryption(encReader, objEncKey)

```
This commit is contained in:
Krishnan Parthasarathi 2021-02-10 08:52:50 -08:00 committed by GitHub
parent b8b44c879f
commit b87fae0049
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 66 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/bucket-handlers.go
View File

@ -919,7 +919,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
return
}
rawReader := hashReader
pReader := NewPutObjReader(rawReader, nil, nil)
pReader := NewPutObjReader(rawReader)
var objectEncryptionKey crypto.ObjectKey
// Check if bucket encryption is enabled
@ -964,7 +964,11 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
pReader = NewPutObjReader(rawReader, hashReader, &objectEncryptionKey)
pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
}
}

2
cmd/bucket-lifecycle.go
View File

@ -696,7 +696,7 @@ func restoreTransitionedObject(ctx context.Context, bucket, object string, objAP
if err != nil {
return err
}
pReader := NewPutObjReader(hashReader, nil, nil)
pReader := NewPutObjReader(hashReader)
opts := putRestoreOpts(bucket, object, rreq, objInfo)
opts.UserDefined[xhttp.AmzRestore] = fmt.Sprintf("ongoing-request=%t, expiry-date=%s", false, restoreExpiry.Format(http.TimeFormat))
if _, err := objAPI.PutObject(ctx, bucket, object, pReader, opts); err != nil {

2
cmd/config-common.go
View File

@ -69,7 +69,7 @@ func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data
return err
}
_, err = objAPI.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader, nil, nil), ObjectOptions{})
_, err = objAPI.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader), ObjectOptions{})
return err
}

2
cmd/data-crawler.go
View File

@ -134,7 +134,7 @@ func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
continue
}
_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageBloomName, NewPutObjReader(r, nil, nil), ObjectOptions{})
_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageBloomName, NewPutObjReader(r), ObjectOptions{})
if !isErrBucketNotFound(err) {
logger.LogIf(ctx, err)
}

2
cmd/data-usage-cache.go
View File

@ -521,7 +521,7 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
_, err = store.PutObject(ctx,
dataUsageBucket,
name,
NewPutObjReader(r, nil, nil),
NewPutObjReader(r),
ObjectOptions{})
if isErrBucketNotFound(err) {
return nil

2
cmd/data-usage.go
View File

@ -52,7 +52,7 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan
logger.LogIf(ctx, err)
continue
}
_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageObjName, NewPutObjReader(r, nil, nil), ObjectOptions{})
_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageObjName, NewPutObjReader(r), ObjectOptions{})
if !isErrBucketNotFound(err) {
logger.LogIf(ctx, err)
}

2
cmd/disk-cache.go
View File

@ -711,7 +711,7 @@ func (c *cacheObjects) uploadObject(ctx context.Context, oi ObjectInfo) {
var opts ObjectOptions
opts.UserDefined = make(map[string]string)
opts.UserDefined[xhttp.ContentMD5] = oi.UserDefined["content-md5"]
objInfo, err := c.InnerPutObjectFn(ctx, oi.Bucket, oi.Name, NewPutObjReader(hashReader, nil, nil), opts)
objInfo, err := c.InnerPutObjectFn(ctx, oi.Bucket, oi.Name, NewPutObjReader(hashReader), opts)
wbCommitStatus := CommitComplete
if err != nil {
wbCommitStatus = CommitFailed

2
cmd/erasure-multipart.go
View File

@ -325,7 +325,7 @@ func (er erasureObjects) NewMultipartUpload(ctx context.Context, bucket, object
//
// Implements S3 compatible Upload Part Copy API.
func (er erasureObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (pi PartInfo, e error) {
partInfo, err := er.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, NewPutObjReader(srcInfo.Reader, nil, nil), dstOpts)
partInfo, err := er.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, NewPutObjReader(srcInfo.Reader), dstOpts)
if err != nil {
return pi, toObjectErr(err, dstBucket, dstObject)
}

2
cmd/erasure-server-pool.go
View File

@ -875,7 +875,7 @@ func (z *erasureServerPools) CopyObjectPart(ctx context.Context, srcBucket, srcO
}
return z.PutObjectPart(ctx, destBucket, destObject, uploadID, partID,
NewPutObjReader(srcInfo.Reader, nil, nil), dstOpts)
NewPutObjReader(srcInfo.Reader), dstOpts)
}
// PutObjectPart - writes part of an object to hashedSet based on the object name.

2
cmd/erasure-sets.go
View File

@ -1076,7 +1076,7 @@ func (s *erasureSets) CopyObjectPart(ctx context.Context, srcBucket, srcObject,
startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (partInfo PartInfo, err error) {
destSet := s.getHashedSet(destObject)
auditObjectErasureSet(ctx, destObject, destSet, s.poolNumber)
return destSet.PutObjectPart(ctx, destBucket, destObject, uploadID, partID, NewPutObjReader(srcInfo.Reader, nil, nil), dstOpts)
return destSet.PutObjectPart(ctx, destBucket, destObject, uploadID, partID, NewPutObjReader(srcInfo.Reader), dstOpts)
}
// PutObjectPart - writes part of an object to hashedSet based on the object name.

2
cmd/gateway/s3/gateway-s3-metadata.go
View File

@ -172,5 +172,5 @@ func getGWMetadata(ctx context.Context, bucket, prefix string, gwMeta gwMetaV1)
if err != nil {
return nil, err
}
return minio.NewPutObjReader(hashReader, nil, nil), nil
return minio.NewPutObjReader(hashReader), nil
}

2
cmd/metacache-bucket.go
View File

@ -180,7 +180,7 @@ func (b *bucketMetacache) save(ctx context.Context) error {
if err != nil {
return err
}
_, err = objAPI.PutObject(ctx, minioMetaBucket, pathJoin("buckets", b.bucket, ".metacache", "index.s2"), NewPutObjReader(hr, nil, nil), ObjectOptions{})
_, err = objAPI.PutObject(ctx, minioMetaBucket, pathJoin("buckets", b.bucket, ".metacache", "index.s2"), NewPutObjReader(hr), ObjectOptions{})
logger.LogIf(ctx, err)
return err
}

2
cmd/metacache-set.go
View File

@ -675,7 +675,7 @@ func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions) (entr
r, err := hash.NewReader(bytes.NewReader(b.data), int64(len(b.data)), "", "", int64(len(b.data)), false)
logger.LogIf(ctx, err)
custom := b.headerKV()
_, err = er.putObject(ctx, minioMetaBucket, o.objectPath(b.n), NewPutObjReader(r, nil, nil), ObjectOptions{
_, err = er.putObject(ctx, minioMetaBucket, o.objectPath(b.n), NewPutObjReader(r), ObjectOptions{
UserDefined: custom,
NoLock: true, // No need to hold namespace lock, each prefix caches uniquely.
ParentIsObject: nil,

27
cmd/object-api-utils.go
View File

@ -20,6 +20,7 @@ import (
"bytes"
"context"
"encoding/hex"
"errors"
"fmt"
"io"
"math/rand"
@ -865,17 +866,23 @@ func (p *PutObjReader) MD5CurrentHexString() string {
return hex.EncodeToString(md5sumCurr)
}
// NewPutObjReader returns a new PutObjReader and holds
// reference to underlying data stream from client and the encrypted
// data reader
func NewPutObjReader(rawReader *hash.Reader, encReader *hash.Reader, key *crypto.ObjectKey) *PutObjReader {
p := PutObjReader{Reader: rawReader, rawReader: rawReader}
if key != nil && encReader != nil {
p.sealMD5Fn = sealETagFn(*key)
p.Reader = encReader
// WithEncryption sets up encrypted reader and the sealing for content md5sum
// using objEncKey. Unsealed md5sum is computed from the rawReader setup when
// NewPutObjReader was called. It returns an error if called on an uninitialized
// PutObjReader.
func (p *PutObjReader) WithEncryption(encReader *hash.Reader, objEncKey *crypto.ObjectKey) (*PutObjReader, error) {
if p.Reader == nil {
return nil, errors.New("put-object reader uninitialized")
}
return &p
p.Reader = encReader
p.sealMD5Fn = sealETagFn(*objEncKey)
return p, nil
}
// NewPutObjReader returns a new PutObjReader. It uses given hash.Reader's
// MD5Current method to construct md5sum when requested downstream.
func NewPutObjReader(rawReader *hash.Reader) *PutObjReader {
return &PutObjReader{Reader: rawReader, rawReader: rawReader}
}
func sealETag(encKey crypto.ObjectKey, md5CurrSum []byte) []byte {

33
cmd/object-handlers.go
View File

@ -1032,8 +1032,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
return
}
rawReader := srcInfo.Reader
pReader := NewPutObjReader(srcInfo.Reader, nil, nil)
pReader := NewPutObjReader(srcInfo.Reader)
// Check if either the source is encrypted or the destination will be encrypted.
_, objectEncryption := crypto.IsRequested(r.Header)
@ -1145,7 +1144,11 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
}
if isTargetEncrypted {
pReader = NewPutObjReader(rawReader, srcInfo.Reader, &objEncKey)
pReader, err = pReader.WithEncryption(srcInfo.Reader, &objEncKey)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
}
}
}
@ -1495,7 +1498,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
}
rawReader := hashReader
pReader := NewPutObjReader(rawReader, nil, nil)
pReader := NewPutObjReader(rawReader)
// get gateway encryption options
var opts ObjectOptions
@ -1564,7 +1567,11 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
pReader = NewPutObjReader(rawReader, hashReader, &objectEncryptionKey)
pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
}
}
@ -2002,7 +2009,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
}
rawReader := srcInfo.Reader
pReader := NewPutObjReader(rawReader, nil, nil)
pReader := NewPutObjReader(rawReader)
_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
var objectEncryptionKey crypto.ObjectKey
@ -2048,7 +2055,11 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
pReader = NewPutObjReader(rawReader, srcInfo.Reader, &objectEncryptionKey)
pReader, err = pReader.WithEncryption(srcInfo.Reader, &objectEncryptionKey)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
}
srcInfo.PutObjReader = pReader
@ -2242,7 +2253,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
return
}
rawReader := hashReader
pReader := NewPutObjReader(rawReader, nil, nil)
pReader := NewPutObjReader(rawReader)
_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
var objectEncryptionKey crypto.ObjectKey
@ -2298,7 +2309,11 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
pReader = NewPutObjReader(rawReader, hashReader, &objectEncryptionKey)
pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
}
putObjectPart := objectAPI.PutObjectPart

2
cmd/test-utils_test.go
View File

@ -164,7 +164,7 @@ func mustGetPutObjReader(t TestErrHandler, data io.Reader, size int64, md5hex, s
if err != nil {
t.Fatal(err)
}
return NewPutObjReader(hr, nil, nil)
return NewPutObjReader(hr)
}
// calculateSignedChunkLength - calculates the length of the overall stream (data + metadata)

9
cmd/web-handlers.go
View File

@ -1234,7 +1234,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
if mustReplicate {
metadata[xhttp.AmzBucketReplicationStatus] = string(replication.Pending)
}
pReader = NewPutObjReader(hashReader, nil, nil)
pReader = NewPutObjReader(hashReader)
// get gateway encryption options
opts, err := putOpts(ctx, r, bucket, object, metadata)
if err != nil {
@ -1244,7 +1244,6 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
if objectAPI.IsEncryptionSupported() {
if _, ok := crypto.IsRequested(r.Header); ok && !HasSuffix(object, SlashSeparator) { // handle SSE requests
rawReader := hashReader
var objectEncryptionKey crypto.ObjectKey
reader, objectEncryptionKey, err = EncryptRequest(hashReader, r, bucket, object, metadata)
if err != nil {
@ -1258,7 +1257,11 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
pReader = NewPutObjReader(rawReader, hashReader, &objectEncryptionKey)
pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
}
}