Vendorize new changes from minio-go repo (#5821)

- When policy is empty delete the bucket policy (#966) (04/13/18) <Harshavardhana>
- Add tests to check if ListObjects/V2 returns expected StorageClass (#963) (04/10/18) <Nitish Tiwari>
- Update get/setBucketPolicy methods to use files instead of pkg/policy (#959) (04/10/18) <Nitish Tiwari>
- avoid unnecessary stat call during single copy (#962) (04/06/18) <Andreas Auernhammer>
- avoid sending SSE-S3 header during GET requests. (#965) (04/05/18) <Andreas Auernhammer>
- Fix stream SSE uploads with S3 encrypt type (#960) (04/02/18) <Jesús Espino>
- Fix xml parsing error for RemoveObjects API (#949) (03/29/18) <poornas>
- Allow to upload empty files in stream based uploads (#958) (03/26/18) <Jesús Espino>
- Add missing doneCh in the example for removeobjects (#955) (03/26/18) <Alexandr Korsak>
- tests: Remove partial related tests (#957) (03/26/18) <Anis Elleuch>
- Add transport connection broken error to retry list (#956) (03/19/18) <poornas>
- [refactor]: simplify client encryption examples (#952) (03/19/18) <Andreas Auernhammer>
- Add tests for putObjectContentLanguage (#950) (03/15/18) <Harshavardhana>
- Add putObject/getObject() client side encryption examples (#948) (03/13/18) <Harshavardhana>
This commit is contained in:
Harshavardhana 2018-04-14 23:09:48 -07:00 committed by Nitish Tiwari
parent 638f01f9e4
commit 97a8d856b6
16 changed files with 158 additions and 155 deletions

View File

@ -18,6 +18,7 @@ package s3
import (
"context"
"encoding/json"
"io"
"github.com/minio/cli"
@ -428,7 +429,11 @@ func (l *s3Objects) CompleteMultipartUpload(ctx context.Context, bucket string,
// SetBucketPolicy sets policy on bucket
func (l *s3Objects) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error {
if err := l.Client.PutBucketPolicy(bucket, policyInfo); err != nil {
data, err := json.Marshal(&policyInfo)
if err != nil {
return err
}
if err := l.Client.SetBucketPolicy(bucket, string(data)); err != nil {
logger.LogIf(ctx, err)
return minio.ErrorRespToObjectError(err, bucket, "")
}
@ -438,17 +443,21 @@ func (l *s3Objects) SetBucketPolicy(ctx context.Context, bucket string, policyIn
// GetBucketPolicy will get policy on bucket
func (l *s3Objects) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) {
policyInfo, err := l.Client.GetBucketPolicy(bucket)
data, err := l.Client.GetBucketPolicy(bucket)
if err != nil {
logger.LogIf(ctx, err)
return policy.BucketAccessPolicy{}, minio.ErrorRespToObjectError(err, bucket, "")
}
var policyInfo policy.BucketAccessPolicy
if err = json.Unmarshal([]byte(data), &policyInfo); err != nil {
return policyInfo, err
}
return policyInfo, nil
}
// DeleteBucketPolicy deletes all policies on bucket
func (l *s3Objects) DeleteBucketPolicy(ctx context.Context, bucket string) error {
if err := l.Client.PutBucketPolicy(bucket, policy.BucketAccessPolicy{}); err != nil {
if err := l.Client.SetBucketPolicy(bucket, ""); err != nil {
logger.LogIf(ctx, err)
return minio.ErrorRespToObjectError(err, bucket, "")
}

View File

@ -130,7 +130,6 @@ The full API Reference is available here.
### API Reference : Bucket policy Operations
* [`SetBucketPolicy`](https://docs.minio.io/docs/golang-client-api-reference#SetBucketPolicy)
* [`GetBucketPolicy`](https://docs.minio.io/docs/golang-client-api-reference#GetBucketPolicy)
* [`ListBucketPolicies`](https://docs.minio.io/docs/golang-client-api-reference#ListBucketPolicies)
### API Reference : Bucket notification Operations
* [`SetBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#SetBucketNotification)

View File

@ -141,7 +141,6 @@ mc ls play/mymusic/
### API文档 : 存储桶策略
* [`SetBucketPolicy`](https://docs.minio.io/docs/golang-client-api-reference#SetBucketPolicy)
* [`GetBucketPolicy`](https://docs.minio.io/docs/golang-client-api-reference#GetBucketPolicy)
* [`ListBucketPolicies`](https://docs.minio.io/docs/golang-client-api-reference#ListBucketPolicies)
### API文档 : 存储桶通知
* [`SetBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#SetBucketNotification)

View File

@ -132,12 +132,6 @@ func NewSourceInfo(bucket, object string, sse encrypt.ServerSide) SourceInfo {
// Set the source header
r.Headers.Set("x-amz-copy-source", s3utils.EncodePath(bucket+"/"+object))
// Assemble decryption headers for upload-part-copy request
if r.encryption != nil {
encrypt.SSECopy(r.encryption).Marshal(r.Headers)
}
return r
}
@ -197,7 +191,7 @@ func (s *SourceInfo) getProps(c Client) (size int64, etag string, userMeta map[s
// Get object info - need size and etag here. Also, decryption
// headers are added to the stat request if given.
var objInfo ObjectInfo
opts := StatObjectOptions{GetObjectOptions{ServerSideEncryption: s.encryption}}
opts := StatObjectOptions{GetObjectOptions{ServerSideEncryption: encrypt.SSE(s.encryption)}}
objInfo, err = c.statObject(context.Background(), s.bucket, s.object, opts)
if err != nil {
err = ErrInvalidArgument(fmt.Sprintf("Could not stat object - %s/%s: %v", s.bucket, s.object, err))
@ -427,37 +421,7 @@ func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error {
// involved, it is being copied wholly and at most 5GiB in
// size, emptyfiles are also supported).
if (totalParts == 1 && srcs[0].start == -1 && totalSize <= maxPartSize) || (totalSize == 0) {
h := srcs[0].Headers
// Add destination encryption headers
if dst.encryption != nil {
dst.encryption.Marshal(h)
}
// If no user metadata is specified (and so, the
// for-loop below is not entered), metadata from the
// source is copied to the destination (due to
// single-part copy-object PUT request behaviour).
for k, v := range dst.getUserMetaHeadersMap(true) {
h.Set(k, v)
}
// Send copy request
resp, err := c.executeMethod(ctx, "PUT", requestMetadata{
bucketName: dst.bucket,
objectName: dst.object,
customHeader: h,
})
defer closeResponse(resp)
if err != nil {
return err
}
// Check if we got an error response.
if resp.StatusCode != http.StatusOK {
return httpRespToErrorResponse(resp, dst.bucket, dst.object)
}
// Return nil on success.
return nil
return c.CopyObject(dst, srcs[0])
}
// Now, handle multipart-copy cases.
@ -487,6 +451,9 @@ func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error {
partIndex := 1
for i, src := range srcs {
h := src.Headers
if src.encryption != nil {
src.encryption.Marshal(h)
}
// Add destination encryption headers
if dst.encryption != nil {
dst.encryption.Marshal(h)

View File

@ -44,7 +44,7 @@ func (o GetObjectOptions) Header() http.Header {
for k, v := range o.headers {
headers.Set(k, v)
}
if o.ServerSideEncryption != nil {
if o.ServerSideEncryption != nil && o.ServerSideEncryption.Type() != encrypt.S3 {
o.ServerSideEncryption.Marshal(headers)
}
return headers

View File

@ -19,62 +19,32 @@ package minio
import (
"context"
"encoding/json"
"io/ioutil"
"net/http"
"net/url"
"github.com/minio/minio-go/pkg/policy"
"github.com/minio/minio-go/pkg/s3utils"
)
// GetBucketPolicy - get bucket policy at a given path.
func (c Client) GetBucketPolicy(bucketName, objectPrefix string) (bucketPolicy policy.BucketPolicy, err error) {
func (c Client) GetBucketPolicy(bucketName string) (string, error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return policy.BucketPolicyNone, err
return "", err
}
if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil {
return policy.BucketPolicyNone, err
}
policyInfo, err := c.getBucketPolicy(bucketName)
bucketPolicy, err := c.getBucketPolicy(bucketName)
if err != nil {
errResponse := ToErrorResponse(err)
if errResponse.Code == "NoSuchBucketPolicy" {
return policy.BucketPolicyNone, nil
return "", nil
}
return policy.BucketPolicyNone, err
return "", err
}
return policy.GetPolicy(policyInfo.Statements, bucketName, objectPrefix), nil
}
// ListBucketPolicies - list all policies for a given prefix and all its children.
func (c Client) ListBucketPolicies(bucketName, objectPrefix string) (bucketPolicies map[string]policy.BucketPolicy, err error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return map[string]policy.BucketPolicy{}, err
}
if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil {
return map[string]policy.BucketPolicy{}, err
}
policyInfo, err := c.getBucketPolicy(bucketName)
if err != nil {
errResponse := ToErrorResponse(err)
if errResponse.Code == "NoSuchBucketPolicy" {
return map[string]policy.BucketPolicy{}, nil
}
return map[string]policy.BucketPolicy{}, err
}
return policy.GetPolicies(policyInfo.Statements, bucketName, objectPrefix), nil
}
// Default empty bucket access policy.
var emptyBucketAccessPolicy = policy.BucketAccessPolicy{
Version: "2012-10-17",
return bucketPolicy, nil
}
// Request server for current bucket policy.
func (c Client) getBucketPolicy(bucketName string) (policy.BucketAccessPolicy, error) {
func (c Client) getBucketPolicy(bucketName string) (string, error) {
// Get resources properly escaped and lined up before
// using them in http request.
urlValues := make(url.Values)
@ -89,21 +59,20 @@ func (c Client) getBucketPolicy(bucketName string) (policy.BucketAccessPolicy, e
defer closeResponse(resp)
if err != nil {
return emptyBucketAccessPolicy, err
return "", err
}
if resp != nil {
if resp.StatusCode != http.StatusOK {
return emptyBucketAccessPolicy, httpRespToErrorResponse(resp, bucketName, "")
return "", httpRespToErrorResponse(resp, bucketName, "")
}
}
bucketPolicyBuf, err := ioutil.ReadAll(resp.Body)
if err != nil {
return emptyBucketAccessPolicy, err
return "", err
}
policy := policy.BucketAccessPolicy{}
err = json.Unmarshal(bucketPolicyBuf, &policy)
policy := string(bucketPolicyBuf)
return policy, err
}

View File

@ -20,13 +20,12 @@ package minio
import (
"bytes"
"context"
"encoding/json"
"encoding/xml"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
"github.com/minio/minio-go/pkg/policy"
"github.com/minio/minio-go/pkg/s3utils"
)
@ -100,74 +99,45 @@ func (c Client) MakeBucket(bucketName string, location string) (err error) {
}
// SetBucketPolicy set the access permissions on an existing bucket.
//
// For example
//
// none - owner gets full access [default].
// readonly - anonymous get access for everyone at a given object prefix.
// readwrite - anonymous list/put/delete access to a given object prefix.
// writeonly - anonymous put/delete access to a given object prefix.
func (c Client) SetBucketPolicy(bucketName string, objectPrefix string, bucketPolicy policy.BucketPolicy) error {
func (c Client) SetBucketPolicy(bucketName, policy string) error {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return err
}
if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil {
return err
}
if !bucketPolicy.IsValidBucketPolicy() {
return ErrInvalidArgument(fmt.Sprintf("Invalid bucket policy provided. %s", bucketPolicy))
// If policy is empty then delete the bucket policy.
if policy == "" {
return c.removeBucketPolicy(bucketName)
}
policyInfo, err := c.getBucketPolicy(bucketName)
errResponse := ToErrorResponse(err)
if err != nil && errResponse.Code != "NoSuchBucketPolicy" {
return err
}
if bucketPolicy == policy.BucketPolicyNone && policyInfo.Statements == nil {
// As the request is for removing policy and the bucket
// has empty policy statements, just return success.
return nil
}
policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, bucketPolicy, bucketName, objectPrefix)
// Save the updated policies.
return c.putBucketPolicy(bucketName, policyInfo)
return c.putBucketPolicy(bucketName, policy)
}
// Saves a new bucket policy.
func (c Client) putBucketPolicy(bucketName string, policyInfo policy.BucketAccessPolicy) error {
func (c Client) putBucketPolicy(bucketName, policy string) error {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return err
}
// If there are no policy statements, we should remove entire policy.
if len(policyInfo.Statements) == 0 {
return c.removeBucketPolicy(bucketName)
}
// Get resources properly escaped and lined up before
// using them in http request.
urlValues := make(url.Values)
urlValues.Set("policy", "")
policyBytes, err := json.Marshal(&policyInfo)
// Content-length is mandatory for put policy request
policyReader := strings.NewReader(policy)
b, err := ioutil.ReadAll(policyReader)
if err != nil {
return err
}
policyBuffer := bytes.NewReader(policyBytes)
reqMetadata := requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
contentBody: policyBuffer,
contentLength: int64(len(policyBytes)),
contentMD5Base64: sumMD5Base64(policyBytes),
contentSHA256Hex: sum256Hex(policyBytes),
contentBody: policyReader,
contentLength: int64(len(b)),
}
// Execute PUT to upload a new bucket policy.

View File

@ -17,7 +17,41 @@
package minio
import (
"context"
"net/http"
"github.com/minio/minio-go/pkg/encrypt"
)
// CopyObject - copy a source object into a new object
func (c Client) CopyObject(dst DestinationInfo, src SourceInfo) error {
return c.ComposeObject(dst, []SourceInfo{src})
header := make(http.Header)
for k, v := range src.Headers {
header[k] = v
}
if src.encryption != nil {
encrypt.SSECopy(src.encryption).Marshal(header)
}
if dst.encryption != nil {
dst.encryption.Marshal(header)
}
for k, v := range dst.getUserMetaHeadersMap(true) {
header.Set(k, v)
}
resp, err := c.executeMethod(context.Background(), "PUT", requestMetadata{
bucketName: dst.bucket,
objectName: dst.object,
customHeader: header,
})
if err != nil {
return err
}
defer closeResponse(resp)
if resp.StatusCode != http.StatusOK {
return httpRespToErrorResponse(resp, dst.bucket, dst.object)
}
return nil
}

View File

@ -259,7 +259,7 @@ func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID
// Set encryption headers, if any.
customHeader := make(http.Header)
if sse != nil {
if sse != nil && sse.Type() != encrypt.S3 && sse.Type() != encrypt.KMS {
sse.Marshal(customHeader)
}

View File

@ -208,7 +208,7 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName
if rErr == io.EOF && partNumber > 1 {
break
}
if rErr != nil && rErr != io.ErrUnexpectedEOF {
if rErr != nil && rErr != io.ErrUnexpectedEOF && rErr != io.EOF {
return 0, rErr
}
// Update progress reader appropriately to the latest offset

View File

@ -195,6 +195,12 @@ func (c Client) RemoveObjectsWithContext(ctx context.Context, bucketName string,
contentMD5Base64: sumMD5Base64(removeBytes),
contentSHA256Hex: sum256Hex(removeBytes),
})
if resp != nil {
if resp.StatusCode != http.StatusOK {
e := httpRespToErrorResponse(resp, bucketName, "")
errorCh <- RemoveObjectError{ObjectName: "", Err: e}
}
}
if err != nil {
for _, b := range batch {
errorCh <- RemoveObjectError{ObjectName: b, Err: err}

View File

@ -99,7 +99,7 @@ type Options struct {
// Global constants.
const (
libraryName = "minio-go"
libraryVersion = "5.0.1"
libraryVersion = "6.0.1"
)
// User Agent should always following the below style.

View File

@ -21,8 +21,6 @@ import (
"context"
"io"
"strings"
"github.com/minio/minio-go/pkg/policy"
)
// Core - Inherits Client and adds new methods to expose the low level S3 APIs.
@ -127,12 +125,12 @@ func (c Core) AbortMultipartUpload(bucket, object, uploadID string) error {
}
// GetBucketPolicy - fetches bucket access policy for a given bucket.
func (c Core) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, error) {
func (c Core) GetBucketPolicy(bucket string) (string, error) {
return c.getBucketPolicy(bucket)
}
// PutBucketPolicy - applies a new bucket access policy for a given bucket.
func (c Core) PutBucketPolicy(bucket string, bucketPolicy policy.BucketAccessPolicy) error {
func (c Core) PutBucketPolicy(bucket, bucketPolicy string) error {
return c.putBucketPolicy(bucket, bucketPolicy)
}

View File

@ -20,6 +20,7 @@ package encrypt
import (
"crypto/md5"
"encoding/base64"
"encoding/json"
"errors"
"net/http"
@ -30,6 +31,11 @@ const (
// sseGenericHeader is the AWS SSE header used for SSE-S3 and SSE-KMS.
sseGenericHeader = "X-Amz-Server-Side-Encryption"
// sseKmsKeyID is the AWS SSE-KMS key id.
sseKmsKeyID = sseGenericHeader + "-Aws-Kms-Key-Id"
// sseEncryptionContext is the AWS SSE-KMS Encryption Context data.
sseEncryptionContext = sseGenericHeader + "-Encryption-Context"
// sseCustomerAlgorithm is the AWS SSE-C algorithm HTTP header key.
sseCustomerAlgorithm = sseGenericHeader + "-Customer-Algorithm"
// sseCustomerKey is the AWS SSE-C encryption key HTTP header key.
@ -90,6 +96,18 @@ type ServerSide interface {
// Using SSE-S3 the server will encrypt the object with server-managed keys.
func NewSSE() ServerSide { return s3{} }
// NewSSEKMS returns a new server-side-encryption using SSE-KMS and the provided Key Id and context.
func NewSSEKMS(keyID string, context interface{}) (ServerSide, error) {
if context == nil {
return kms{key: keyID, hasContext: false}, nil
}
serializedContext, err := json.Marshal(context)
if err != nil {
return nil, err
}
return kms{key: keyID, context: serializedContext, hasContext: true}, nil
}
// NewSSEC returns a new server-side-encryption using SSE-C and the provided key.
// The key must be 32 bytes long.
func NewSSEC(key []byte) (ServerSide, error) {
@ -101,6 +119,21 @@ func NewSSEC(key []byte) (ServerSide, error) {
return sse, nil
}
// SSE transforms a SSE-C copy encryption into a SSE-C encryption.
// It is the inverse of SSECopy(...).
//
// If the provided sse is no SSE-C copy encryption SSE returns
// sse unmodified.
func SSE(sse ServerSide) ServerSide {
if sse == nil || sse.Type() != SSEC {
return sse
}
if sse, ok := sse.(ssecCopy); ok {
return ssec(sse)
}
return sse
}
// SSECopy transforms a SSE-C encryption into a SSE-C copy
// encryption. This is required for SSE-C key rotation or a SSE-C
// copy where the source and the destination should be encrypted.
@ -144,3 +177,19 @@ type s3 struct{}
func (s s3) Type() Type { return S3 }
func (s s3) Marshal(h http.Header) { h.Set(sseGenericHeader, "AES256") }
type kms struct {
key string
context []byte
hasContext bool
}
func (s kms) Type() Type { return KMS }
func (s kms) Marshal(h http.Header) {
h.Set(sseGenericHeader, "aws:kms")
h.Set(sseKmsKeyID, s.key)
if s.hasContext {
h.Set(sseEncryptionContext, base64.StdEncoding.EncodeToString(s.context))
}
}

View File

@ -111,6 +111,9 @@ func isNetErrorRetryable(err error) bool {
} else if strings.Contains(err.Error(), "connection timed out") {
// If err is a net.Dial timeout, retry.
return true
} else if strings.Contains(err.Error(), "net/http: HTTP/1.x transport connection broken") {
// If error is transport connection broken, retry.
return true
}
}
}

32
vendor/vendor.json vendored
View File

@ -404,46 +404,46 @@
"revisionTime": "2016-02-29T08:42:30-08:00"
},
{
"checksumSHA1": "XSYD1N4v3EtbiwJQpIO71ZaucP0=",
"checksumSHA1": "w7jIUKw0cR7Z34JykcQXGJEPEmI=",
"path": "github.com/minio/minio-go",
"revision": "bfac4536aea242ea4d05fb552d2a85f397a85369",
"revisionTime": "2018-03-12T23:14:54Z"
"revision": "a2238f0e60facba1c2ccff86ddc624e23fae4f23",
"revisionTime": "2018-04-13T18:58:16Z"
},
{
"checksumSHA1": "Qsj+6JPmJ8R5rFNQSHqRb8xAwOw=",
"path": "github.com/minio/minio-go/pkg/credentials",
"revision": "bfac4536aea242ea4d05fb552d2a85f397a85369",
"revisionTime": "2018-03-12T23:14:54Z"
"revision": "a2238f0e60facba1c2ccff86ddc624e23fae4f23",
"revisionTime": "2018-04-13T18:58:16Z"
},
{
"checksumSHA1": "p21oRW905lv0UBfpAm5Vs6EBmB8=",
"checksumSHA1": "Md5pOKYfoKtrG7xNvs2FtiDPfDc=",
"path": "github.com/minio/minio-go/pkg/encrypt",
"revision": "bfac4536aea242ea4d05fb552d2a85f397a85369",
"revisionTime": "2018-03-12T23:14:54Z"
"revision": "a2238f0e60facba1c2ccff86ddc624e23fae4f23",
"revisionTime": "2018-04-13T18:58:16Z"
},
{
"checksumSHA1": "Df4SG4ratsLn8yE9SkHcZiKhKRU=",
"path": "github.com/minio/minio-go/pkg/policy",
"revision": "bfac4536aea242ea4d05fb552d2a85f397a85369",
"revisionTime": "2018-03-12T23:14:54Z"
"revision": "a2238f0e60facba1c2ccff86ddc624e23fae4f23",
"revisionTime": "2018-04-13T18:58:16Z"
},
{
"checksumSHA1": "bbWjcrOQsV57qK+BSsrNAsI+Q/o=",
"path": "github.com/minio/minio-go/pkg/s3signer",
"revision": "bfac4536aea242ea4d05fb552d2a85f397a85369",
"revisionTime": "2018-03-12T23:14:54Z"
"revision": "a2238f0e60facba1c2ccff86ddc624e23fae4f23",
"revisionTime": "2018-04-13T18:58:16Z"
},
{
"checksumSHA1": "xrJThFwwkVrJdwd5iYFHqfx4wRY=",
"path": "github.com/minio/minio-go/pkg/s3utils",
"revision": "bfac4536aea242ea4d05fb552d2a85f397a85369",
"revisionTime": "2018-03-12T23:14:54Z"
"revision": "a2238f0e60facba1c2ccff86ddc624e23fae4f23",
"revisionTime": "2018-04-13T18:58:16Z"
},
{
"checksumSHA1": "Wt8ej+rZXTdNBR9Xyw1eGo3Iq5o=",
"path": "github.com/minio/minio-go/pkg/set",
"revision": "bfac4536aea242ea4d05fb552d2a85f397a85369",
"revisionTime": "2018-03-12T23:14:54Z"
"revision": "a2238f0e60facba1c2ccff86ddc624e23fae4f23",
"revisionTime": "2018-04-13T18:58:16Z"
},
{
"checksumSHA1": "cYuXpiVBMypgkEr0Wqd79jPPyBg=",