Update vendorized minio-go to support start-after param (#6043)

Fixes #6032

vendor/github.com/minio/minio-go/api-compose-object.go

@@ -1,6 +1,6 @@
 /*
  * Minio Go Library for Amazon S3 Compatible Cloud Storage
- * Copyright 2017 Minio, Inc.
+ * Copyright 2017, 2018 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,6 +20,8 @@ package minio
 import (
 	"context"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -343,11 +345,12 @@ func (c Client) uploadPartCopy(ctx context.Context, bucket, object, uploadID str
 	return p, nil
 }
 
-// ComposeObject - creates an object using server-side copying of
+// ComposeObjectWithProgress - creates an object using server-side copying of
 // existing objects. It takes a list of source objects (with optional
 // offsets) and concatenates them into a new object using only
-// server-side copying operations.
-func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error {
+// server-side copying operations. Optionally takes progress reader hook
+// for applications to look at current progress.
+func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo, progress io.Reader) error {
 	if len(srcs) < 1 || len(srcs) > maxPartsCount {
 		return ErrInvalidArgument("There must be as least one and up to 10000 source objects.")
 	}
@@ -421,7 +424,7 @@ func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error {
 	// involved, it is being copied wholly and at most 5GiB in
 	// size, emptyfiles are also supported).
 	if (totalParts == 1 && srcs[0].start == -1 && totalSize <= maxPartSize) || (totalSize == 0) {
-		return c.CopyObject(dst, srcs[0])
+		return c.CopyObjectWithProgress(dst, srcs[0], progress)
 	}
 
 	// Now, handle multipart-copy cases.
@@ -476,6 +479,9 @@ func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error {
 			if err != nil {
 				return err
 			}
+			if progress != nil {
+				io.CopyN(ioutil.Discard, progress, start+end-1)
+			}
 			objParts = append(objParts, complPart)
 			partIndex++
 		}
@@ -490,10 +496,20 @@ func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error {
 	return nil
 }
 
-// partsRequired is ceiling(size / copyPartSize)
+// ComposeObject - creates an object using server-side copying of
+// existing objects. It takes a list of source objects (with optional
+// offsets) and concatenates them into a new object using only
+// server-side copying operations.
+func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error {
+	return c.ComposeObjectWithProgress(dst, srcs, nil)
+}
+
+// partsRequired is maximum parts possible with
+// max part size of ceiling(maxMultipartPutObjectSize / (maxPartsCount - 1))
 func partsRequired(size int64) int64 {
-	r := size / copyPartSize
-	if size%copyPartSize > 0 {
+	maxPartSize := maxMultipartPutObjectSize / (maxPartsCount - 1)
+	r := size / int64(maxPartSize)
+	if size%int64(maxPartSize) > 0 {
 		r++
 	}
 	return r

vendor/github.com/minio/minio-go/api-list.go

@@ -118,7 +118,7 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d
 		var continuationToken string
 		for {
 			// Get list of objects a maximum of 1000 per request.
-			result, err := c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, 1000)
+			result, err := c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, 1000, "")
 			if err != nil {
 				objectStatCh <- ObjectInfo{
 					Err: err,
@@ -171,11 +171,12 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d
 // You can use the request parameters as selection criteria to return a subset of the objects in a bucket.
 // request parameters :-
 // ---------
-// ?continuation-token - Specifies the key to start with when listing objects in a bucket.
+// ?continuation-token - Used to continue iterating over a set of objects
 // ?delimiter - A delimiter is a character you use to group keys.
 // ?prefix - Limits the response to keys that begin with the specified prefix.
 // ?max-keys - Sets the maximum number of keys returned in the response body.
-func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int) (ListBucketV2Result, error) {
+// ?start-after - Specifies the key to start after when listing objects in a bucket.
+func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) {
 	// Validate bucket name.
 	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
 		return ListBucketV2Result{}, err
@@ -216,6 +217,11 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s
 	// Set max keys.
 	urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys))
 
+	// Set start-after
+	if startAfter != "" {
+		urlValues.Set("start-after", startAfter)
+	}
+
 	// Execute GET on bucket to list objects.
 	resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{
 		bucketName:       bucketName,

vendor/github.com/minio/minio-go/api-put-object-copy.go

@@ -1,6 +1,6 @@
 /*
  * Minio Go Library for Amazon S3 Compatible Cloud Storage
- * Copyright 2017 Minio, Inc.
+ * Copyright 2017, 2018 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,6 +19,8 @@ package minio
 
 import (
 	"context"
+	"io"
+	"io/ioutil"
 	"net/http"
 
 	"github.com/minio/minio-go/pkg/encrypt"
@@ -26,13 +28,31 @@ import (
 
 // CopyObject - copy a source object into a new object
 func (c Client) CopyObject(dst DestinationInfo, src SourceInfo) error {
+	return c.CopyObjectWithProgress(dst, src, nil)
+}
+
+// CopyObjectWithProgress - copy a source object into a new object, optionally takes
+// progress bar input to notify current progress.
+func (c Client) CopyObjectWithProgress(dst DestinationInfo, src SourceInfo, progress io.Reader) error {
 	header := make(http.Header)
 	for k, v := range src.Headers {
 		header[k] = v
 	}
+
+	var err error
+	var size int64
+	// If progress bar is specified, size should be requested as well initiate a StatObject request.
+	if progress != nil {
+		size, _, _, err = src.getProps(c)
+		if err != nil {
+			return err
+		}
+	}
+
 	if src.encryption != nil {
 		encrypt.SSECopy(src.encryption).Marshal(header)
 	}
+
 	if dst.encryption != nil {
 		dst.encryption.Marshal(header)
 	}
@@ -53,5 +73,11 @@ func (c Client) CopyObject(dst DestinationInfo, src SourceInfo) error {
 	if resp.StatusCode != http.StatusOK {
 		return httpRespToErrorResponse(resp, dst.bucket, dst.object)
 	}
+
+	// Update the progress properly after successful copy.
+	if progress != nil {
+		io.CopyN(ioutil.Discard, progress, size)
+	}
+
 	return nil
 }

vendor/github.com/minio/minio-go/api-put-object.go

@@ -28,21 +28,22 @@ import (
 
 	"github.com/minio/minio-go/pkg/encrypt"
 	"github.com/minio/minio-go/pkg/s3utils"
-	"golang.org/x/net/lex/httplex"
+	"golang.org/x/net/http/httpguts"
 )
 
 // PutObjectOptions represents options specified by user for PutObject call
 type PutObjectOptions struct {
-	UserMetadata         map[string]string
-	Progress             io.Reader
-	ContentType          string
-	ContentEncoding      string
-	ContentDisposition   string
-	ContentLanguage      string
-	CacheControl         string
-	ServerSideEncryption encrypt.ServerSide
-	NumThreads           uint
-	StorageClass         string
+	UserMetadata            map[string]string
+	Progress                io.Reader
+	ContentType             string
+	ContentEncoding         string
+	ContentDisposition      string
+	ContentLanguage         string
+	CacheControl            string
+	ServerSideEncryption    encrypt.ServerSide
+	NumThreads              uint
+	StorageClass            string
+	WebsiteRedirectLocation string
 }
 
 // getNumThreads - gets the number of threads to be used in the multipart
@@ -84,6 +85,9 @@ func (opts PutObjectOptions) Header() (header http.Header) {
 	if opts.StorageClass != "" {
 		header[amzStorageClass] = []string{opts.StorageClass}
 	}
+	if opts.WebsiteRedirectLocation != "" {
+		header[amzWebsiteRedirectLocation] = []string{opts.WebsiteRedirectLocation}
+	}
 	for k, v := range opts.UserMetadata {
 		if !isAmzHeader(k) && !isStandardHeader(k) && !isStorageClassHeader(k) {
 			header["X-Amz-Meta-"+k] = []string{v}
@@ -97,10 +101,10 @@ func (opts PutObjectOptions) Header() (header http.Header) {
 // validate() checks if the UserMetadata map has standard headers or and raises an error if so.
 func (opts PutObjectOptions) validate() (err error) {
 	for k, v := range opts.UserMetadata {
-		if !httplex.ValidHeaderFieldName(k) || isStandardHeader(k) || isSSEHeader(k) || isStorageClassHeader(k) {
+		if !httpguts.ValidHeaderFieldName(k) || isStandardHeader(k) || isSSEHeader(k) || isStorageClassHeader(k) {
 			return ErrInvalidArgument(k + " unsupported user defined metadata name")
 		}
-		if !httplex.ValidHeaderFieldValue(v) {
+		if !httpguts.ValidHeaderFieldValue(v) {
 			return ErrInvalidArgument(v + " unsupported user defined metadata value")
 		}
 	}

vendor/github.com/minio/minio-go/api-stat.go

@@ -66,6 +66,9 @@ var defaultFilterKeys = []string{
 	"x-amz-bucket-region",
 	"x-amz-request-id",
 	"x-amz-id-2",
+	"Content-Security-Policy",
+	"X-Xss-Protection",
+
 	// Add new headers to be ignored.
 }
 

vendor/github.com/minio/minio-go/api.go

@@ -99,7 +99,7 @@ type Options struct {
 // Global constants.
 const (
 	libraryName    = "minio-go"
-	libraryVersion = "6.0.1"
+	libraryVersion = "v6.0.3"
 )
 
 // User Agent should always following the below style.

vendor/github.com/minio/minio-go/constants.go

@@ -27,10 +27,6 @@ const absMinPartSize = 1024 * 1024 * 5
 // putObject behaves internally as multipart.
 const minPartSize = 1024 * 1024 * 64
 
-// copyPartSize - default (and maximum) part size to copy in a
-// copy-object request (5GiB)
-const copyPartSize = 1024 * 1024 * 1024 * 5
-
 // maxPartsCount - maximum number of parts for a single multipart session.
 const maxPartsCount = 10000
 
@@ -61,3 +57,6 @@ const (
 
 // Storage class header constant.
 const amzStorageClass = "X-Amz-Storage-Class"
+
+// Website redirect location header constant
+const amzWebsiteRedirectLocation = "X-Amz-Website-Redirect-Location"

vendor/github.com/minio/minio-go/core.go

@@ -48,9 +48,9 @@ func (c Core) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int)
 }
 
 // ListObjectsV2 - Lists all the objects at a prefix, similar to ListObjects() but uses
-// continuationToken instead of marker to further filter the results.
-func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int) (ListBucketV2Result, error) {
-	return c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, maxkeys)
+// continuationToken instead of marker to support iteration over the results.
+func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) {
+	return c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, maxkeys, startAfter)
 }
 
 // CopyObject - copies an object from source object to destination object on server side.
@@ -82,6 +82,8 @@ func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Ba
 			opts.ContentType = v
 		} else if strings.ToLower(k) == "cache-control" {
 			opts.CacheControl = v
+		} else if strings.ToLower(k) == strings.ToLower(amzWebsiteRedirectLocation) {
+			opts.WebsiteRedirectLocation = v
 		} else {
 			m[k] = metadata[k]
 		}

vendor/github.com/minio/minio-go/retry.go

@@ -131,6 +131,7 @@ var retryableS3Codes = map[string]struct{}{
 	"InternalError":         {},
 	"ExpiredToken":          {},
 	"ExpiredTokenException": {},
+	"SlowDown":              {},
 	// Add more AWS S3 codes here.
 }
 

vendor/github.com/minio/minio-go/utils.go

@@ -222,6 +222,7 @@ var supportedHeaders = []string{
 	"content-encoding",
 	"content-disposition",
 	"content-language",
+	"x-amz-website-redirect-location",
 	// Add more supported headers here.
 }