From d85199e9de2b9c913700712086786bc1a943377f Mon Sep 17 00:00:00 2001 From: kannappanr <30541348+kannappanr@users.noreply.github.com> Date: Thu, 29 Nov 2018 11:13:03 -0800 Subject: [PATCH] Vendorize minio-go (#6883) Fixes #6873 --- cmd/gateway/s3/gateway-s3.go | 4 +- vendor/github.com/minio/minio-go/README.md | 17 +- .../minio/minio-go/api-compose-object.go | 8 +- .../minio-go/api-put-object-multipart.go | 6 +- .../github.com/minio/minio-go/api-select.go | 520 ++++++++++++++++++ vendor/github.com/minio/minio-go/api-stat.go | 4 + vendor/github.com/minio/minio-go/api.go | 4 +- vendor/github.com/minio/minio-go/appveyor.yml | 2 +- vendor/github.com/minio/minio-go/core.go | 6 +- .../pkg/credentials/file_minio_client.go | 18 +- .../minio/minio-go/pkg/credentials/iam_aws.go | 54 +- .../minio-go/pkg/policy/bucket-policy.go | 10 +- .../minio/minio-go/pkg/s3utils/utils.go | 31 +- .../github.com/minio/minio-go/post-policy.go | 22 + vendor/github.com/minio/minio-go/s3-error.go | 2 +- vendor/vendor.json | 36 +- 16 files changed, 686 insertions(+), 58 deletions(-) create mode 100644 vendor/github.com/minio/minio-go/api-select.go diff --git a/cmd/gateway/s3/gateway-s3.go b/cmd/gateway/s3/gateway-s3.go index c03971e11..fd6e4f267 100644 --- a/cmd/gateway/s3/gateway-s3.go +++ b/cmd/gateway/s3/gateway-s3.go @@ -510,12 +510,12 @@ func (l *s3Objects) AbortMultipartUpload(ctx context.Context, bucket string, obj // CompleteMultipartUpload completes ongoing multipart upload and finalizes object func (l *s3Objects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []minio.CompletePart, opts minio.ObjectOptions) (oi minio.ObjectInfo, e error) { - err := l.Client.CompleteMultipartUpload(bucket, object, uploadID, minio.ToMinioClientCompleteParts(uploadedParts)) + etag, err := l.Client.CompleteMultipartUpload(bucket, object, uploadID, minio.ToMinioClientCompleteParts(uploadedParts)) if err != nil { return oi, minio.ErrorRespToObjectError(err, bucket, object) } - return l.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{}) + return minio.ObjectInfo{Bucket: bucket, Name: object, ETag: etag}, nil } // SetBucketPolicy sets policy on bucket diff --git a/vendor/github.com/minio/minio-go/README.md b/vendor/github.com/minio/minio-go/README.md index 85ff2cb3d..ad9d5e60b 100644 --- a/vendor/github.com/minio/minio-go/README.md +++ b/vendor/github.com/minio/minio-go/README.md @@ -14,9 +14,9 @@ go get -u github.com/minio/minio-go ## Initialize Minio Client Minio client requires the following four parameters specified to connect to an Amazon S3 compatible object storage. -| Parameter | Description| +| Parameter | Description| | :--- | :--- | -| endpoint | URL to object storage service. | +| endpoint | URL to object storage service. | | accessKeyID | Access key is the user ID that uniquely identifies your account. | | secretAccessKey | Secret key is the password to your account. | | secure | Set this value to 'true' to enable secure (HTTPS) access. | @@ -85,8 +85,9 @@ func main() { } else { log.Fatalln(err) } + } else { + log.Printf("Successfully created %s\n", bucketName) } - log.Printf("Successfully created %s\n", bucketName) // Upload the zip file objectName := "golden-oldies.zip" @@ -106,7 +107,7 @@ func main() { ### Run FileUploader ```sh go run file-uploader.go -2016/08/13 17:03:28 Successfully created mymusic +2016/08/13 17:03:28 Successfully created mymusic 2016/08/13 17:03:40 Successfully uploaded golden-oldies.zip of size 16253413 mc ls play/mymusic/ @@ -114,7 +115,7 @@ mc ls play/mymusic/ ``` ## API Reference -The full API Reference is available here. +The full API Reference is available here. * [Complete API Reference](https://docs.minio.io/docs/golang-client-api-reference) @@ -154,6 +155,8 @@ The full API Reference is available here. * [`RemoveObject`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObject) * [`RemoveObjects`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObjects) * [`RemoveIncompleteUpload`](https://docs.minio.io/docs/golang-client-api-reference#RemoveIncompleteUpload) +* [`SelectObjectContent`](https://docs.minio.io/docs/golang-client-api-reference#SelectObjectContent) + ### API Reference : Presigned Operations * [`PresignedGetObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedGetObject) @@ -182,7 +185,7 @@ The full API Reference is available here. * [setbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketpolicy.go) * [getbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketpolicy.go) * [listbucketpolicies.go](https://github.com/minio/minio-go/blob/master/examples/s3/listbucketpolicies.go) - + ### Full Examples : Bucket lifecycle Operations * [setbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketlifecycle.go) * [getbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketlifecycle.go) @@ -223,7 +226,7 @@ The full API Reference is available here. ## Explore Further * [Complete Documentation](https://docs.minio.io) -* [Minio Go Client SDK API Reference](https://docs.minio.io/docs/golang-client-api-reference) +* [Minio Go Client SDK API Reference](https://docs.minio.io/docs/golang-client-api-reference) * [Go Music Player App Full Application Example](https://docs.minio.io/docs/go-music-player-app) ## Contribute diff --git a/vendor/github.com/minio/minio-go/api-compose-object.go b/vendor/github.com/minio/minio-go/api-compose-object.go index a69122e40..3ac36c502 100644 --- a/vendor/github.com/minio/minio-go/api-compose-object.go +++ b/vendor/github.com/minio/minio-go/api-compose-object.go @@ -362,10 +362,10 @@ func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo srcSizes := make([]int64, len(srcs)) var totalSize, size, totalParts int64 var srcUserMeta map[string]string - var etag string + etags := make([]string, len(srcs)) var err error for i, src := range srcs { - size, etag, srcUserMeta, err = src.getProps(c) + size, etags[i], srcUserMeta, err = src.getProps(c) if err != nil { return err } @@ -426,9 +426,9 @@ func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo // 1. Ensure that the object has not been changed while // we are copying data. - for _, src := range srcs { + for i, src := range srcs { if src.Headers.Get("x-amz-copy-source-if-match") == "" { - src.SetMatchETagCond(etag) + src.SetMatchETagCond(etags[i]) } } diff --git a/vendor/github.com/minio/minio-go/api-put-object-multipart.go b/vendor/github.com/minio/minio-go/api-put-object-multipart.go index 8805ecf96..db92520e8 100644 --- a/vendor/github.com/minio/minio-go/api-put-object-multipart.go +++ b/vendor/github.com/minio/minio-go/api-put-object-multipart.go @@ -259,7 +259,11 @@ func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID // Set encryption headers, if any. customHeader := make(http.Header) - if sse != nil { + // https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html + // Server-side encryption is supported by the S3 Multipart Upload actions. + // Unless you are using a customer-provided encryption key, you don't need + // to specify the encryption parameters in each UploadPart request. + if sse != nil && sse.Type() == encrypt.SSEC { sse.Marshal(customHeader) } diff --git a/vendor/github.com/minio/minio-go/api-select.go b/vendor/github.com/minio/minio-go/api-select.go new file mode 100644 index 000000000..a9b6f17ca --- /dev/null +++ b/vendor/github.com/minio/minio-go/api-select.go @@ -0,0 +1,520 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/xml" + "errors" + "fmt" + "hash" + "hash/crc32" + "io" + "net/http" + "net/url" + "strings" + + "github.com/minio/minio-go/pkg/encrypt" + "github.com/minio/minio-go/pkg/s3utils" +) + +// CSVFileHeaderInfo - is the parameter for whether to utilize headers. +type CSVFileHeaderInfo string + +// Constants for file header info. +const ( + CSVFileHeaderInfoNone CSVFileHeaderInfo = "NONE" + CSVFileHeaderInfoIgnore = "IGNORE" + CSVFileHeaderInfoUse = "USE" +) + +// SelectCompressionType - is the parameter for what type of compression is +// present +type SelectCompressionType string + +// Constants for compression types under select API. +const ( + SelectCompressionNONE SelectCompressionType = "NONE" + SelectCompressionGZIP = "GZIP" + SelectCompressionBZIP = "BZIP2" +) + +// CSVQuoteFields - is the parameter for how CSV fields are quoted. +type CSVQuoteFields string + +// Constants for csv quote styles. +const ( + CSVQuoteFieldsAlways CSVQuoteFields = "Always" + CSVQuoteFieldsAsNeeded = "AsNeeded" +) + +// QueryExpressionType - is of what syntax the expression is, this should only +// be SQL +type QueryExpressionType string + +// Constants for expression type. +const ( + QueryExpressionTypeSQL QueryExpressionType = "SQL" +) + +// JSONType determines json input serialization type. +type JSONType string + +// Constants for JSONTypes. +const ( + JSONDocumentType JSONType = "DOCUMENT" + JSONLinesType = "LINES" +) + +// ParquetInputOptions parquet input specific options +type ParquetInputOptions struct{} + +// CSVInputOptions csv input specific options +type CSVInputOptions struct { + FileHeaderInfo CSVFileHeaderInfo + RecordDelimiter string + FieldDelimiter string + QuoteCharacter string + QuoteEscapeCharacter string + Comments string +} + +// CSVOutputOptions csv output specific options +type CSVOutputOptions struct { + QuoteFields CSVQuoteFields + RecordDelimiter string + FieldDelimiter string + QuoteCharacter string + QuoteEscapeCharacter string +} + +// JSONInputOptions json input specific options +type JSONInputOptions struct { + Type JSONType +} + +// JSONOutputOptions - json output specific options +type JSONOutputOptions struct { + RecordDelimiter string +} + +// SelectObjectInputSerialization - input serialization parameters +type SelectObjectInputSerialization struct { + CompressionType SelectCompressionType + Parquet *ParquetInputOptions `xml:"Parquet,omitempty"` + CSV *CSVInputOptions `xml:"CSV,omitempty"` + JSON *JSONInputOptions `xml:"JSON,omitempty"` +} + +// SelectObjectOutputSerialization - output serialization parameters. +type SelectObjectOutputSerialization struct { + CSV *CSVOutputOptions `xml:"CSV,omitempty"` + JSON *JSONOutputOptions `xml:"JSON,omitempty"` +} + +// SelectObjectOptions - represents the input select body +type SelectObjectOptions struct { + XMLName xml.Name `xml:"SelectObjectContentRequest" json:"-"` + ServerSideEncryption encrypt.ServerSide `xml:"-"` + Expression string + ExpressionType QueryExpressionType + InputSerialization SelectObjectInputSerialization + OutputSerialization SelectObjectOutputSerialization + RequestProgress struct { + Enabled bool + } +} + +// Header returns the http.Header representation of the SelectObject options. +func (o SelectObjectOptions) Header() http.Header { + headers := make(http.Header) + if o.ServerSideEncryption != nil && o.ServerSideEncryption.Type() == encrypt.SSEC { + o.ServerSideEncryption.Marshal(headers) + } + return headers +} + +// SelectObjectType - is the parameter which defines what type of object the +// operation is being performed on. +type SelectObjectType string + +// Constants for input data types. +const ( + SelectObjectTypeCSV SelectObjectType = "CSV" + SelectObjectTypeJSON = "JSON" + SelectObjectTypeParquet = "Parquet" +) + +// preludeInfo is used for keeping track of necessary information from the +// prelude. +type preludeInfo struct { + totalLen uint32 + headerLen uint32 +} + +// SelectResults is used for the streaming responses from the server. +type SelectResults struct { + pipeReader *io.PipeReader + resp *http.Response + stats *StatsMessage + progress *ProgressMessage +} + +// ProgressMessage is a struct for progress xml message. +type ProgressMessage struct { + XMLName xml.Name `xml:"Progress" json:"-"` + StatsMessage +} + +// StatsMessage is a struct for stat xml message. +type StatsMessage struct { + XMLName xml.Name `xml:"Stats" json:"-"` + BytesScanned int64 + BytesProcessed int64 + BytesReturned int64 +} + +// eventType represents the type of event. +type eventType string + +// list of event-types returned by Select API. +const ( + endEvent eventType = "End" + errorEvent = "Error" + recordsEvent = "Records" + progressEvent = "Progress" + statsEvent = "Stats" +) + +// contentType represents content type of event. +type contentType string + +const ( + xmlContent contentType = "text/xml" +) + +// SelectObjectContent is a implementation of http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html AWS S3 API. +func (c Client) SelectObjectContent(ctx context.Context, bucketName, objectName string, opts SelectObjectOptions) (*SelectResults, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return nil, err + } + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return nil, err + } + + selectReqBytes, err := xml.Marshal(opts) + if err != nil { + return nil, err + } + + urlValues := make(url.Values) + urlValues.Set("select", "") + urlValues.Set("select-type", "2") + + // Execute POST on bucket/object. + resp, err := c.executeMethod(ctx, "POST", requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + customHeader: opts.Header(), + contentMD5Base64: sumMD5Base64(selectReqBytes), + contentSHA256Hex: sum256Hex(selectReqBytes), + contentBody: bytes.NewReader(selectReqBytes), + contentLength: int64(len(selectReqBytes)), + }) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, httpRespToErrorResponse(resp, bucketName, "") + } + + pipeReader, pipeWriter := io.Pipe() + streamer := &SelectResults{ + resp: resp, + stats: &StatsMessage{}, + progress: &ProgressMessage{}, + pipeReader: pipeReader, + } + streamer.start(pipeWriter) + return streamer, nil +} + +// Close - closes the underlying response body and the stream reader. +func (s *SelectResults) Close() error { + defer closeResponse(s.resp) + return s.pipeReader.Close() +} + +// Read - is a reader compatible implementation for SelectObjectContent records. +func (s *SelectResults) Read(b []byte) (n int, err error) { + return s.pipeReader.Read(b) +} + +// Stats - information about a request's stats when processing is complete. +func (s *SelectResults) Stats() *StatsMessage { + return s.stats +} + +// Progress - information about the progress of a request. +func (s *SelectResults) Progress() *ProgressMessage { + return s.progress +} + +// start is the main function that decodes the large byte array into +// several events that are sent through the eventstream. +func (s *SelectResults) start(pipeWriter *io.PipeWriter) { + go func() { + for { + var prelude preludeInfo + var headers = make(http.Header) + var err error + + // Create CRC code + crc := crc32.New(crc32.IEEETable) + crcReader := io.TeeReader(s.resp.Body, crc) + + // Extract the prelude(12 bytes) into a struct to extract relevant information. + prelude, err = processPrelude(crcReader, crc) + if err != nil { + pipeWriter.CloseWithError(err) + closeResponse(s.resp) + return + } + + // Extract the headers(variable bytes) into a struct to extract relevant information + if prelude.headerLen > 0 { + if err = extractHeader(io.LimitReader(crcReader, int64(prelude.headerLen)), headers); err != nil { + pipeWriter.CloseWithError(err) + closeResponse(s.resp) + return + } + } + + // Get the actual payload length so that the appropriate amount of + // bytes can be read or parsed. + payloadLen := prelude.PayloadLen() + + // Get content-type of the payload. + c := contentType(headers.Get("content-type")) + + // Get event type of the payload. + e := eventType(headers.Get("event-type")) + + // Handle all supported events. + switch e { + case endEvent: + pipeWriter.Close() + closeResponse(s.resp) + return + case errorEvent: + pipeWriter.CloseWithError(errors.New("Error Type of " + headers.Get("error-type") + " " + headers.Get("error-message"))) + closeResponse(s.resp) + return + case recordsEvent: + if _, err = io.Copy(pipeWriter, io.LimitReader(crcReader, payloadLen)); err != nil { + pipeWriter.CloseWithError(err) + closeResponse(s.resp) + return + } + case progressEvent: + switch c { + case xmlContent: + if err = xmlDecoder(io.LimitReader(crcReader, payloadLen), s.progress); err != nil { + pipeWriter.CloseWithError(err) + closeResponse(s.resp) + return + } + default: + pipeWriter.CloseWithError(fmt.Errorf("Unexpected content-type %s sent for event-type %s", c, progressEvent)) + closeResponse(s.resp) + return + } + case statsEvent: + switch c { + case xmlContent: + if err = xmlDecoder(io.LimitReader(crcReader, payloadLen), s.stats); err != nil { + pipeWriter.CloseWithError(err) + closeResponse(s.resp) + return + } + default: + pipeWriter.CloseWithError(fmt.Errorf("Unexpected content-type %s sent for event-type %s", c, statsEvent)) + closeResponse(s.resp) + return + } + } + + // Ensures that the full message's CRC is correct and + // that the message is not corrupted + if err := checkCRC(s.resp.Body, crc.Sum32()); err != nil { + pipeWriter.CloseWithError(err) + closeResponse(s.resp) + return + } + + } + }() +} + +// PayloadLen is a function that calculates the length of the payload. +func (p preludeInfo) PayloadLen() int64 { + return int64(p.totalLen - p.headerLen - 16) +} + +// processPrelude is the function that reads the 12 bytes of the prelude and +// ensures the CRC is correct while also extracting relevant information into +// the struct, +func processPrelude(prelude io.Reader, crc hash.Hash32) (preludeInfo, error) { + var err error + var pInfo = preludeInfo{} + + // reads total length of the message (first 4 bytes) + pInfo.totalLen, err = extractUint32(prelude) + if err != nil { + return pInfo, err + } + + // reads total header length of the message (2nd 4 bytes) + pInfo.headerLen, err = extractUint32(prelude) + if err != nil { + return pInfo, err + } + + // checks that the CRC is correct (3rd 4 bytes) + preCRC := crc.Sum32() + if err := checkCRC(prelude, preCRC); err != nil { + return pInfo, err + } + + return pInfo, nil +} + +// extracts the relevant information from the Headers. +func extractHeader(body io.Reader, myHeaders http.Header) error { + for { + // extracts the first part of the header, + headerTypeName, err := extractHeaderType(body) + if err != nil { + // Since end of file, we have read all of our headers + if err == io.EOF { + break + } + return err + } + + // reads the 7 present in the header and ignores it. + extractUint8(body) + + headerValueName, err := extractHeaderValue(body) + if err != nil { + return err + } + + myHeaders.Set(headerTypeName, headerValueName) + + } + return nil +} + +// extractHeaderType extracts the first half of the header message, the header type. +func extractHeaderType(body io.Reader) (string, error) { + // extracts 2 bit integer + headerNameLen, err := extractUint8(body) + if err != nil { + return "", err + } + // extracts the string with the appropriate number of bytes + headerName, err := extractString(body, int(headerNameLen)) + if err != nil { + return "", err + } + return strings.TrimPrefix(headerName, ":"), nil +} + +// extractsHeaderValue extracts the second half of the header message, the +// header value +func extractHeaderValue(body io.Reader) (string, error) { + bodyLen, err := extractUint16(body) + if err != nil { + return "", err + } + bodyName, err := extractString(body, int(bodyLen)) + if err != nil { + return "", err + } + return bodyName, nil +} + +// extracts a string from byte array of a particular number of bytes. +func extractString(source io.Reader, lenBytes int) (string, error) { + myVal := make([]byte, lenBytes) + _, err := source.Read(myVal) + if err != nil { + return "", err + } + return string(myVal), nil +} + +// extractUint32 extracts a 4 byte integer from the byte array. +func extractUint32(r io.Reader) (uint32, error) { + buf := make([]byte, 4) + _, err := io.ReadFull(r, buf) + if err != nil { + return 0, err + } + return binary.BigEndian.Uint32(buf), nil +} + +// extractUint16 extracts a 2 byte integer from the byte array. +func extractUint16(r io.Reader) (uint16, error) { + buf := make([]byte, 2) + _, err := io.ReadFull(r, buf) + if err != nil { + return 0, err + } + return binary.BigEndian.Uint16(buf), nil +} + +// extractUint8 extracts a 1 byte integer from the byte array. +func extractUint8(r io.Reader) (uint8, error) { + buf := make([]byte, 1) + _, err := io.ReadFull(r, buf) + if err != nil { + return 0, err + } + return buf[0], nil +} + +// checkCRC ensures that the CRC matches with the one from the reader. +func checkCRC(r io.Reader, expect uint32) error { + msgCRC, err := extractUint32(r) + if err != nil { + return err + } + + if msgCRC != expect { + return fmt.Errorf("Checksum Mismatch, MessageCRC of 0x%X does not equal expected CRC of 0x%X", msgCRC, expect) + + } + return nil +} diff --git a/vendor/github.com/minio/minio-go/api-stat.go b/vendor/github.com/minio/minio-go/api-stat.go index 3b054c34a..91e9d3964 100644 --- a/vendor/github.com/minio/minio-go/api-stat.go +++ b/vendor/github.com/minio/minio-go/api-stat.go @@ -47,6 +47,10 @@ func (c Client) BucketExists(bucketName string) (bool, error) { return false, err } if resp != nil { + resperr := httpRespToErrorResponse(resp, bucketName, "") + if ToErrorResponse(resperr).Code == "NoSuchBucket" { + return false, nil + } if resp.StatusCode != http.StatusOK { return false, httpRespToErrorResponse(resp, bucketName, "") } diff --git a/vendor/github.com/minio/minio-go/api.go b/vendor/github.com/minio/minio-go/api.go index 7c93ca1bb..79b0bc25e 100644 --- a/vendor/github.com/minio/minio-go/api.go +++ b/vendor/github.com/minio/minio-go/api.go @@ -99,7 +99,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.6" + libraryVersion = "v6.0.11" ) // User Agent should always following the below style. @@ -820,7 +820,7 @@ func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, isV host = c.s3AccelerateEndpoint } else { // Do not change the host if the endpoint URL is a FIPS S3 endpoint. - if !s3utils.IsAmazonFIPSGovCloudEndpoint(*c.endpointURL) { + if !s3utils.IsAmazonFIPSEndpoint(*c.endpointURL) { // Fetch new host based on the bucket location. host = getS3Endpoint(bucketLocation) } diff --git a/vendor/github.com/minio/minio-go/appveyor.yml b/vendor/github.com/minio/minio-go/appveyor.yml index aa9f840e5..48ea6e77d 100644 --- a/vendor/github.com/minio/minio-go/appveyor.yml +++ b/vendor/github.com/minio/minio-go/appveyor.yml @@ -16,7 +16,7 @@ install: - set PATH=%GOPATH%\bin;c:\go\bin;%PATH% - go version - go env - - go get -u github.com/golang/lint/golint + - go get -u golang.org/x/lint/golint - go get -u github.com/remyoudompheng/go-misc/deadcode - go get -u github.com/gordonklaus/ineffassign - go get -u golang.org/x/crypto/argon2 diff --git a/vendor/github.com/minio/minio-go/core.go b/vendor/github.com/minio/minio-go/core.go index 0c651d600..4d51363f0 100644 --- a/vendor/github.com/minio/minio-go/core.go +++ b/vendor/github.com/minio/minio-go/core.go @@ -117,11 +117,11 @@ func (c Core) ListObjectParts(bucket, object, uploadID string, partNumberMarker } // CompleteMultipartUpload - Concatenate uploaded parts and commit to an object. -func (c Core) CompleteMultipartUpload(bucket, object, uploadID string, parts []CompletePart) error { - _, err := c.completeMultipartUpload(context.Background(), bucket, object, uploadID, completeMultipartUpload{ +func (c Core) CompleteMultipartUpload(bucket, object, uploadID string, parts []CompletePart) (string, error) { + res, err := c.completeMultipartUpload(context.Background(), bucket, object, uploadID, completeMultipartUpload{ Parts: parts, }) - return err + return res.ETag, err } // AbortMultipartUpload - Abort an incomplete upload. diff --git a/vendor/github.com/minio/minio-go/pkg/credentials/file_minio_client.go b/vendor/github.com/minio/minio-go/pkg/credentials/file_minio_client.go index c282c2a2c..6a6827e37 100644 --- a/vendor/github.com/minio/minio-go/pkg/credentials/file_minio_client.go +++ b/vendor/github.com/minio/minio-go/pkg/credentials/file_minio_client.go @@ -62,13 +62,17 @@ func NewFileMinioClient(filename string, alias string) *Credentials { // users home directory. func (p *FileMinioClient) Retrieve() (Value, error) { if p.filename == "" { - homeDir, err := homedir.Dir() - if err != nil { - return Value{}, err - } - p.filename = filepath.Join(homeDir, ".mc", "config.json") - if runtime.GOOS == "windows" { - p.filename = filepath.Join(homeDir, "mc", "config.json") + if value, ok := os.LookupEnv("MINIO_SHARED_CREDENTIALS_FILE"); ok { + p.filename = value + } else { + homeDir, err := homedir.Dir() + if err != nil { + return Value{}, err + } + p.filename = filepath.Join(homeDir, ".mc", "config.json") + if runtime.GOOS == "windows" { + p.filename = filepath.Join(homeDir, "mc", "config.json") + } } } diff --git a/vendor/github.com/minio/minio-go/pkg/credentials/iam_aws.go b/vendor/github.com/minio/minio-go/pkg/credentials/iam_aws.go index 637df7466..6845c9a29 100644 --- a/vendor/github.com/minio/minio-go/pkg/credentials/iam_aws.go +++ b/vendor/github.com/minio/minio-go/pkg/credentials/iam_aws.go @@ -21,8 +21,10 @@ import ( "bufio" "encoding/json" "errors" + "fmt" "net/http" "net/url" + "os" "path" "time" ) @@ -50,16 +52,25 @@ type IAM struct { // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html const ( defaultIAMRoleEndpoint = "http://169.254.169.254" + defaultECSRoleEndpoint = "http://169.254.170.2" defaultIAMSecurityCredsPath = "/latest/meta-data/iam/security-credentials" ) +// https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html +func getEndpoint(endpoint string) (string, bool) { + if endpoint != "" { + return endpoint, os.Getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") != "" + } + if ecsURI := os.Getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"); ecsURI != "" { + return fmt.Sprintf("%s%s", defaultECSRoleEndpoint, ecsURI), true + } + return defaultIAMRoleEndpoint, false +} + // NewIAM returns a pointer to a new Credentials object wrapping // the IAM. Takes a ConfigProvider to create a EC2Metadata client. // The ConfigProvider is satisfied by the session.Session type. func NewIAM(endpoint string) *Credentials { - if endpoint == "" { - endpoint = defaultIAMRoleEndpoint - } p := &IAM{ Client: &http.Client{ Transport: http.DefaultTransport, @@ -73,11 +84,17 @@ func NewIAM(endpoint string) *Credentials { // Error will be returned if the request fails, or unable to extract // the desired func (m *IAM) Retrieve() (Value, error) { - roleCreds, err := getCredentials(m.Client, m.endpoint) + endpoint, isEcsTask := getEndpoint(m.endpoint) + var roleCreds ec2RoleCredRespBody + var err error + if isEcsTask { + roleCreds, err = getEcsTaskCredentials(m.Client, endpoint) + } else { + roleCreds, err = getCredentials(m.Client, endpoint) + } if err != nil { return Value{}, err } - // Expiry window is set to 10secs. m.SetExpiration(roleCreds.Expiration, DefaultExpiryWindow) @@ -111,9 +128,6 @@ type ec2RoleCredRespBody struct { // be sent to fetch the rolling access credentials. // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html func getIAMRoleURL(endpoint string) (*url.URL, error) { - if endpoint == "" { - endpoint = defaultIAMRoleEndpoint - } u, err := url.Parse(endpoint) if err != nil { return nil, err @@ -153,12 +167,36 @@ func listRoleNames(client *http.Client, u *url.URL) ([]string, error) { return credsList, nil } +func getEcsTaskCredentials(client *http.Client, endpoint string) (ec2RoleCredRespBody, error) { + req, err := http.NewRequest("GET", endpoint, nil) + if err != nil { + return ec2RoleCredRespBody{}, err + } + + resp, err := client.Do(req) + if err != nil { + return ec2RoleCredRespBody{}, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return ec2RoleCredRespBody{}, errors.New(resp.Status) + } + + respCreds := ec2RoleCredRespBody{} + if err := json.NewDecoder(resp.Body).Decode(&respCreds); err != nil { + return ec2RoleCredRespBody{}, err + } + + return respCreds, nil +} + // getCredentials - obtains the credentials from the IAM role name associated with // the current EC2 service. // // If the credentials cannot be found, or there is an error // reading the response an error will be returned. func getCredentials(client *http.Client, endpoint string) (ec2RoleCredRespBody, error) { + // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html u, err := getIAMRoleURL(endpoint) if err != nil { diff --git a/vendor/github.com/minio/minio-go/pkg/policy/bucket-policy.go b/vendor/github.com/minio/minio-go/pkg/policy/bucket-policy.go index 9d5f5b3fa..79fd8019a 100644 --- a/vendor/github.com/minio/minio-go/pkg/policy/bucket-policy.go +++ b/vendor/github.com/minio/minio-go/pkg/policy/bucket-policy.go @@ -557,7 +557,6 @@ func GetPolicy(statements []Statement, bucketName string, prefix string) BucketP } else { matchedObjResources = s.Resources.FuncMatch(resourceMatch, objectResource) } - if !matchedObjResources.IsEmpty() { readOnly, writeOnly := getObjectPolicy(s) for resource := range matchedObjResources { @@ -571,7 +570,8 @@ func GetPolicy(statements []Statement, bucketName string, prefix string) BucketP matchedResource = resource } } - } else if s.Resources.Contains(bucketResource) { + } + if s.Resources.Contains(bucketResource) { commonFound, readOnly, writeOnly := getBucketPolicy(s, prefix) bucketCommonFound = bucketCommonFound || commonFound bucketReadOnly = bucketReadOnly || readOnly @@ -605,6 +605,7 @@ func GetPolicies(statements []Statement, bucketName, prefix string) map[string]B } } } + // Pretend that policy resource as an actual object and fetch its policy for r := range objResources { // Put trailing * if exists in asterisk @@ -613,7 +614,10 @@ func GetPolicies(statements []Statement, bucketName, prefix string) map[string]B r = r[:len(r)-1] asterisk = "*" } - objectPath := r[len(awsResourcePrefix+bucketName)+1:] + var objectPath string + if len(r) >= len(awsResourcePrefix+bucketName)+1 { + objectPath = r[len(awsResourcePrefix+bucketName)+1:] + } p := GetPolicy(statements, bucketName, objectPath) policyRules[bucketName+"/"+objectPath+asterisk] = p } diff --git a/vendor/github.com/minio/minio-go/pkg/s3utils/utils.go b/vendor/github.com/minio/minio-go/pkg/s3utils/utils.go index bfeb73e41..adceb7f2a 100644 --- a/vendor/github.com/minio/minio-go/pkg/s3utils/utils.go +++ b/vendor/github.com/minio/minio-go/pkg/s3utils/utils.go @@ -143,11 +143,40 @@ func IsAmazonGovCloudEndpoint(endpointURL url.URL) bool { } // IsAmazonFIPSGovCloudEndpoint - Match if it is exactly Amazon S3 FIPS GovCloud endpoint. +// See https://aws.amazon.com/compliance/fips. func IsAmazonFIPSGovCloudEndpoint(endpointURL url.URL) bool { if endpointURL == sentinelURL { return false } - return endpointURL.Host == "s3-fips-us-gov-west-1.amazonaws.com" + return endpointURL.Host == "s3-fips-us-gov-west-1.amazonaws.com" || + endpointURL.Host == "s3-fips.dualstack.us-gov-west-1.amazonaws.com" +} + +// IsAmazonFIPSUSEastWestEndpoint - Match if it is exactly Amazon S3 FIPS US East/West endpoint. +// See https://aws.amazon.com/compliance/fips. +func IsAmazonFIPSUSEastWestEndpoint(endpointURL url.URL) bool { + if endpointURL == sentinelURL { + return false + } + switch endpointURL.Host { + case "s3-fips.us-east-2.amazonaws.com": + case "s3-fips.dualstack.us-west-1.amazonaws.com": + case "s3-fips.dualstack.us-west-2.amazonaws.com": + case "s3-fips.dualstack.us-east-2.amazonaws.com": + case "s3-fips.dualstack.us-east-1.amazonaws.com": + case "s3-fips.us-west-1.amazonaws.com": + case "s3-fips.us-west-2.amazonaws.com": + case "s3-fips.us-east-1.amazonaws.com": + default: + return false + } + return true +} + +// IsAmazonFIPSEndpoint - Match if it is exactly Amazon S3 FIPS endpoint. +// See https://aws.amazon.com/compliance/fips. +func IsAmazonFIPSEndpoint(endpointURL url.URL) bool { + return IsAmazonFIPSUSEastWestEndpoint(endpointURL) || IsAmazonFIPSGovCloudEndpoint(endpointURL) } // IsGoogleEndpoint - Match if it is exactly Google cloud storage endpoint. diff --git a/vendor/github.com/minio/minio-go/post-policy.go b/vendor/github.com/minio/minio-go/post-policy.go index b3ae7050a..c285fdefd 100644 --- a/vendor/github.com/minio/minio-go/post-policy.go +++ b/vendor/github.com/minio/minio-go/post-policy.go @@ -206,6 +206,28 @@ func (p *PostPolicy) SetUserMetadata(key string, value string) error { return nil } +// SetUserData - Set user data as a key/value couple. +// Can be retrieved through a HEAD request or an event. +func (p *PostPolicy) SetUserData(key string, value string) error { + if key == "" { + return ErrInvalidArgument("Key is empty") + } + if value == "" { + return ErrInvalidArgument("Value is empty") + } + headerName := fmt.Sprintf("x-amz-%s", key) + policyCond := policyCondition{ + matchType: "eq", + condition: fmt.Sprintf("$%s", headerName), + value: value, + } + if err := p.addNewPolicy(policyCond); err != nil { + return err + } + p.formData[headerName] = value + return nil +} + // addNewPolicy - internal helper to validate adding new policies. func (p *PostPolicy) addNewPolicy(policyCond policyCondition) error { if policyCond.matchType == "" || policyCond.condition == "" || policyCond.value == "" { diff --git a/vendor/github.com/minio/minio-go/s3-error.go b/vendor/github.com/minio/minio-go/s3-error.go index f9e82334a..3b11776c2 100644 --- a/vendor/github.com/minio/minio-go/s3-error.go +++ b/vendor/github.com/minio/minio-go/s3-error.go @@ -34,7 +34,7 @@ var s3ErrorResponseMap = map[string]string{ "MissingContentLength": "You must provide the Content-Length HTTP header.", "MissingContentMD5": "Missing required header for this request: Content-Md5.", "MissingRequestBodyError": "Request body is empty.", - "NoSuchBucket": "The specified bucket does not exist", + "NoSuchBucket": "The specified bucket does not exist.", "NoSuchBucketPolicy": "The bucket policy does not exist", "NoSuchKey": "The specified key does not exist.", "NoSuchUpload": "The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.", diff --git a/vendor/vendor.json b/vendor/vendor.json index ae5920ab2..e5e6e19c1 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -651,46 +651,46 @@ "revisionTime": "2016-02-29T08:42:30-08:00" }, { - "checksumSHA1": "Wbe5TjRIOZiWVu4l4dwzCw/uP9w=", + "checksumSHA1": "iiDPCgPen0lGZrkUw7qwoz7bFNU=", "path": "github.com/minio/minio-go", - "revision": "519049881e73150d1bbeac1d443e7c96b76e1b8d", - "revisionTime": "2018-09-05T00:47:51Z" + "revision": "39381cf62425050629c7264228fc2f9e0c6616f6", + "revisionTime": "2018-11-15T04:56:45Z" }, { - "checksumSHA1": "Qsj+6JPmJ8R5rFNQSHqRb8xAwOw=", + "checksumSHA1": "+Zp42S4+zz4vVF2jDZw9UPbSLt8=", "path": "github.com/minio/minio-go/pkg/credentials", - "revision": "a2238f0e60facba1c2ccff86ddc624e23fae4f23", - "revisionTime": "2018-04-13T18:58:16Z" + "revision": "39381cf62425050629c7264228fc2f9e0c6616f6", + "revisionTime": "2018-11-15T04:56:45Z" }, { "checksumSHA1": "Md5pOKYfoKtrG7xNvs2FtiDPfDc=", "path": "github.com/minio/minio-go/pkg/encrypt", - "revision": "a2238f0e60facba1c2ccff86ddc624e23fae4f23", - "revisionTime": "2018-04-13T18:58:16Z" + "revision": "39381cf62425050629c7264228fc2f9e0c6616f6", + "revisionTime": "2018-11-15T04:56:45Z" }, { - "checksumSHA1": "Df4SG4ratsLn8yE9SkHcZiKhKRU=", + "checksumSHA1": "BxFHeQVFZ0t/ZFBJTeM2uegFZI8=", "path": "github.com/minio/minio-go/pkg/policy", - "revision": "a2238f0e60facba1c2ccff86ddc624e23fae4f23", - "revisionTime": "2018-04-13T18:58:16Z" + "revision": "39381cf62425050629c7264228fc2f9e0c6616f6", + "revisionTime": "2018-11-15T04:56:45Z" }, { "checksumSHA1": "bbWjcrOQsV57qK+BSsrNAsI+Q/o=", "path": "github.com/minio/minio-go/pkg/s3signer", - "revision": "a2238f0e60facba1c2ccff86ddc624e23fae4f23", - "revisionTime": "2018-04-13T18:58:16Z" + "revision": "39381cf62425050629c7264228fc2f9e0c6616f6", + "revisionTime": "2018-11-15T04:56:45Z" }, { - "checksumSHA1": "xrJThFwwkVrJdwd5iYFHqfx4wRY=", + "checksumSHA1": "7iUaZkEJdhkyAu3F07vrX8pyavI=", "path": "github.com/minio/minio-go/pkg/s3utils", - "revision": "a2238f0e60facba1c2ccff86ddc624e23fae4f23", - "revisionTime": "2018-04-13T18:58:16Z" + "revision": "39381cf62425050629c7264228fc2f9e0c6616f6", + "revisionTime": "2018-11-15T04:56:45Z" }, { "checksumSHA1": "Wt8ej+rZXTdNBR9Xyw1eGo3Iq5o=", "path": "github.com/minio/minio-go/pkg/set", - "revision": "a2238f0e60facba1c2ccff86ddc624e23fae4f23", - "revisionTime": "2018-04-13T18:58:16Z" + "revision": "39381cf62425050629c7264228fc2f9e0c6616f6", + "revisionTime": "2018-11-15T04:56:45Z" }, { "checksumSHA1": "cYuXpiVBMypgkEr0Wqd79jPPyBg=",