Update vendorized minio-go to support start-after param (#6043)

Fixes #6032
This commit is contained in:
Nitish Tiwari 2018-06-15 03:38:02 +05:30 committed by kannappanr
parent 28d526bc68
commit 36c39d04da
15 changed files with 8014 additions and 40 deletions

View File

@ -255,7 +255,7 @@ func (l *s3Objects) ListObjects(ctx context.Context, bucket string, prefix strin
// ListObjectsV2 lists all blobs in S3 bucket filtered by prefix // ListObjectsV2 lists all blobs in S3 bucket filtered by prefix
func (l *s3Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, e error) { func (l *s3Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, e error) {
result, err := l.Client.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys) result, err := l.Client.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys, startAfter)
if err != nil { if err != nil {
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
return loi, minio.ErrorRespToObjectError(err, bucket) return loi, minio.ErrorRespToObjectError(err, bucket)

View File

@ -1,6 +1,6 @@
/* /*
* Minio Go Library for Amazon S3 Compatible Cloud Storage * Minio Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2017 Minio, Inc. * Copyright 2017, 2018 Minio, Inc.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -20,6 +20,8 @@ package minio
import ( import (
"context" "context"
"fmt" "fmt"
"io"
"io/ioutil"
"net/http" "net/http"
"net/url" "net/url"
"strconv" "strconv"
@ -343,11 +345,12 @@ func (c Client) uploadPartCopy(ctx context.Context, bucket, object, uploadID str
return p, nil return p, nil
} }
// ComposeObject - creates an object using server-side copying of // ComposeObjectWithProgress - creates an object using server-side copying of
// existing objects. It takes a list of source objects (with optional // existing objects. It takes a list of source objects (with optional
// offsets) and concatenates them into a new object using only // offsets) and concatenates them into a new object using only
// server-side copying operations. // server-side copying operations. Optionally takes progress reader hook
func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error { // for applications to look at current progress.
func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo, progress io.Reader) error {
if len(srcs) < 1 || len(srcs) > maxPartsCount { if len(srcs) < 1 || len(srcs) > maxPartsCount {
return ErrInvalidArgument("There must be as least one and up to 10000 source objects.") return ErrInvalidArgument("There must be as least one and up to 10000 source objects.")
} }
@ -421,7 +424,7 @@ func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error {
// involved, it is being copied wholly and at most 5GiB in // involved, it is being copied wholly and at most 5GiB in
// size, emptyfiles are also supported). // size, emptyfiles are also supported).
if (totalParts == 1 && srcs[0].start == -1 && totalSize <= maxPartSize) || (totalSize == 0) { if (totalParts == 1 && srcs[0].start == -1 && totalSize <= maxPartSize) || (totalSize == 0) {
return c.CopyObject(dst, srcs[0]) return c.CopyObjectWithProgress(dst, srcs[0], progress)
} }
// Now, handle multipart-copy cases. // Now, handle multipart-copy cases.
@ -476,6 +479,9 @@ func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error {
if err != nil { if err != nil {
return err return err
} }
if progress != nil {
io.CopyN(ioutil.Discard, progress, start+end-1)
}
objParts = append(objParts, complPart) objParts = append(objParts, complPart)
partIndex++ partIndex++
} }
@ -490,10 +496,20 @@ func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error {
return nil return nil
} }
// partsRequired is ceiling(size / copyPartSize) // ComposeObject - creates an object using server-side copying of
// existing objects. It takes a list of source objects (with optional
// offsets) and concatenates them into a new object using only
// server-side copying operations.
func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error {
return c.ComposeObjectWithProgress(dst, srcs, nil)
}
// partsRequired is maximum parts possible with
// max part size of ceiling(maxMultipartPutObjectSize / (maxPartsCount - 1))
func partsRequired(size int64) int64 { func partsRequired(size int64) int64 {
r := size / copyPartSize maxPartSize := maxMultipartPutObjectSize / (maxPartsCount - 1)
if size%copyPartSize > 0 { r := size / int64(maxPartSize)
if size%int64(maxPartSize) > 0 {
r++ r++
} }
return r return r

View File

@ -118,7 +118,7 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d
var continuationToken string var continuationToken string
for { for {
// Get list of objects a maximum of 1000 per request. // Get list of objects a maximum of 1000 per request.
result, err := c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, 1000) result, err := c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, 1000, "")
if err != nil { if err != nil {
objectStatCh <- ObjectInfo{ objectStatCh <- ObjectInfo{
Err: err, Err: err,
@ -171,11 +171,12 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d
// You can use the request parameters as selection criteria to return a subset of the objects in a bucket. // You can use the request parameters as selection criteria to return a subset of the objects in a bucket.
// request parameters :- // request parameters :-
// --------- // ---------
// ?continuation-token - Specifies the key to start with when listing objects in a bucket. // ?continuation-token - Used to continue iterating over a set of objects
// ?delimiter - A delimiter is a character you use to group keys. // ?delimiter - A delimiter is a character you use to group keys.
// ?prefix - Limits the response to keys that begin with the specified prefix. // ?prefix - Limits the response to keys that begin with the specified prefix.
// ?max-keys - Sets the maximum number of keys returned in the response body. // ?max-keys - Sets the maximum number of keys returned in the response body.
func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int) (ListBucketV2Result, error) { // ?start-after - Specifies the key to start after when listing objects in a bucket.
func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) {
// Validate bucket name. // Validate bucket name.
if err := s3utils.CheckValidBucketName(bucketName); err != nil { if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return ListBucketV2Result{}, err return ListBucketV2Result{}, err
@ -216,6 +217,11 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s
// Set max keys. // Set max keys.
urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys))
// Set start-after
if startAfter != "" {
urlValues.Set("start-after", startAfter)
}
// Execute GET on bucket to list objects. // Execute GET on bucket to list objects.
resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{
bucketName: bucketName, bucketName: bucketName,

View File

@ -1,6 +1,6 @@
/* /*
* Minio Go Library for Amazon S3 Compatible Cloud Storage * Minio Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2017 Minio, Inc. * Copyright 2017, 2018 Minio, Inc.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -19,6 +19,8 @@ package minio
import ( import (
"context" "context"
"io"
"io/ioutil"
"net/http" "net/http"
"github.com/minio/minio-go/pkg/encrypt" "github.com/minio/minio-go/pkg/encrypt"
@ -26,13 +28,31 @@ import (
// CopyObject - copy a source object into a new object // CopyObject - copy a source object into a new object
func (c Client) CopyObject(dst DestinationInfo, src SourceInfo) error { func (c Client) CopyObject(dst DestinationInfo, src SourceInfo) error {
return c.CopyObjectWithProgress(dst, src, nil)
}
// CopyObjectWithProgress - copy a source object into a new object, optionally takes
// progress bar input to notify current progress.
func (c Client) CopyObjectWithProgress(dst DestinationInfo, src SourceInfo, progress io.Reader) error {
header := make(http.Header) header := make(http.Header)
for k, v := range src.Headers { for k, v := range src.Headers {
header[k] = v header[k] = v
} }
var err error
var size int64
// If progress bar is specified, size should be requested as well initiate a StatObject request.
if progress != nil {
size, _, _, err = src.getProps(c)
if err != nil {
return err
}
}
if src.encryption != nil { if src.encryption != nil {
encrypt.SSECopy(src.encryption).Marshal(header) encrypt.SSECopy(src.encryption).Marshal(header)
} }
if dst.encryption != nil { if dst.encryption != nil {
dst.encryption.Marshal(header) dst.encryption.Marshal(header)
} }
@ -53,5 +73,11 @@ func (c Client) CopyObject(dst DestinationInfo, src SourceInfo) error {
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
return httpRespToErrorResponse(resp, dst.bucket, dst.object) return httpRespToErrorResponse(resp, dst.bucket, dst.object)
} }
// Update the progress properly after successful copy.
if progress != nil {
io.CopyN(ioutil.Discard, progress, size)
}
return nil return nil
} }

View File

@ -28,21 +28,22 @@ import (
"github.com/minio/minio-go/pkg/encrypt" "github.com/minio/minio-go/pkg/encrypt"
"github.com/minio/minio-go/pkg/s3utils" "github.com/minio/minio-go/pkg/s3utils"
"golang.org/x/net/lex/httplex" "golang.org/x/net/http/httpguts"
) )
// PutObjectOptions represents options specified by user for PutObject call // PutObjectOptions represents options specified by user for PutObject call
type PutObjectOptions struct { type PutObjectOptions struct {
UserMetadata map[string]string UserMetadata map[string]string
Progress io.Reader Progress io.Reader
ContentType string ContentType string
ContentEncoding string ContentEncoding string
ContentDisposition string ContentDisposition string
ContentLanguage string ContentLanguage string
CacheControl string CacheControl string
ServerSideEncryption encrypt.ServerSide ServerSideEncryption encrypt.ServerSide
NumThreads uint NumThreads uint
StorageClass string StorageClass string
WebsiteRedirectLocation string
} }
// getNumThreads - gets the number of threads to be used in the multipart // getNumThreads - gets the number of threads to be used in the multipart
@ -84,6 +85,9 @@ func (opts PutObjectOptions) Header() (header http.Header) {
if opts.StorageClass != "" { if opts.StorageClass != "" {
header[amzStorageClass] = []string{opts.StorageClass} header[amzStorageClass] = []string{opts.StorageClass}
} }
if opts.WebsiteRedirectLocation != "" {
header[amzWebsiteRedirectLocation] = []string{opts.WebsiteRedirectLocation}
}
for k, v := range opts.UserMetadata { for k, v := range opts.UserMetadata {
if !isAmzHeader(k) && !isStandardHeader(k) && !isStorageClassHeader(k) { if !isAmzHeader(k) && !isStandardHeader(k) && !isStorageClassHeader(k) {
header["X-Amz-Meta-"+k] = []string{v} header["X-Amz-Meta-"+k] = []string{v}
@ -97,10 +101,10 @@ func (opts PutObjectOptions) Header() (header http.Header) {
// validate() checks if the UserMetadata map has standard headers or and raises an error if so. // validate() checks if the UserMetadata map has standard headers or and raises an error if so.
func (opts PutObjectOptions) validate() (err error) { func (opts PutObjectOptions) validate() (err error) {
for k, v := range opts.UserMetadata { for k, v := range opts.UserMetadata {
if !httplex.ValidHeaderFieldName(k) || isStandardHeader(k) || isSSEHeader(k) || isStorageClassHeader(k) { if !httpguts.ValidHeaderFieldName(k) || isStandardHeader(k) || isSSEHeader(k) || isStorageClassHeader(k) {
return ErrInvalidArgument(k + " unsupported user defined metadata name") return ErrInvalidArgument(k + " unsupported user defined metadata name")
} }
if !httplex.ValidHeaderFieldValue(v) { if !httpguts.ValidHeaderFieldValue(v) {
return ErrInvalidArgument(v + " unsupported user defined metadata value") return ErrInvalidArgument(v + " unsupported user defined metadata value")
} }
} }

View File

@ -66,6 +66,9 @@ var defaultFilterKeys = []string{
"x-amz-bucket-region", "x-amz-bucket-region",
"x-amz-request-id", "x-amz-request-id",
"x-amz-id-2", "x-amz-id-2",
"Content-Security-Policy",
"X-Xss-Protection",
// Add new headers to be ignored. // Add new headers to be ignored.
} }

View File

@ -99,7 +99,7 @@ type Options struct {
// Global constants. // Global constants.
const ( const (
libraryName = "minio-go" libraryName = "minio-go"
libraryVersion = "6.0.1" libraryVersion = "v6.0.3"
) )
// User Agent should always following the below style. // User Agent should always following the below style.

View File

@ -27,10 +27,6 @@ const absMinPartSize = 1024 * 1024 * 5
// putObject behaves internally as multipart. // putObject behaves internally as multipart.
const minPartSize = 1024 * 1024 * 64 const minPartSize = 1024 * 1024 * 64
// copyPartSize - default (and maximum) part size to copy in a
// copy-object request (5GiB)
const copyPartSize = 1024 * 1024 * 1024 * 5
// maxPartsCount - maximum number of parts for a single multipart session. // maxPartsCount - maximum number of parts for a single multipart session.
const maxPartsCount = 10000 const maxPartsCount = 10000
@ -61,3 +57,6 @@ const (
// Storage class header constant. // Storage class header constant.
const amzStorageClass = "X-Amz-Storage-Class" const amzStorageClass = "X-Amz-Storage-Class"
// Website redirect location header constant
const amzWebsiteRedirectLocation = "X-Amz-Website-Redirect-Location"

View File

@ -48,9 +48,9 @@ func (c Core) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int)
} }
// ListObjectsV2 - Lists all the objects at a prefix, similar to ListObjects() but uses // ListObjectsV2 - Lists all the objects at a prefix, similar to ListObjects() but uses
// continuationToken instead of marker to further filter the results. // continuationToken instead of marker to support iteration over the results.
func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int) (ListBucketV2Result, error) { func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) {
return c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, maxkeys) return c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, maxkeys, startAfter)
} }
// CopyObject - copies an object from source object to destination object on server side. // CopyObject - copies an object from source object to destination object on server side.
@ -82,6 +82,8 @@ func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Ba
opts.ContentType = v opts.ContentType = v
} else if strings.ToLower(k) == "cache-control" { } else if strings.ToLower(k) == "cache-control" {
opts.CacheControl = v opts.CacheControl = v
} else if strings.ToLower(k) == strings.ToLower(amzWebsiteRedirectLocation) {
opts.WebsiteRedirectLocation = v
} else { } else {
m[k] = metadata[k] m[k] = metadata[k]
} }

7499
vendor/github.com/minio/minio-go/functional_tests.go generated vendored Normal file

File diff suppressed because it is too large Load Diff

View File

@ -131,6 +131,7 @@ var retryableS3Codes = map[string]struct{}{
"InternalError": {}, "InternalError": {},
"ExpiredToken": {}, "ExpiredToken": {},
"ExpiredTokenException": {}, "ExpiredTokenException": {},
"SlowDown": {},
// Add more AWS S3 codes here. // Add more AWS S3 codes here.
} }

View File

@ -222,6 +222,7 @@ var supportedHeaders = []string{
"content-encoding", "content-encoding",
"content-disposition", "content-disposition",
"content-language", "content-language",
"x-amz-website-redirect-location",
// Add more supported headers here. // Add more supported headers here.
} }

65
vendor/golang.org/x/net/http/httpguts/guts.go generated vendored Normal file
View File

@ -0,0 +1,65 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package httpguts provides functions implementing various details
// of the HTTP specification.
//
// This package is shared by the standard library (which vendors it)
// and x/net/http2. It comes with no API stability promise.
package httpguts
import (
"net/textproto"
"strings"
)
// SniffedContentType reports whether ct is a Content-Type that is known
// to cause client-side content sniffing.
//
// This provides just a partial implementation of mime.ParseMediaType
// with the assumption that the Content-Type is not attacker controlled.
func SniffedContentType(ct string) bool {
if i := strings.Index(ct, ";"); i != -1 {
ct = ct[:i]
}
ct = strings.ToLower(strings.TrimSpace(ct))
return ct == "text/plain" || ct == "application/octet-stream" ||
ct == "application/unknown" || ct == "unknown/unknown" || ct == "*/*" ||
!strings.Contains(ct, "/")
}
// ValidTrailerHeader reports whether name is a valid header field name to appear
// in trailers.
// See RFC 7230, Section 4.1.2
func ValidTrailerHeader(name string) bool {
name = textproto.CanonicalMIMEHeaderKey(name)
if strings.HasPrefix(name, "If-") || badTrailer[name] {
return false
}
return true
}
var badTrailer = map[string]bool{
"Authorization": true,
"Cache-Control": true,
"Connection": true,
"Content-Encoding": true,
"Content-Length": true,
"Content-Range": true,
"Content-Type": true,
"Expect": true,
"Host": true,
"Keep-Alive": true,
"Max-Forwards": true,
"Pragma": true,
"Proxy-Authenticate": true,
"Proxy-Authorization": true,
"Proxy-Connection": true,
"Range": true,
"Realm": true,
"Te": true,
"Trailer": true,
"Transfer-Encoding": true,
"Www-Authenticate": true,
}

346
vendor/golang.org/x/net/http/httpguts/httplex.go generated vendored Normal file
View File

@ -0,0 +1,346 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package httpguts
import (
"net"
"strings"
"unicode/utf8"
"golang.org/x/net/idna"
)
var isTokenTable = [127]bool{
'!': true,
'#': true,
'$': true,
'%': true,
'&': true,
'\'': true,
'*': true,
'+': true,
'-': true,
'.': true,
'0': true,
'1': true,
'2': true,
'3': true,
'4': true,
'5': true,
'6': true,
'7': true,
'8': true,
'9': true,
'A': true,
'B': true,
'C': true,
'D': true,
'E': true,
'F': true,
'G': true,
'H': true,
'I': true,
'J': true,
'K': true,
'L': true,
'M': true,
'N': true,
'O': true,
'P': true,
'Q': true,
'R': true,
'S': true,
'T': true,
'U': true,
'W': true,
'V': true,
'X': true,
'Y': true,
'Z': true,
'^': true,
'_': true,
'`': true,
'a': true,
'b': true,
'c': true,
'd': true,
'e': true,
'f': true,
'g': true,
'h': true,
'i': true,
'j': true,
'k': true,
'l': true,
'm': true,
'n': true,
'o': true,
'p': true,
'q': true,
'r': true,
's': true,
't': true,
'u': true,
'v': true,
'w': true,
'x': true,
'y': true,
'z': true,
'|': true,
'~': true,
}
func IsTokenRune(r rune) bool {
i := int(r)
return i < len(isTokenTable) && isTokenTable[i]
}
func isNotToken(r rune) bool {
return !IsTokenRune(r)
}
// HeaderValuesContainsToken reports whether any string in values
// contains the provided token, ASCII case-insensitively.
func HeaderValuesContainsToken(values []string, token string) bool {
for _, v := range values {
if headerValueContainsToken(v, token) {
return true
}
}
return false
}
// isOWS reports whether b is an optional whitespace byte, as defined
// by RFC 7230 section 3.2.3.
func isOWS(b byte) bool { return b == ' ' || b == '\t' }
// trimOWS returns x with all optional whitespace removes from the
// beginning and end.
func trimOWS(x string) string {
// TODO: consider using strings.Trim(x, " \t") instead,
// if and when it's fast enough. See issue 10292.
// But this ASCII-only code will probably always beat UTF-8
// aware code.
for len(x) > 0 && isOWS(x[0]) {
x = x[1:]
}
for len(x) > 0 && isOWS(x[len(x)-1]) {
x = x[:len(x)-1]
}
return x
}
// headerValueContainsToken reports whether v (assumed to be a
// 0#element, in the ABNF extension described in RFC 7230 section 7)
// contains token amongst its comma-separated tokens, ASCII
// case-insensitively.
func headerValueContainsToken(v string, token string) bool {
v = trimOWS(v)
if comma := strings.IndexByte(v, ','); comma != -1 {
return tokenEqual(trimOWS(v[:comma]), token) || headerValueContainsToken(v[comma+1:], token)
}
return tokenEqual(v, token)
}
// lowerASCII returns the ASCII lowercase version of b.
func lowerASCII(b byte) byte {
if 'A' <= b && b <= 'Z' {
return b + ('a' - 'A')
}
return b
}
// tokenEqual reports whether t1 and t2 are equal, ASCII case-insensitively.
func tokenEqual(t1, t2 string) bool {
if len(t1) != len(t2) {
return false
}
for i, b := range t1 {
if b >= utf8.RuneSelf {
// No UTF-8 or non-ASCII allowed in tokens.
return false
}
if lowerASCII(byte(b)) != lowerASCII(t2[i]) {
return false
}
}
return true
}
// isLWS reports whether b is linear white space, according
// to http://www.w3.org/Protocols/rfc2616/rfc2616-sec2.html#sec2.2
// LWS = [CRLF] 1*( SP | HT )
func isLWS(b byte) bool { return b == ' ' || b == '\t' }
// isCTL reports whether b is a control byte, according
// to http://www.w3.org/Protocols/rfc2616/rfc2616-sec2.html#sec2.2
// CTL = <any US-ASCII control character
// (octets 0 - 31) and DEL (127)>
func isCTL(b byte) bool {
const del = 0x7f // a CTL
return b < ' ' || b == del
}
// ValidHeaderFieldName reports whether v is a valid HTTP/1.x header name.
// HTTP/2 imposes the additional restriction that uppercase ASCII
// letters are not allowed.
//
// RFC 7230 says:
// header-field = field-name ":" OWS field-value OWS
// field-name = token
// token = 1*tchar
// tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" / "+" / "-" / "." /
// "^" / "_" / "`" / "|" / "~" / DIGIT / ALPHA
func ValidHeaderFieldName(v string) bool {
if len(v) == 0 {
return false
}
for _, r := range v {
if !IsTokenRune(r) {
return false
}
}
return true
}
// ValidHostHeader reports whether h is a valid host header.
func ValidHostHeader(h string) bool {
// The latest spec is actually this:
//
// http://tools.ietf.org/html/rfc7230#section-5.4
// Host = uri-host [ ":" port ]
//
// Where uri-host is:
// http://tools.ietf.org/html/rfc3986#section-3.2.2
//
// But we're going to be much more lenient for now and just
// search for any byte that's not a valid byte in any of those
// expressions.
for i := 0; i < len(h); i++ {
if !validHostByte[h[i]] {
return false
}
}
return true
}
// See the validHostHeader comment.
var validHostByte = [256]bool{
'0': true, '1': true, '2': true, '3': true, '4': true, '5': true, '6': true, '7': true,
'8': true, '9': true,
'a': true, 'b': true, 'c': true, 'd': true, 'e': true, 'f': true, 'g': true, 'h': true,
'i': true, 'j': true, 'k': true, 'l': true, 'm': true, 'n': true, 'o': true, 'p': true,
'q': true, 'r': true, 's': true, 't': true, 'u': true, 'v': true, 'w': true, 'x': true,
'y': true, 'z': true,
'A': true, 'B': true, 'C': true, 'D': true, 'E': true, 'F': true, 'G': true, 'H': true,
'I': true, 'J': true, 'K': true, 'L': true, 'M': true, 'N': true, 'O': true, 'P': true,
'Q': true, 'R': true, 'S': true, 'T': true, 'U': true, 'V': true, 'W': true, 'X': true,
'Y': true, 'Z': true,
'!': true, // sub-delims
'$': true, // sub-delims
'%': true, // pct-encoded (and used in IPv6 zones)
'&': true, // sub-delims
'(': true, // sub-delims
')': true, // sub-delims
'*': true, // sub-delims
'+': true, // sub-delims
',': true, // sub-delims
'-': true, // unreserved
'.': true, // unreserved
':': true, // IPv6address + Host expression's optional port
';': true, // sub-delims
'=': true, // sub-delims
'[': true,
'\'': true, // sub-delims
']': true,
'_': true, // unreserved
'~': true, // unreserved
}
// ValidHeaderFieldValue reports whether v is a valid "field-value" according to
// http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2 :
//
// message-header = field-name ":" [ field-value ]
// field-value = *( field-content | LWS )
// field-content = <the OCTETs making up the field-value
// and consisting of either *TEXT or combinations
// of token, separators, and quoted-string>
//
// http://www.w3.org/Protocols/rfc2616/rfc2616-sec2.html#sec2.2 :
//
// TEXT = <any OCTET except CTLs,
// but including LWS>
// LWS = [CRLF] 1*( SP | HT )
// CTL = <any US-ASCII control character
// (octets 0 - 31) and DEL (127)>
//
// RFC 7230 says:
// field-value = *( field-content / obs-fold )
// obj-fold = N/A to http2, and deprecated
// field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ]
// field-vchar = VCHAR / obs-text
// obs-text = %x80-FF
// VCHAR = "any visible [USASCII] character"
//
// http2 further says: "Similarly, HTTP/2 allows header field values
// that are not valid. While most of the values that can be encoded
// will not alter header field parsing, carriage return (CR, ASCII
// 0xd), line feed (LF, ASCII 0xa), and the zero character (NUL, ASCII
// 0x0) might be exploited by an attacker if they are translated
// verbatim. Any request or response that contains a character not
// permitted in a header field value MUST be treated as malformed
// (Section 8.1.2.6). Valid characters are defined by the
// field-content ABNF rule in Section 3.2 of [RFC7230]."
//
// This function does not (yet?) properly handle the rejection of
// strings that begin or end with SP or HTAB.
func ValidHeaderFieldValue(v string) bool {
for i := 0; i < len(v); i++ {
b := v[i]
if isCTL(b) && !isLWS(b) {
return false
}
}
return true
}
func isASCII(s string) bool {
for i := 0; i < len(s); i++ {
if s[i] >= utf8.RuneSelf {
return false
}
}
return true
}
// PunycodeHostPort returns the IDNA Punycode version
// of the provided "host" or "host:port" string.
func PunycodeHostPort(v string) (string, error) {
if isASCII(v) {
return v, nil
}
host, port, err := net.SplitHostPort(v)
if err != nil {
// The input 'v' argument was just a "host" argument,
// without a port. This error should not be returned
// to the caller.
host = v
port = ""
}
host, err = idna.ToASCII(host)
if err != nil {
// Non-UTF-8? Not representable in Punycode, in any
// case.
return "", err
}
if port == "" {
return host, nil
}
return net.JoinHostPort(host, port), nil
}

18
vendor/vendor.json vendored
View File

@ -414,9 +414,9 @@
"checksumSHA1": "bKMZjd2wPw13VwoE7mBeSv5djFA=", "checksumSHA1": "bKMZjd2wPw13VwoE7mBeSv5djFA=",
"path": "github.com/matttproud/golang_protobuf_extensions/pbutil", "path": "github.com/matttproud/golang_protobuf_extensions/pbutil",
"revision": "c12348ce28de40eed0136aa2b644d0ee0650e56c", "revision": "c12348ce28de40eed0136aa2b644d0ee0650e56c",
"revisionTime": "2016-04-24T11:30:07Z" "revisionTime": "2016-04-24T11:30:07Z"
}, },
{ {
"checksumSHA1": "OUZ1FFXyKs+Cfg9M9rmXqqweQck=", "checksumSHA1": "OUZ1FFXyKs+Cfg9M9rmXqqweQck=",
"path": "github.com/miekg/dns", "path": "github.com/miekg/dns",
"revision": "db96a2b759cdef4f11a34506a42eb8d1290c598e", "revision": "db96a2b759cdef4f11a34506a42eb8d1290c598e",
@ -470,10 +470,10 @@
"revisionTime": "2016-02-29T08:42:30-08:00" "revisionTime": "2016-02-29T08:42:30-08:00"
}, },
{ {
"checksumSHA1": "w7jIUKw0cR7Z34JykcQXGJEPEmI=", "checksumSHA1": "hdWmWbGpljSQMBOcpcwhAnP2aaQ=",
"path": "github.com/minio/minio-go", "path": "github.com/minio/minio-go",
"revision": "a2238f0e60facba1c2ccff86ddc624e23fae4f23", "revision": "10531abd0af1579a12dc1977d67c0fec2b348679",
"revisionTime": "2018-04-13T18:58:16Z" "revisionTime": "2018-06-13T23:01:28Z"
}, },
{ {
"checksumSHA1": "Qsj+6JPmJ8R5rFNQSHqRb8xAwOw=", "checksumSHA1": "Qsj+6JPmJ8R5rFNQSHqRb8xAwOw=",
@ -783,6 +783,12 @@
"revision": "f2499483f923065a842d38eb4c7f1927e6fc6e6d", "revision": "f2499483f923065a842d38eb4c7f1927e6fc6e6d",
"revisionTime": "2017-01-14T04:22:49Z" "revisionTime": "2017-01-14T04:22:49Z"
}, },
{
"checksumSHA1": "c3H2wB/3tGrw6VqCnlye+kSdoXU=",
"path": "golang.org/x/net/http/httpguts",
"revision": "1e491301e022f8f977054da4c2d852decd59571f",
"revisionTime": "2018-05-30T06:29:46Z"
},
{ {
"checksumSHA1": "Zh++JEDfXo0DKQtoKVfScwVdvww=", "checksumSHA1": "Zh++JEDfXo0DKQtoKVfScwVdvww=",
"path": "golang.org/x/net/http2", "path": "golang.org/x/net/http2",