Filter Expires header from user metadata (#7269)

Instead save it as a struct field in ObjectInfo as it is
a standard HTTP header - Fixes minio/mc#2690

cmd/api-headers.go

@@ -87,6 +87,9 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		w.Header().Set("Content-Encoding", objInfo.ContentEncoding)
 	}
 
+	if !objInfo.Expires.IsZero() {
+		w.Header().Set("Expires", objInfo.Expires.UTC().Format(http.TimeFormat))
+	}
 	// Set all other user defined metadata.
 	for k, v := range objInfo.UserDefined {
 		if hasPrefix(k, ReservedMetadataPrefix) {

cmd/fs-v1-metadata.go

@@ -22,8 +22,10 @@ import (
 	"encoding/json"
 	"io"
 	"io/ioutil"
+	"net/http"
 	"os"
 	pathutil "path"
+	"time"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lock"
@@ -169,7 +171,15 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 	} else {
 		objInfo.StorageClass = globalMinioDefaultStorageClass
 	}
-
+	var (
+		t time.Time
+		e error
+	)
+	if exp, ok := m.Meta["expires"]; ok {
+		if t, e = time.Parse(http.TimeFormat, exp); e == nil {
+			objInfo.Expires = t.UTC()
+		}
+	}
 	// etag/md5Sum has already been extracted. We need to
 	// remove to avoid it from appearing as part of
 	// response headers. e.g, X-Minio-* or X-Amz-*.

cmd/fs-v1-metadata_test.go

@@ -38,6 +38,9 @@ func TestFSV1MetadataObjInfo(t *testing.T) {
 	if objInfo.IsDir {
 		t.Fatal("Unexpected object info value for IsDir", objInfo.IsDir)
 	}
+	if !objInfo.Expires.IsZero() {
+		t.Fatal("Unexpected object info value for Expires ", objInfo.Expires)
+	}
 }
 
 // TestReadFSMetadata - readFSMetadata testing with a healthy and faulty disk

cmd/gateway-common.go

@@ -166,6 +166,7 @@ func FromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo {
 		ContentType:     oi.ContentType,
 		ContentEncoding: oi.Metadata.Get("Content-Encoding"),
 		StorageClass:    oi.StorageClass,
+		Expires:         oi.Expires,
 	}
 }
 

cmd/gateway/gcs/gateway-gcs.go

@@ -797,12 +797,22 @@ func fromGCSAttrsToObjectInfo(attrs *storage.ObjectAttrs) minio.ObjectInfo {
 	// All google cloud storage objects have a CRC32c hash, whereas composite objects may not have a MD5 hash
 	// Refer https://cloud.google.com/storage/docs/hashes-etags. Use CRC32C for ETag
 	metadata := make(map[string]string)
+	var (
+		expiry time.Time
+		e      error
+	)
 	for k, v := range attrs.Metadata {
 		k = http.CanonicalHeaderKey(k)
 		// Translate the GCS custom metadata prefix
 		if strings.HasPrefix(k, "X-Goog-Meta-") {
 			k = strings.Replace(k, "X-Goog-Meta-", "X-Amz-Meta-", 1)
 		}
+		if k == "Expires" {
+			if expiry, e = time.Parse(http.TimeFormat, v); e == nil {
+				expiry = expiry.UTC()
+			}
+			continue
+		}
 		metadata[k] = v
 	}
 	if attrs.ContentType != "" {
@@ -829,6 +839,7 @@ func fromGCSAttrsToObjectInfo(attrs *storage.ObjectAttrs) minio.ObjectInfo {
 		UserDefined:     metadata,
 		ContentType:     attrs.ContentType,
 		ContentEncoding: attrs.ContentEncoding,
+		Expires:         expiry,
 	}
 }
 

cmd/gateway/s3/gateway-s3-metadata.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"net/http"
 	"time"
 
 	minio "github.com/minio/minio/cmd"
@@ -83,6 +84,7 @@ func (m gwMetaV1) ToObjectInfo(bucket, object string) minio.ObjectInfo {
 		"Content-Length",
 		"Last-Modified",
 		"Content-Type",
+		"Expires",
 	}, defaultFilterKeys...)
 	objInfo := minio.ObjectInfo{
 		IsDir:           false,
@@ -100,6 +102,15 @@ func (m gwMetaV1) ToObjectInfo(bucket, object string) minio.ObjectInfo {
 	if sc, ok := m.Meta["x-amz-storage-class"]; ok {
 		objInfo.StorageClass = sc
 	}
+	var (
+		t time.Time
+		e error
+	)
+	if exp, ok := m.Meta["expires"]; ok {
+		if t, e = time.Parse(http.TimeFormat, exp); e == nil {
+			objInfo.Expires = t.UTC()
+		}
+	}
 	// Success.
 	return objInfo
 }

cmd/object-api-datatypes.go

@@ -96,6 +96,9 @@ type ObjectInfo struct {
 	// by the Content-Type header field.
 	ContentEncoding string
 
+	// Date and time at which the object is no longer able to be cached
+	Expires time.Time
+
 	// Specify object storage class
 	StorageClass string
 

cmd/object-api-utils.go

@@ -207,8 +207,8 @@ func getCompleteMultipartMD5(ctx context.Context, parts []CompletePart) (string,
 func cleanMetadata(metadata map[string]string) map[string]string {
 	// Remove STANDARD StorageClass
 	metadata = removeStandardStorageClass(metadata)
-	// Clean meta etag keys 'md5Sum', 'etag'.
+	// Clean meta etag keys 'md5Sum', 'etag', "expires".
-	return cleanMetadataKeys(metadata, "md5Sum", "etag")
+	return cleanMetadataKeys(metadata, "md5Sum", "etag", "expires")
 }
 
 // Filter X-Amz-Storage-Class field only if it is set to STANDARD.

cmd/xl-v1-metadata.go

@@ -21,6 +21,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"net/http"
 	"path"
 	"sort"
 	"sync"
@@ -226,7 +227,16 @@ func (m xlMetaV1) ToObjectInfo(bucket, object string) ObjectInfo {
 		ContentType:     m.Meta["content-type"],
 		ContentEncoding: m.Meta["content-encoding"],
 	}
-
+	// Update expires
+	var (
+		t time.Time
+		e error
+	)
+	if exp, ok := m.Meta["expires"]; ok {
+		if t, e = time.Parse(http.TimeFormat, exp); e == nil {
+			objInfo.Expires = t.UTC()
+		}
+	}
 	objInfo.backendType = BackendErasure
 
 	// Extract etag from metadata.

vendor/github.com/minio/minio-go/api-datatypes.go

@@ -41,6 +41,7 @@ type ObjectInfo struct {
 	LastModified time.Time `json:"lastModified"` // Date and time the object was last modified.
 	Size         int64     `json:"size"`         // Size in bytes of the object.
 	ContentType  string    `json:"contentType"`  // A standard MIME type describing the format of the object data.
+	Expires      time.Time `json:"expires"`      // The date and time at which the object is no longer able to be cached.
 
 	// Collection of additional metadata on the object.
 	// eg: x-amz-meta-*, content-encoding etc.

vendor/github.com/minio/minio-go/api-error-response.go

@@ -36,6 +36,8 @@ import (
 */
 
 // ErrorResponse - Is the typed error returned by all API operations.
+// ErrorResponse struct should be comparable since it is compared inside
+// golang http API (https://github.com/golang/go/issues/29768)
 type ErrorResponse struct {
 	XMLName    xml.Name `xml:"Error" json:"-"`
 	Code       string
@@ -51,9 +53,6 @@ type ErrorResponse struct {
 
 	// Underlying HTTP status code for the returned error
 	StatusCode int `xml:"-" json:"-"`
-
-	// Headers of the returned S3 XML error
-	Headers http.Header `xml:"-" json:"-"`
 }
 
 // ToErrorResponse - Returns parsed ErrorResponse struct from body and
@@ -177,9 +176,6 @@ func httpRespToErrorResponse(resp *http.Response, bucketName, objectName string)
 		errResp.Message = fmt.Sprintf("Region does not match, expecting region ‘%s’.", errResp.Region)
 	}
 
-	// Save headers returned in the API XML error
-	errResp.Headers = resp.Header
-
 	return errResp
 }
 

vendor/github.com/minio/minio-go/api-put-object-common.go

@@ -34,26 +34,25 @@ func isObject(reader io.Reader) (ok bool) {
 
 // Verify if reader is a generic ReaderAt
 func isReadAt(reader io.Reader) (ok bool) {
-	_, ok = reader.(io.ReaderAt)
+	var v *os.File
+	v, ok = reader.(*os.File)
 	if ok {
-		var v *os.File
+		// Stdin, Stdout and Stderr all have *os.File type
-		v, ok = reader.(*os.File)
+		// which happen to also be io.ReaderAt compatible
-		if ok {
+		// we need to add special conditions for them to
-			// Stdin, Stdout and Stderr all have *os.File type
+		// be ignored by this function.
-			// which happen to also be io.ReaderAt compatible
+		for _, f := range []string{
-			// we need to add special conditions for them to
+			"/dev/stdin",
-			// be ignored by this function.
+			"/dev/stdout",
-			for _, f := range []string{
+			"/dev/stderr",
-				"/dev/stdin",
+		} {
-				"/dev/stdout",
+			if f == v.Name() {
-				"/dev/stderr",
+				ok = false
-			} {
+				break
-				if f == v.Name() {
-					ok = false
-					break
-				}
 			}
 		}
+	} else {
+		_, ok = reader.(io.ReaderAt)
 	}
 	return
 }

vendor/github.com/minio/minio-go/api-select.go

@@ -325,7 +325,7 @@ func (s *SelectResults) start(pipeWriter *io.PipeWriter) {
 
 			switch m {
 			case errorMsg:
-				pipeWriter.CloseWithError(errors.New("Error Type of " + headers.Get("error-type") + " " + headers.Get("error-message")))
+				pipeWriter.CloseWithError(errors.New(headers.Get("error-code") + ":\"" + headers.Get("error-message") + "\""))
 				closeResponse(s.resp)
 				return
 			case commonMsg:

vendor/github.com/minio/minio-go/api-stat.go

@@ -84,6 +84,7 @@ func extractObjMetadata(header http.Header) http.Header {
 		"Content-Length",
 		"Last-Modified",
 		"Content-Type",
+		"Expires",
 	}, defaultFilterKeys...)
 	return filterHeader(header, filterKeys)
 }
@@ -170,6 +171,11 @@ func (c Client) statObject(ctx context.Context, bucketName, objectName string, o
 		contentType = "application/octet-stream"
 	}
 
+	expiryStr := resp.Header.Get("Expires")
+	var expTime time.Time
+	if t, err := time.Parse(http.TimeFormat, expiryStr); err == nil {
+		expTime = t.UTC()
+	}
 	// Save object metadata info.
 	return ObjectInfo{
 		ETag:         md5sum,
@@ -177,6 +183,7 @@ func (c Client) statObject(ctx context.Context, bucketName, objectName string, o
 		Size:         size,
 		LastModified: date,
 		ContentType:  contentType,
+		Expires:      expTime,
 		// Extract only the relevant header keys describing the object.
 		// following function filters out a list of standard set of keys
 		// which are not part of object metadata.

vendor/github.com/minio/minio-go/api.go

@@ -102,7 +102,7 @@ type Options struct {
 // Global constants.
 const (
 	libraryName    = "minio-go"
-	libraryVersion = "v6.0.14"
+	libraryVersion = "v6.0.19"
 )
 
 // User Agent should always following the below style.
@@ -295,10 +295,15 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re
 	// Save endpoint URL, user agent for future uses.
 	clnt.endpointURL = endpointURL
 
+	transport, err := DefaultTransport(secure)
+	if err != nil {
+		return nil, err
+	}
+
 	// Instantiate http client and bucket location cache.
 	clnt.httpClient = &http.Client{
 		Jar:           jar,
-		Transport:     DefaultTransport,
+		Transport:     transport,
 		CheckRedirect: clnt.redirectHeaders,
 	}
 

vendor/github.com/minio/minio-go/bucket-cache.go

@@ -18,6 +18,7 @@
 package minio
 
 import (
+	"net"
 	"net/http"
 	"net/url"
 	"path"
@@ -162,6 +163,14 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro
 
 	// Set get bucket location always as path style.
 	targetURL := c.endpointURL
+
+	// as it works in makeTargetURL method from api.go file
+	if h, p, err := net.SplitHostPort(targetURL.Host); err == nil {
+		if targetURL.Scheme == "http" && p == "80" || targetURL.Scheme == "https" && p == "443" {
+			targetURL.Host = h
+		}
+	}
+
 	targetURL.Path = path.Join(bucketName, "") + "/"
 	targetURL.RawQuery = urlValues.Encode()
 

vendor/github.com/minio/minio-go/go.mod

@@ -0,0 +1,14 @@
+module github.com/minio/minio-go
+
+require (
+	github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect
+	github.com/jtolds/gls v4.2.1+incompatible // indirect
+	github.com/mitchellh/go-homedir v1.1.0
+	github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 // indirect
+	github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c // indirect
+	golang.org/x/crypto v0.0.0-20190128193316-c7b33c32a30b
+	golang.org/x/net v0.0.0-20190213061140-3a22650c66bd
+	golang.org/x/sys v0.0.0-20190124100055-b90733256f2e // indirect
+	golang.org/x/text v0.3.0 // indirect
+	gopkg.in/ini.v1 v1.41.0
+)

vendor/github.com/minio/minio-go/go.sum

@@ -0,0 +1,20 @@
+github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=
+github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
+github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 h1:Jpy1PXuP99tXNrhbq2BaPz9B+jNAvH1JPQQpG/9GCXY=
+github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c h1:Ho+uVpkel/udgjbwB5Lktg9BtvJSh2DT0Hi6LPSyI2w=
+github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
+golang.org/x/crypto v0.0.0-20190128193316-c7b33c32a30b h1:Ib/yptP38nXZFMwqWSip+OKuMP9OkyDe3p+DssP8n9w=
+golang.org/x/crypto v0.0.0-20190128193316-c7b33c32a30b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd h1:HuTn7WObtcDo9uEEU7rEqL0jYthdXAmZ6PP+meazmaU=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/sys v0.0.0-20190124100055-b90733256f2e h1:3GIlrlVLfkoipSReOMNAgApI0ajnalyLa/EZHHca/XI=
+golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+gopkg.in/ini.v1 v1.41.0 h1:Ka3ViY6gNYSKiVy71zXBEqKplnV35ImDLVG+8uoIklE=
+gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=

vendor/github.com/minio/minio-go/hook-reader.go

@@ -17,7 +17,10 @@
 
 package minio
 
-import "io"
+import (
+	"fmt"
+	"io"
+)
 
 // hookReader hooks additional reader in the source stream. It is
 // useful for making progress bars. Second reader is appropriately
@@ -34,12 +37,23 @@ func (hr *hookReader) Seek(offset int64, whence int) (n int64, err error) {
 	// Verify for source has embedded Seeker, use it.
 	sourceSeeker, ok := hr.source.(io.Seeker)
 	if ok {
-		return sourceSeeker.Seek(offset, whence)
+		n, err = sourceSeeker.Seek(offset, whence)
+		if err != nil {
+			return 0, err
+		}
 	}
+
 	// Verify if hook has embedded Seeker, use it.
 	hookSeeker, ok := hr.hook.(io.Seeker)
 	if ok {
-		return hookSeeker.Seek(offset, whence)
+		var m int64
+		m, err = hookSeeker.Seek(offset, whence)
+		if err != nil {
+			return 0, err
+		}
+		if n != m {
+			return 0, fmt.Errorf("hook seeker seeked %d bytes, expected source %d bytes", m, n)
+		}
 	}
 	return n, nil
 }

vendor/github.com/minio/minio-go/transport.go

@@ -20,31 +20,63 @@
 package minio
 
 import (
+	"crypto/tls"
+	"crypto/x509"
 	"net"
 	"net/http"
 	"time"
+
+	"golang.org/x/net/http2"
 )
 
 // DefaultTransport - this default transport is similar to
 // http.DefaultTransport but with additional param  DisableCompression
 // is set to true to avoid decompressing content with 'gzip' encoding.
-var DefaultTransport http.RoundTripper = &http.Transport{
+var DefaultTransport = func(secure bool) (http.RoundTripper, error) {
-	Proxy: http.ProxyFromEnvironment,
+	tr := &http.Transport{
-	DialContext: (&net.Dialer{
+		Proxy: http.ProxyFromEnvironment,
-		Timeout:   30 * time.Second,
+		DialContext: (&net.Dialer{
-		KeepAlive: 30 * time.Second,
+			Timeout:   30 * time.Second,
-		DualStack: true,
+			KeepAlive: 30 * time.Second,
-	}).DialContext,
+		}).DialContext,
-	MaxIdleConns:          100,
+		MaxIdleConns:          1024,
-	MaxIdleConnsPerHost:   100,
+		MaxIdleConnsPerHost:   1024,
-	IdleConnTimeout:       90 * time.Second,
+		IdleConnTimeout:       90 * time.Second,
-	TLSHandshakeTimeout:   10 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
-	ExpectContinueTimeout: 1 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
-	// Set this value so that the underlying transport round-tripper
+		// Set this value so that the underlying transport round-tripper
-	// doesn't try to auto decode the body of objects with
+		// doesn't try to auto decode the body of objects with
-	// content-encoding set to `gzip`.
+		// content-encoding set to `gzip`.
-	//
+		//
-	// Refer:
+		// Refer:
-	//    https://golang.org/src/net/http/transport.go?h=roundTrip#L1843
+		//    https://golang.org/src/net/http/transport.go?h=roundTrip#L1843
-	DisableCompression: true,
+		DisableCompression: true,
+	}
+
+	if secure {
+		rootCAs, _ := x509.SystemCertPool()
+		if rootCAs == nil {
+			// In some systems (like Windows) system cert pool is
+			// not supported or no certificates are present on the
+			// system - so we create a new cert pool.
+			rootCAs = x509.NewCertPool()
+		}
+
+		// Keep TLS config.
+		tlsConfig := &tls.Config{
+			RootCAs: rootCAs,
+			// Can't use SSLv3 because of POODLE and BEAST
+			// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
+			// Can't use TLSv1.1 because of RC4 cipher usage
+			MinVersion: tls.VersionTLS12,
+		}
+		tr.TLSClientConfig = tlsConfig
+
+		// Because we create a custom TLSClientConfig, we have to opt-in to HTTP/2.
+		// See https://github.com/golang/go/issues/14275
+		if err := http2.ConfigureTransport(tr); err != nil {
+			return nil, err
+		}
+	}
+	return tr, nil
 }

vendor/vendor.json

@@ -632,10 +632,10 @@
 			"revisionTime": "2016-02-29T08:42:30-08:00"
 		},
 		{
-			"checksumSHA1": "Sbze8wr7T6Avtc+4K8BbcHlIx4E=",
+			"checksumSHA1": "0f4Bah8pQ9Vd8Mw3RNPh298BYpQ=",
 			"path": "github.com/minio/minio-go",
-			"revision": "a42b0e14697ffdcb4ef223384c1cac12738f574f",
+			"revision": "59af836a7e6d99cbefa093475fbde0a4552d483f",
-			"revisionTime": "2019-01-20T10:05:29Z"
+			"revisionTime": "2019-02-27T18:09:23Z"
 		},
 		{
 			"checksumSHA1": "kgQZ7iWmuKVboL2d4DUU9l5isng=",