Filter Expires header from user metadata (#7269)

Instead save it as a struct field in ObjectInfo as it is
a standard HTTP header - Fixes minio/mc#2690
This commit is contained in:
poornas 2019-02-28 11:01:25 -08:00 committed by kannappanr
parent c3ca954684
commit 2564147ab4
22 changed files with 204 additions and 55 deletions

View File

@ -87,6 +87,9 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
w.Header().Set("Content-Encoding", objInfo.ContentEncoding)
}
if !objInfo.Expires.IsZero() {
w.Header().Set("Expires", objInfo.Expires.UTC().Format(http.TimeFormat))
}
// Set all other user defined metadata.
for k, v := range objInfo.UserDefined {
if hasPrefix(k, ReservedMetadataPrefix) {

View File

@ -22,8 +22,10 @@ import (
"encoding/json"
"io"
"io/ioutil"
"net/http"
"os"
pathutil "path"
"time"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/lock"
@ -169,7 +171,15 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
} else {
objInfo.StorageClass = globalMinioDefaultStorageClass
}
var (
t time.Time
e error
)
if exp, ok := m.Meta["expires"]; ok {
if t, e = time.Parse(http.TimeFormat, exp); e == nil {
objInfo.Expires = t.UTC()
}
}
// etag/md5Sum has already been extracted. We need to
// remove to avoid it from appearing as part of
// response headers. e.g, X-Minio-* or X-Amz-*.

View File

@ -38,6 +38,9 @@ func TestFSV1MetadataObjInfo(t *testing.T) {
if objInfo.IsDir {
t.Fatal("Unexpected object info value for IsDir", objInfo.IsDir)
}
if !objInfo.Expires.IsZero() {
t.Fatal("Unexpected object info value for Expires ", objInfo.Expires)
}
}
// TestReadFSMetadata - readFSMetadata testing with a healthy and faulty disk

View File

@ -166,6 +166,7 @@ func FromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo {
ContentType: oi.ContentType,
ContentEncoding: oi.Metadata.Get("Content-Encoding"),
StorageClass: oi.StorageClass,
Expires: oi.Expires,
}
}

View File

@ -797,12 +797,22 @@ func fromGCSAttrsToObjectInfo(attrs *storage.ObjectAttrs) minio.ObjectInfo {
// All google cloud storage objects have a CRC32c hash, whereas composite objects may not have a MD5 hash
// Refer https://cloud.google.com/storage/docs/hashes-etags. Use CRC32C for ETag
metadata := make(map[string]string)
var (
expiry time.Time
e error
)
for k, v := range attrs.Metadata {
k = http.CanonicalHeaderKey(k)
// Translate the GCS custom metadata prefix
if strings.HasPrefix(k, "X-Goog-Meta-") {
k = strings.Replace(k, "X-Goog-Meta-", "X-Amz-Meta-", 1)
}
if k == "Expires" {
if expiry, e = time.Parse(http.TimeFormat, v); e == nil {
expiry = expiry.UTC()
}
continue
}
metadata[k] = v
}
if attrs.ContentType != "" {
@ -829,6 +839,7 @@ func fromGCSAttrsToObjectInfo(attrs *storage.ObjectAttrs) minio.ObjectInfo {
UserDefined: metadata,
ContentType: attrs.ContentType,
ContentEncoding: attrs.ContentEncoding,
Expires: expiry,
}
}

View File

@ -21,6 +21,7 @@ import (
"context"
"encoding/json"
"errors"
"net/http"
"time"
minio "github.com/minio/minio/cmd"
@ -83,6 +84,7 @@ func (m gwMetaV1) ToObjectInfo(bucket, object string) minio.ObjectInfo {
"Content-Length",
"Last-Modified",
"Content-Type",
"Expires",
}, defaultFilterKeys...)
objInfo := minio.ObjectInfo{
IsDir: false,
@ -100,6 +102,15 @@ func (m gwMetaV1) ToObjectInfo(bucket, object string) minio.ObjectInfo {
if sc, ok := m.Meta["x-amz-storage-class"]; ok {
objInfo.StorageClass = sc
}
var (
t time.Time
e error
)
if exp, ok := m.Meta["expires"]; ok {
if t, e = time.Parse(http.TimeFormat, exp); e == nil {
objInfo.Expires = t.UTC()
}
}
// Success.
return objInfo
}

View File

@ -96,6 +96,9 @@ type ObjectInfo struct {
// by the Content-Type header field.
ContentEncoding string
// Date and time at which the object is no longer able to be cached
Expires time.Time
// Specify object storage class
StorageClass string

View File

@ -207,8 +207,8 @@ func getCompleteMultipartMD5(ctx context.Context, parts []CompletePart) (string,
func cleanMetadata(metadata map[string]string) map[string]string {
// Remove STANDARD StorageClass
metadata = removeStandardStorageClass(metadata)
// Clean meta etag keys 'md5Sum', 'etag'.
return cleanMetadataKeys(metadata, "md5Sum", "etag")
// Clean meta etag keys 'md5Sum', 'etag', "expires".
return cleanMetadataKeys(metadata, "md5Sum", "etag", "expires")
}
// Filter X-Amz-Storage-Class field only if it is set to STANDARD.

View File

@ -21,6 +21,7 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"net/http"
"path"
"sort"
"sync"
@ -226,7 +227,16 @@ func (m xlMetaV1) ToObjectInfo(bucket, object string) ObjectInfo {
ContentType: m.Meta["content-type"],
ContentEncoding: m.Meta["content-encoding"],
}
// Update expires
var (
t time.Time
e error
)
if exp, ok := m.Meta["expires"]; ok {
if t, e = time.Parse(http.TimeFormat, exp); e == nil {
objInfo.Expires = t.UTC()
}
}
objInfo.backendType = BackendErasure
// Extract etag from metadata.

View File

@ -41,6 +41,7 @@ type ObjectInfo struct {
LastModified time.Time `json:"lastModified"` // Date and time the object was last modified.
Size int64 `json:"size"` // Size in bytes of the object.
ContentType string `json:"contentType"` // A standard MIME type describing the format of the object data.
Expires time.Time `json:"expires"` // The date and time at which the object is no longer able to be cached.
// Collection of additional metadata on the object.
// eg: x-amz-meta-*, content-encoding etc.

View File

@ -36,6 +36,8 @@ import (
*/
// ErrorResponse - Is the typed error returned by all API operations.
// ErrorResponse struct should be comparable since it is compared inside
// golang http API (https://github.com/golang/go/issues/29768)
type ErrorResponse struct {
XMLName xml.Name `xml:"Error" json:"-"`
Code string
@ -51,9 +53,6 @@ type ErrorResponse struct {
// Underlying HTTP status code for the returned error
StatusCode int `xml:"-" json:"-"`
// Headers of the returned S3 XML error
Headers http.Header `xml:"-" json:"-"`
}
// ToErrorResponse - Returns parsed ErrorResponse struct from body and
@ -177,9 +176,6 @@ func httpRespToErrorResponse(resp *http.Response, bucketName, objectName string)
errResp.Message = fmt.Sprintf("Region does not match, expecting region %s.", errResp.Region)
}
// Save headers returned in the API XML error
errResp.Headers = resp.Header
return errResp
}

View File

@ -34,26 +34,25 @@ func isObject(reader io.Reader) (ok bool) {
// Verify if reader is a generic ReaderAt
func isReadAt(reader io.Reader) (ok bool) {
_, ok = reader.(io.ReaderAt)
var v *os.File
v, ok = reader.(*os.File)
if ok {
var v *os.File
v, ok = reader.(*os.File)
if ok {
// Stdin, Stdout and Stderr all have *os.File type
// which happen to also be io.ReaderAt compatible
// we need to add special conditions for them to
// be ignored by this function.
for _, f := range []string{
"/dev/stdin",
"/dev/stdout",
"/dev/stderr",
} {
if f == v.Name() {
ok = false
break
}
// Stdin, Stdout and Stderr all have *os.File type
// which happen to also be io.ReaderAt compatible
// we need to add special conditions for them to
// be ignored by this function.
for _, f := range []string{
"/dev/stdin",
"/dev/stdout",
"/dev/stderr",
} {
if f == v.Name() {
ok = false
break
}
}
} else {
_, ok = reader.(io.ReaderAt)
}
return
}

View File

@ -325,7 +325,7 @@ func (s *SelectResults) start(pipeWriter *io.PipeWriter) {
switch m {
case errorMsg:
pipeWriter.CloseWithError(errors.New("Error Type of " + headers.Get("error-type") + " " + headers.Get("error-message")))
pipeWriter.CloseWithError(errors.New(headers.Get("error-code") + ":\"" + headers.Get("error-message") + "\""))
closeResponse(s.resp)
return
case commonMsg:

View File

@ -84,6 +84,7 @@ func extractObjMetadata(header http.Header) http.Header {
"Content-Length",
"Last-Modified",
"Content-Type",
"Expires",
}, defaultFilterKeys...)
return filterHeader(header, filterKeys)
}
@ -170,6 +171,11 @@ func (c Client) statObject(ctx context.Context, bucketName, objectName string, o
contentType = "application/octet-stream"
}
expiryStr := resp.Header.Get("Expires")
var expTime time.Time
if t, err := time.Parse(http.TimeFormat, expiryStr); err == nil {
expTime = t.UTC()
}
// Save object metadata info.
return ObjectInfo{
ETag: md5sum,
@ -177,6 +183,7 @@ func (c Client) statObject(ctx context.Context, bucketName, objectName string, o
Size: size,
LastModified: date,
ContentType: contentType,
Expires: expTime,
// Extract only the relevant header keys describing the object.
// following function filters out a list of standard set of keys
// which are not part of object metadata.

View File

@ -102,7 +102,7 @@ type Options struct {
// Global constants.
const (
libraryName = "minio-go"
libraryVersion = "v6.0.14"
libraryVersion = "v6.0.19"
)
// User Agent should always following the below style.
@ -295,10 +295,15 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re
// Save endpoint URL, user agent for future uses.
clnt.endpointURL = endpointURL
transport, err := DefaultTransport(secure)
if err != nil {
return nil, err
}
// Instantiate http client and bucket location cache.
clnt.httpClient = &http.Client{
Jar: jar,
Transport: DefaultTransport,
Transport: transport,
CheckRedirect: clnt.redirectHeaders,
}

View File

@ -18,6 +18,7 @@
package minio
import (
"net"
"net/http"
"net/url"
"path"
@ -162,6 +163,14 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro
// Set get bucket location always as path style.
targetURL := c.endpointURL
// as it works in makeTargetURL method from api.go file
if h, p, err := net.SplitHostPort(targetURL.Host); err == nil {
if targetURL.Scheme == "http" && p == "80" || targetURL.Scheme == "https" && p == "443" {
targetURL.Host = h
}
}
targetURL.Path = path.Join(bucketName, "") + "/"
targetURL.RawQuery = urlValues.Encode()

14
vendor/github.com/minio/minio-go/go.mod generated vendored Normal file
View File

@ -0,0 +1,14 @@
module github.com/minio/minio-go
require (
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect
github.com/jtolds/gls v4.2.1+incompatible // indirect
github.com/mitchellh/go-homedir v1.1.0
github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 // indirect
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c // indirect
golang.org/x/crypto v0.0.0-20190128193316-c7b33c32a30b
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd
golang.org/x/sys v0.0.0-20190124100055-b90733256f2e // indirect
golang.org/x/text v0.3.0 // indirect
gopkg.in/ini.v1 v1.41.0
)

20
vendor/github.com/minio/minio-go/go.sum generated vendored Normal file
View File

@ -0,0 +1,20 @@
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 h1:Jpy1PXuP99tXNrhbq2BaPz9B+jNAvH1JPQQpG/9GCXY=
github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c h1:Ho+uVpkel/udgjbwB5Lktg9BtvJSh2DT0Hi6LPSyI2w=
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
golang.org/x/crypto v0.0.0-20190128193316-c7b33c32a30b h1:Ib/yptP38nXZFMwqWSip+OKuMP9OkyDe3p+DssP8n9w=
golang.org/x/crypto v0.0.0-20190128193316-c7b33c32a30b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd h1:HuTn7WObtcDo9uEEU7rEqL0jYthdXAmZ6PP+meazmaU=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/sys v0.0.0-20190124100055-b90733256f2e h1:3GIlrlVLfkoipSReOMNAgApI0ajnalyLa/EZHHca/XI=
golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
gopkg.in/ini.v1 v1.41.0 h1:Ka3ViY6gNYSKiVy71zXBEqKplnV35ImDLVG+8uoIklE=
gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=

View File

@ -17,7 +17,10 @@
package minio
import "io"
import (
"fmt"
"io"
)
// hookReader hooks additional reader in the source stream. It is
// useful for making progress bars. Second reader is appropriately
@ -34,12 +37,23 @@ func (hr *hookReader) Seek(offset int64, whence int) (n int64, err error) {
// Verify for source has embedded Seeker, use it.
sourceSeeker, ok := hr.source.(io.Seeker)
if ok {
return sourceSeeker.Seek(offset, whence)
n, err = sourceSeeker.Seek(offset, whence)
if err != nil {
return 0, err
}
}
// Verify if hook has embedded Seeker, use it.
hookSeeker, ok := hr.hook.(io.Seeker)
if ok {
return hookSeeker.Seek(offset, whence)
var m int64
m, err = hookSeeker.Seek(offset, whence)
if err != nil {
return 0, err
}
if n != m {
return 0, fmt.Errorf("hook seeker seeked %d bytes, expected source %d bytes", m, n)
}
}
return n, nil
}

View File

@ -20,31 +20,63 @@
package minio
import (
"crypto/tls"
"crypto/x509"
"net"
"net/http"
"time"
"golang.org/x/net/http2"
)
// DefaultTransport - this default transport is similar to
// http.DefaultTransport but with additional param DisableCompression
// is set to true to avoid decompressing content with 'gzip' encoding.
var DefaultTransport http.RoundTripper = &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).DialContext,
MaxIdleConns: 100,
MaxIdleConnsPerHost: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
// Set this value so that the underlying transport round-tripper
// doesn't try to auto decode the body of objects with
// content-encoding set to `gzip`.
//
// Refer:
// https://golang.org/src/net/http/transport.go?h=roundTrip#L1843
DisableCompression: true,
var DefaultTransport = func(secure bool) (http.RoundTripper, error) {
tr := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
MaxIdleConns: 1024,
MaxIdleConnsPerHost: 1024,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
// Set this value so that the underlying transport round-tripper
// doesn't try to auto decode the body of objects with
// content-encoding set to `gzip`.
//
// Refer:
// https://golang.org/src/net/http/transport.go?h=roundTrip#L1843
DisableCompression: true,
}
if secure {
rootCAs, _ := x509.SystemCertPool()
if rootCAs == nil {
// In some systems (like Windows) system cert pool is
// not supported or no certificates are present on the
// system - so we create a new cert pool.
rootCAs = x509.NewCertPool()
}
// Keep TLS config.
tlsConfig := &tls.Config{
RootCAs: rootCAs,
// Can't use SSLv3 because of POODLE and BEAST
// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
// Can't use TLSv1.1 because of RC4 cipher usage
MinVersion: tls.VersionTLS12,
}
tr.TLSClientConfig = tlsConfig
// Because we create a custom TLSClientConfig, we have to opt-in to HTTP/2.
// See https://github.com/golang/go/issues/14275
if err := http2.ConfigureTransport(tr); err != nil {
return nil, err
}
}
return tr, nil
}

BIN
vendor/github.com/minio/minio-go/validator generated vendored Executable file

Binary file not shown.

6
vendor/vendor.json vendored
View File

@ -632,10 +632,10 @@
"revisionTime": "2016-02-29T08:42:30-08:00"
},
{
"checksumSHA1": "Sbze8wr7T6Avtc+4K8BbcHlIx4E=",
"checksumSHA1": "0f4Bah8pQ9Vd8Mw3RNPh298BYpQ=",
"path": "github.com/minio/minio-go",
"revision": "a42b0e14697ffdcb4ef223384c1cac12738f574f",
"revisionTime": "2019-01-20T10:05:29Z"
"revision": "59af836a7e6d99cbefa093475fbde0a4552d483f",
"revisionTime": "2019-02-27T18:09:23Z"
},
{
"checksumSHA1": "kgQZ7iWmuKVboL2d4DUU9l5isng=",