diff --git a/cmd/gateway-s3-anonymous.go b/cmd/gateway-s3-anonymous.go index 1bfce8dc6..2237774df 100644 --- a/cmd/gateway-s3-anonymous.go +++ b/cmd/gateway-s3-anonymous.go @@ -55,11 +55,11 @@ func (l *s3Objects) AnonPutObject(bucket string, object string, size int64, data // AnonGetObject - Get object anonymously func (l *s3Objects) AnonGetObject(bucket string, key string, startOffset int64, length int64, writer io.Writer) error { - r := minio.NewGetReqHeaders() - if err := r.SetRange(startOffset, startOffset+length-1); err != nil { + opts := minio.GetObjectOptions{} + if err := opts.SetRange(startOffset, startOffset+length-1); err != nil { return s3ToObjectError(traceError(err), bucket, key) } - object, _, err := l.anonClient.GetObject(bucket, key, r) + object, _, err := l.anonClient.GetObject(bucket, key, opts) if err != nil { return s3ToObjectError(traceError(err), bucket, key) } @@ -75,8 +75,7 @@ func (l *s3Objects) AnonGetObject(bucket string, key string, startOffset int64, // AnonGetObjectInfo - Get object info anonymously func (l *s3Objects) AnonGetObjectInfo(bucket string, object string) (objInfo ObjectInfo, e error) { - r := minio.NewHeadReqHeaders() - oi, err := l.anonClient.StatObject(bucket, object, r) + oi, err := l.anonClient.StatObject(bucket, object, minio.StatObjectOptions{}) if err != nil { return objInfo, s3ToObjectError(traceError(err), bucket, object) } diff --git a/cmd/gateway-s3.go b/cmd/gateway-s3.go index 591c3d6ee..454fcae98 100644 --- a/cmd/gateway-s3.go +++ b/cmd/gateway-s3.go @@ -19,7 +19,6 @@ package cmd import ( "io" "net/http" - "strings" "encoding/hex" @@ -290,22 +289,20 @@ func fromMinioClientListBucketResult(bucket string, result minio.ListBucketResul // startOffset indicates the starting read location of the object. // length indicates the total length of the object. func (l *s3Objects) GetObject(bucket string, key string, startOffset int64, length int64, writer io.Writer) error { - r := minio.NewGetReqHeaders() - if length < 0 && length != -1 { return s3ToObjectError(traceError(errInvalidArgument), bucket, key) } + opts := minio.GetObjectOptions{} if startOffset >= 0 && length >= 0 { - if err := r.SetRange(startOffset, startOffset+length-1); err != nil { + if err := opts.SetRange(startOffset, startOffset+length-1); err != nil { return s3ToObjectError(traceError(err), bucket, key) } } - object, _, err := l.Client.GetObject(bucket, key, r) + object, _, err := l.Client.GetObject(bucket, key, opts) if err != nil { return s3ToObjectError(traceError(err), bucket, key) } - defer object.Close() if _, err := io.Copy(writer, object); err != nil { @@ -333,8 +330,7 @@ func fromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo { // GetObjectInfo reads object info and replies back ObjectInfo func (l *s3Objects) GetObjectInfo(bucket string, object string) (objInfo ObjectInfo, err error) { - r := minio.NewHeadReqHeaders() - oi, err := l.Client.StatObject(bucket, object, r) + oi, err := l.Client.StatObject(bucket, object, minio.StatObjectOptions{}) if err != nil { return ObjectInfo{}, s3ToObjectError(traceError(err), bucket, object) } @@ -361,35 +357,17 @@ func (l *s3Objects) PutObject(bucket string, object string, data *HashReader, me return fromMinioClientObjectInfo(bucket, oi), nil } -// CopyObject copies a blob from source container to destination container. +// CopyObject copies an object from source bucket to a destination bucket. func (l *s3Objects) CopyObject(srcBucket string, srcObject string, dstBucket string, dstObject string, metadata map[string]string) (objInfo ObjectInfo, err error) { - // Source object - src := minio.NewSourceInfo(srcBucket, srcObject, nil) - - // Destination object - var xamzMeta = map[string]string{} - for key := range metadata { - for _, prefix := range userMetadataKeyPrefixes { - if strings.HasPrefix(key, prefix) { - xamzMeta[key] = metadata[key] - } - } - } - dst, err := minio.NewDestinationInfo(dstBucket, dstObject, nil, xamzMeta) - if err != nil { - return objInfo, s3ToObjectError(traceError(err), dstBucket, dstObject) - } - - if err = l.Client.CopyObject(dst, src); err != nil { + // Set this header such that following CopyObject() always sets the right metadata on the destination. + // metadata input is already a trickled down value from interpreting x-amz-metadata-directive at + // handler layer. So what we have right now is supposed to be applied on the destination object anyways. + // So preserve it by adding "REPLACE" directive to save all the metadata set by CopyObject API. + metadata["x-amz-metadata-directive"] = "REPLACE" + if _, err = l.Client.CopyObject(srcBucket, srcObject, dstBucket, dstObject, metadata); err != nil { return objInfo, s3ToObjectError(traceError(err), srcBucket, srcObject) } - - oi, err := l.GetObjectInfo(dstBucket, dstObject) - if err != nil { - return objInfo, s3ToObjectError(traceError(err), dstBucket, dstObject) - } - - return oi, nil + return l.GetObjectInfo(dstBucket, dstObject) } // DeleteObject deletes a blob in bucket diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 0ec67c795..856c594f8 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -398,7 +398,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re if err != nil { errorIf(err, "found invalid http request header") writeErrorResponse(w, ErrInternalError, r.URL) + return } + // Check if x-amz-metadata-directive was not set to REPLACE and source, // desination are same objects. if !isMetadataReplace(r.Header) && cpSrcDstSame { diff --git a/vendor/github.com/minio/minio-go/README.md b/vendor/github.com/minio/minio-go/README.md index b37fc9a7f..f4f9806ec 100644 --- a/vendor/github.com/minio/minio-go/README.md +++ b/vendor/github.com/minio/minio-go/README.md @@ -224,6 +224,7 @@ The full API Reference is available here. ### Full Examples : Encrypted Object Operations * [put-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/put-encrypted-object.go) * [get-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/get-encrypted-object.go) +* [fput-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/fputencrypted-object.go) ### Full Examples : Presigned Operations * [presignedgetobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedgetobject.go) diff --git a/vendor/github.com/minio/minio-go/api-compose-object.go b/vendor/github.com/minio/minio-go/api-compose-object.go index 2a11cb4c1..7763026db 100644 --- a/vendor/github.com/minio/minio-go/api-compose-object.go +++ b/vendor/github.com/minio/minio-go/api-compose-object.go @@ -244,11 +244,11 @@ func (s *SourceInfo) getProps(c Client) (size int64, etag string, userMeta map[s // Get object info - need size and etag here. Also, decryption // headers are added to the stat request if given. var objInfo ObjectInfo - rh := NewGetReqHeaders() + opts := StatObjectOptions{} for k, v := range s.decryptKey.getSSEHeaders(false) { - rh.Set(k, v) + opts.Set(k, v) } - objInfo, err = c.statObject(s.bucket, s.object, rh) + objInfo, err = c.statObject(s.bucket, s.object, opts) if err != nil { err = fmt.Errorf("Could not stat object - %s/%s: %v", s.bucket, s.object, err) } else { @@ -266,6 +266,51 @@ func (s *SourceInfo) getProps(c Client) (size int64, etag string, userMeta map[s return } +// Low level implementation of CopyObject API, supports only upto 5GiB worth of copy. +func (c Client) copyObjectDo(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, + metadata map[string]string) (ObjectInfo, error) { + + // Build headers. + headers := make(http.Header) + + // Set all the metadata headers. + for k, v := range metadata { + headers.Set(k, v) + } + + // Set the source header + headers.Set("x-amz-copy-source", s3utils.EncodePath(srcBucket+"/"+srcObject)) + + // Send upload-part-copy request + resp, err := c.executeMethod(ctx, "PUT", requestMetadata{ + bucketName: destBucket, + objectName: destObject, + customHeader: headers, + }) + defer closeResponse(resp) + if err != nil { + return ObjectInfo{}, err + } + + // Check if we got an error response. + if resp.StatusCode != http.StatusOK { + return ObjectInfo{}, httpRespToErrorResponse(resp, srcBucket, srcObject) + } + + cpObjRes := copyObjectResult{} + err = xmlDecoder(resp.Body, &cpObjRes) + if err != nil { + return ObjectInfo{}, err + } + + objInfo := ObjectInfo{ + Key: destObject, + ETag: strings.Trim(cpObjRes.ETag, "\""), + LastModified: cpObjRes.LastModified, + } + return objInfo, nil +} + // uploadPartCopy - helper function to create a part in a multipart // upload via an upload-part-copy request // https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPartCopy.html diff --git a/vendor/github.com/minio/minio-go/api-get-object-context.go b/vendor/github.com/minio/minio-go/api-get-object-context.go index 1ff6e06df..f9ed29007 100644 --- a/vendor/github.com/minio/minio-go/api-get-object-context.go +++ b/vendor/github.com/minio/minio-go/api-get-object-context.go @@ -19,6 +19,7 @@ package minio import "context" // GetObjectWithContext - returns an seekable, readable object. -func (c Client) GetObjectWithContext(ctx context.Context, bucketName, objectName string) (*Object, error) { - return c.getObjectWithContext(ctx, bucketName, objectName) +// The options can be used to specify the GET request further. +func (c Client) GetObjectWithContext(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (*Object, error) { + return c.getObjectWithContext(ctx, bucketName, objectName, opts) } diff --git a/vendor/github.com/minio/minio-go/api-get-object-file.go b/vendor/github.com/minio/minio-go/api-get-object-file.go index fff6d1f99..5a3e728dd 100644 --- a/vendor/github.com/minio/minio-go/api-get-object-file.go +++ b/vendor/github.com/minio/minio-go/api-get-object-file.go @@ -21,23 +21,34 @@ import ( "os" "path/filepath" + "github.com/minio/minio-go/pkg/encrypt" + "context" "github.com/minio/minio-go/pkg/s3utils" ) // FGetObjectWithContext - download contents of an object to a local file. -func (c Client) FGetObjectWithContext(ctx context.Context, bucketName, objectName, filePath string) error { - return c.fGetObjectWithContext(ctx, bucketName, objectName, filePath) +// The options can be used to specify the GET request further. +func (c Client) FGetObjectWithContext(ctx context.Context, bucketName, objectName, filePath string, opts GetObjectOptions) error { + return c.fGetObjectWithContext(ctx, bucketName, objectName, filePath, opts) } // FGetObject - download contents of an object to a local file. -func (c Client) FGetObject(bucketName, objectName, filePath string) error { - return c.fGetObjectWithContext(context.Background(), bucketName, objectName, filePath) +func (c Client) FGetObject(bucketName, objectName, filePath string, opts GetObjectOptions) error { + return c.fGetObjectWithContext(context.Background(), bucketName, objectName, filePath, opts) +} + +// FGetEncryptedObject - Decrypt and store an object at filePath. +func (c Client) FGetEncryptedObject(bucketName, objectName, filePath string, materials encrypt.Materials) error { + if materials == nil { + return ErrInvalidArgument("Unable to recognize empty encryption properties") + } + return c.FGetObject(bucketName, objectName, filePath, GetObjectOptions{Materials: materials}) } // fGetObjectWithContext - fgetObject wrapper function with context -func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectName, filePath string) error { +func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectName, filePath string, opts GetObjectOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -72,7 +83,7 @@ func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectNam } // Gather md5sum. - objectStat, err := c.StatObject(bucketName, objectName) + objectStat, err := c.StatObject(bucketName, objectName, StatObjectOptions{opts}) if err != nil { return err } @@ -94,13 +105,12 @@ func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectNam // Initialize get object request headers to set the // appropriate range offsets to read from. - reqHeaders := NewGetReqHeaders() if st.Size() > 0 { - reqHeaders.SetRange(st.Size(), 0) + opts.SetRange(st.Size(), 0) } // Seek to current position for incoming reader. - objectReader, objectStat, err := c.getObject(ctx, bucketName, objectName, reqHeaders) + objectReader, objectStat, err := c.getObject(ctx, bucketName, objectName, opts) if err != nil { return err } diff --git a/vendor/github.com/minio/minio-go/api-get-object.go b/vendor/github.com/minio/minio-go/api-get-object.go index 02b3c3ff6..c6904e3f8 100644 --- a/vendor/github.com/minio/minio-go/api-get-object.go +++ b/vendor/github.com/minio/minio-go/api-get-object.go @@ -37,32 +37,16 @@ func (c Client) GetEncryptedObject(bucketName, objectName string, encryptMateria return nil, ErrInvalidArgument("Unable to recognize empty encryption properties") } - // Fetch encrypted object - encReader, err := c.GetObject(bucketName, objectName) - if err != nil { - return nil, err - } - // Stat object to get its encryption metadata - st, err := encReader.Stat() - if err != nil { - return nil, err - } - - // Setup object for decrytion, object is transparently - // decrypted as the consumer starts reading. - encryptMaterials.SetupDecryptMode(encReader, st.Metadata.Get(amzHeaderIV), st.Metadata.Get(amzHeaderKey)) - - // Success. - return encryptMaterials, nil + return c.GetObject(bucketName, objectName, GetObjectOptions{Materials: encryptMaterials}) } // GetObject - returns an seekable, readable object. -func (c Client) GetObject(bucketName, objectName string) (*Object, error) { - return c.getObjectWithContext(context.Background(), bucketName, objectName) +func (c Client) GetObject(bucketName, objectName string, opts GetObjectOptions) (*Object, error) { + return c.getObjectWithContext(context.Background(), bucketName, objectName, opts) } // GetObject wrapper function that accepts a request context -func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName string) (*Object, error) { +func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (*Object, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return nil, err @@ -108,34 +92,26 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName if req.isFirstReq { // First request is a Read/ReadAt. if req.isReadOp { - reqHeaders := NewGetReqHeaders() // Differentiate between wanting the whole object and just a range. if req.isReadAt { // If this is a ReadAt request only get the specified range. // Range is set with respect to the offset and length of the buffer requested. // Do not set objectInfo from the first readAt request because it will not get // the whole object. - reqHeaders.SetRange(req.Offset, req.Offset+int64(len(req.Buffer))-1) - httpReader, objectInfo, err = c.getObject(ctx, bucketName, objectName, reqHeaders) - } else { - if req.Offset > 0 { - reqHeaders.SetRange(req.Offset, 0) - } - - // First request is a Read request. - httpReader, objectInfo, err = c.getObject(ctx, bucketName, objectName, reqHeaders) + opts.SetRange(req.Offset, req.Offset+int64(len(req.Buffer))-1) + } else if req.Offset > 0 { + opts.SetRange(req.Offset, 0) } + httpReader, objectInfo, err = c.getObject(ctx, bucketName, objectName, opts) if err != nil { - resCh <- getResponse{ - Error: err, - } + resCh <- getResponse{Error: err} return } etag = objectInfo.ETag // Read at least firstReq.Buffer bytes, if not we have // reached our EOF. size, err := io.ReadFull(httpReader, req.Buffer) - if err == io.ErrUnexpectedEOF { + if size > 0 && err == io.ErrUnexpectedEOF { // If an EOF happens after reading some but not // all the bytes ReadFull returns ErrUnexpectedEOF err = io.EOF @@ -150,7 +126,7 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName } else { // First request is a Stat or Seek call. // Only need to run a StatObject until an actual Read or ReadAt request comes through. - objectInfo, err = c.StatObject(bucketName, objectName) + objectInfo, err = c.statObject(bucketName, objectName, StatObjectOptions{opts}) if err != nil { resCh <- getResponse{ Error: err, @@ -165,11 +141,10 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName } } } else if req.settingObjectInfo { // Request is just to get objectInfo. - reqHeaders := NewGetReqHeaders() if etag != "" { - reqHeaders.SetMatchETag(etag) + opts.SetMatchETag(etag) } - objectInfo, err := c.statObject(bucketName, objectName, reqHeaders) + objectInfo, err := c.statObject(bucketName, objectName, StatObjectOptions{opts}) if err != nil { resCh <- getResponse{ Error: err, @@ -189,9 +164,8 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName // new ones when they haven't been already. // All readAt requests are new requests. if req.DidOffsetChange || !req.beenRead { - reqHeaders := NewGetReqHeaders() if etag != "" { - reqHeaders.SetMatchETag(etag) + opts.SetMatchETag(etag) } if httpReader != nil { // Close previously opened http reader. @@ -200,16 +174,11 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName // If this request is a readAt only get the specified range. if req.isReadAt { // Range is set with respect to the offset and length of the buffer requested. - reqHeaders.SetRange(req.Offset, req.Offset+int64(len(req.Buffer))-1) - httpReader, _, err = c.getObject(ctx, bucketName, objectName, reqHeaders) - } else { - // Range is set with respect to the offset. - if req.Offset > 0 { - reqHeaders.SetRange(req.Offset, 0) - } - - httpReader, objectInfo, err = c.getObject(ctx, bucketName, objectName, reqHeaders) + opts.SetRange(req.Offset, req.Offset+int64(len(req.Buffer))-1) + } else if req.Offset > 0 { // Range is set with respect to the offset. + opts.SetRange(req.Offset, 0) } + httpReader, objectInfo, err = c.getObject(ctx, bucketName, objectName, opts) if err != nil { resCh <- getResponse{ Error: err, @@ -632,7 +601,7 @@ func newObject(reqCh chan<- getRequest, resCh <-chan getResponse, doneCh chan<- // // For more information about the HTTP Range header. // go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35. -func (c Client) getObject(ctx context.Context, bucketName, objectName string, reqHeaders RequestHeaders) (io.ReadCloser, ObjectInfo, error) { +func (c Client) getObject(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, error) { // Validate input arguments. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return nil, ObjectInfo{}, err @@ -641,17 +610,11 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, re return nil, ObjectInfo{}, err } - // Set all the necessary reqHeaders. - customHeader := make(http.Header) - for key, value := range reqHeaders.Header { - customHeader[key] = value - } - // Execute GET on objectName. resp, err := c.executeMethod(ctx, "GET", requestMetadata{ bucketName: bucketName, objectName: objectName, - customHeader: customHeader, + customHeader: opts.Header(), contentSHA256Bytes: emptySHA256, }) if err != nil { @@ -698,6 +661,15 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, re Metadata: extractObjMetadata(resp.Header), } + reader := resp.Body + if opts.Materials != nil { + err = opts.Materials.SetupDecryptMode(reader, objectStat.Metadata.Get(amzHeaderIV), objectStat.Metadata.Get(amzHeaderKey)) + if err != nil { + return nil, ObjectInfo{}, err + } + reader = opts.Materials + } + // do not close body here, caller will close - return resp.Body, objectStat, nil + return reader, objectStat, nil } diff --git a/vendor/github.com/minio/minio-go/request-headers.go b/vendor/github.com/minio/minio-go/api-get-options.go similarity index 55% rename from vendor/github.com/minio/minio-go/request-headers.go rename to vendor/github.com/minio/minio-go/api-get-options.go index 76c87202d..64279f754 100644 --- a/vendor/github.com/minio/minio-go/request-headers.go +++ b/vendor/github.com/minio/minio-go/api-get-options.go @@ -20,80 +20,94 @@ import ( "fmt" "net/http" "time" + + "github.com/minio/minio-go/pkg/encrypt" ) -// RequestHeaders - implement methods for setting special -// request headers for GET, HEAD object operations. -// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html -type RequestHeaders struct { - http.Header +// GetObjectOptions are used to specify additional headers or options +// during GET requests. +type GetObjectOptions struct { + headers map[string]string + + Materials encrypt.Materials } -// NewGetReqHeaders - initializes a new request headers for GET request. -func NewGetReqHeaders() RequestHeaders { - return RequestHeaders{ - Header: make(http.Header), - } +// StatObjectOptions are used to specify additional headers or options +// during GET info/stat requests. +type StatObjectOptions struct { + GetObjectOptions } -// NewHeadReqHeaders - initializes a new request headers for HEAD request. -func NewHeadReqHeaders() RequestHeaders { - return RequestHeaders{ - Header: make(http.Header), +// Header returns the http.Header representation of the GET options. +func (o GetObjectOptions) Header() http.Header { + headers := make(http.Header, len(o.headers)) + for k, v := range o.headers { + headers.Set(k, v) } + return headers +} + +// Set adds a key value pair to the options. The +// key-value pair will be part of the HTTP GET request +// headers. +func (o *GetObjectOptions) Set(key, value string) { + if o.headers == nil { + o.headers = make(map[string]string) + } + o.headers[http.CanonicalHeaderKey(key)] = value } // SetMatchETag - set match etag. -func (c RequestHeaders) SetMatchETag(etag string) error { +func (o *GetObjectOptions) SetMatchETag(etag string) error { if etag == "" { return ErrInvalidArgument("ETag cannot be empty.") } - c.Set("If-Match", "\""+etag+"\"") + o.Set("If-Match", "\""+etag+"\"") return nil } // SetMatchETagExcept - set match etag except. -func (c RequestHeaders) SetMatchETagExcept(etag string) error { +func (o *GetObjectOptions) SetMatchETagExcept(etag string) error { if etag == "" { return ErrInvalidArgument("ETag cannot be empty.") } - c.Set("If-None-Match", "\""+etag+"\"") + o.Set("If-None-Match", "\""+etag+"\"") return nil } // SetUnmodified - set unmodified time since. -func (c RequestHeaders) SetUnmodified(modTime time.Time) error { +func (o *GetObjectOptions) SetUnmodified(modTime time.Time) error { if modTime.IsZero() { return ErrInvalidArgument("Modified since cannot be empty.") } - c.Set("If-Unmodified-Since", modTime.Format(http.TimeFormat)) + o.Set("If-Unmodified-Since", modTime.Format(http.TimeFormat)) return nil } // SetModified - set modified time since. -func (c RequestHeaders) SetModified(modTime time.Time) error { +func (o *GetObjectOptions) SetModified(modTime time.Time) error { if modTime.IsZero() { return ErrInvalidArgument("Modified since cannot be empty.") } - c.Set("If-Modified-Since", modTime.Format(http.TimeFormat)) + o.Set("If-Modified-Since", modTime.Format(http.TimeFormat)) return nil } // SetRange - set the start and end offset of the object to be read. // See https://tools.ietf.org/html/rfc7233#section-3.1 for reference. -func (c RequestHeaders) SetRange(start, end int64) error { +func (o *GetObjectOptions) SetRange(start, end int64) error { switch { case start == 0 && end < 0: // Read last '-end' bytes. `bytes=-N`. - c.Set("Range", fmt.Sprintf("bytes=%d", end)) + o.Set("Range", fmt.Sprintf("bytes=%d", end)) case 0 < start && end == 0: // Read everything starting from offset // 'start'. `bytes=N-`. - c.Set("Range", fmt.Sprintf("bytes=%d-", start)) + o.Set("Range", fmt.Sprintf("bytes=%d-", start)) case 0 <= start && start <= end: // Read everything starting at 'start' till the // 'end'. `bytes=N-M` - c.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end)) + o.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end)) default: // All other cases such as // bytes=-3- diff --git a/vendor/github.com/minio/minio-go/api-s3-datatypes.go b/vendor/github.com/minio/minio-go/api-s3-datatypes.go index 4b297407b..7193a8ae6 100644 --- a/vendor/github.com/minio/minio-go/api-s3-datatypes.go +++ b/vendor/github.com/minio/minio-go/api-s3-datatypes.go @@ -128,7 +128,7 @@ type initiator struct { // copyObjectResult container for copy object response. type copyObjectResult struct { ETag string - LastModified string // time string format "2006-01-02T15:04:05.000Z" + LastModified time.Time // time string format "2006-01-02T15:04:05.000Z" } // ObjectPart container for particular part of an object. diff --git a/vendor/github.com/minio/minio-go/api-stat.go b/vendor/github.com/minio/minio-go/api-stat.go index cb3017f1d..d9a4cf51a 100644 --- a/vendor/github.com/minio/minio-go/api-stat.go +++ b/vendor/github.com/minio/minio-go/api-stat.go @@ -81,7 +81,7 @@ func extractObjMetadata(header http.Header) http.Header { } // StatObject verifies if object exists and you have permission to access. -func (c Client) StatObject(bucketName, objectName string) (ObjectInfo, error) { +func (c Client) StatObject(bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ObjectInfo{}, err @@ -89,12 +89,11 @@ func (c Client) StatObject(bucketName, objectName string) (ObjectInfo, error) { if err := s3utils.CheckValidObjectName(objectName); err != nil { return ObjectInfo{}, err } - reqHeaders := NewHeadReqHeaders() - return c.statObject(bucketName, objectName, reqHeaders) + return c.statObject(bucketName, objectName, opts) } // Lower level API for statObject supporting pre-conditions and range headers. -func (c Client) statObject(bucketName, objectName string, reqHeaders RequestHeaders) (ObjectInfo, error) { +func (c Client) statObject(bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ObjectInfo{}, err @@ -103,17 +102,12 @@ func (c Client) statObject(bucketName, objectName string, reqHeaders RequestHead return ObjectInfo{}, err } - customHeader := make(http.Header) - for k, v := range reqHeaders.Header { - customHeader[k] = v - } - // Execute HEAD on objectName. resp, err := c.executeMethod(context.Background(), "HEAD", requestMetadata{ bucketName: bucketName, objectName: objectName, contentSHA256Bytes: emptySHA256, - customHeader: customHeader, + customHeader: opts.Header(), }) defer closeResponse(resp) if err != nil { diff --git a/vendor/github.com/minio/minio-go/core.go b/vendor/github.com/minio/minio-go/core.go index d94cf3214..1c88d0df2 100644 --- a/vendor/github.com/minio/minio-go/core.go +++ b/vendor/github.com/minio/minio-go/core.go @@ -19,6 +19,7 @@ package minio import ( "context" "io" + "strings" "github.com/minio/minio-go/pkg/policy" ) @@ -53,9 +54,30 @@ func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, return c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, maxkeys) } +// CopyObject - copies an object from source object to destination object on server side. +func (c Core) CopyObject(sourceBucket, sourceObject, destBucket, destObject string, metadata map[string]string) (ObjectInfo, error) { + return c.copyObjectDo(context.Background(), sourceBucket, sourceObject, destBucket, destObject, metadata) +} + // PutObject - Upload object. Uploads using single PUT call. func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Sum, sha256Sum []byte, metadata map[string]string) (ObjectInfo, error) { - return c.putObjectDo(context.Background(), bucket, object, data, md5Sum, sha256Sum, size, PutObjectOptions{UserMetadata: metadata}) + opts := PutObjectOptions{} + m := make(map[string]string) + for k, v := range metadata { + if strings.ToLower(k) == "content-encoding" { + opts.ContentEncoding = v + } else if strings.ToLower(k) == "content-disposition" { + opts.ContentDisposition = v + } else if strings.ToLower(k) == "content-type" { + opts.ContentType = v + } else if strings.ToLower(k) == "cache-control" { + opts.CacheControl = v + } else { + m[k] = metadata[k] + } + } + opts.UserMetadata = m + return c.putObjectDo(context.Background(), bucket, object, data, md5Sum, sha256Sum, size, opts) } // NewMultipartUpload - Initiates new multipart upload and returns the new uploadID. @@ -111,12 +133,12 @@ func (c Core) PutBucketPolicy(bucket string, bucketPolicy policy.BucketAccessPol // GetObject is a lower level API implemented to support reading // partial objects and also downloading objects with special conditions // matching etag, modtime etc. -func (c Core) GetObject(bucketName, objectName string, reqHeaders RequestHeaders) (io.ReadCloser, ObjectInfo, error) { - return c.getObject(context.Background(), bucketName, objectName, reqHeaders) +func (c Core) GetObject(bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, error) { + return c.getObject(context.Background(), bucketName, objectName, opts) } // StatObject is a lower level API implemented to support special // conditions matching etag, modtime on a request. -func (c Core) StatObject(bucketName, objectName string, reqHeaders RequestHeaders) (ObjectInfo, error) { - return c.statObject(bucketName, objectName, reqHeaders) +func (c Core) StatObject(bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { + return c.statObject(bucketName, objectName, opts) } diff --git a/vendor/github.com/minio/minio-go/functional_tests.go b/vendor/github.com/minio/minio-go/functional_tests.go deleted file mode 100644 index d7852252f..000000000 --- a/vendor/github.com/minio/minio-go/functional_tests.go +++ /dev/null @@ -1,5889 +0,0 @@ -// +build ignore - -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "bytes" - "context" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "math/rand" - "net/http" - "net/url" - "os" - "reflect" - "strconv" - "strings" - "time" - - humanize "github.com/dustin/go-humanize" - minio "github.com/minio/minio-go" - log "github.com/sirupsen/logrus" - - "github.com/minio/minio-go/pkg/encrypt" - "github.com/minio/minio-go/pkg/policy" -) - -const ( - sixtyFiveMiB = 65 * humanize.MiByte // 65MiB - thirtyThreeKiB = 33 * humanize.KiByte // 33KiB - oneMiB = 1 * humanize.MiByte // 1MiB -) - -const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" -const ( - letterIdxBits = 6 // 6 bits to represent a letter index - letterIdxMask = 1<= 0; { - if remain == 0 { - cache, remain = src.Int63(), letterIdxMax - } - if idx := int(cache & letterIdxMask); idx < len(letterBytes) { - b[i] = letterBytes[idx] - i-- - } - cache >>= letterIdxBits - remain-- - } - return prefix + string(b[0:30-len(prefix)]) -} - -func isQuickMode() bool { - return os.Getenv("MODE") == "quick" -} - -// Tests bucket re-create errors. -func testMakeBucketError() { - region := "eu-central-1" - - // initialize logging params - startTime := time.Now() - function := "MakeBucket(bucketName, region)" - // initialize logging params - args := map[string]interface{}{ - "bucketName": "", - "region": region, - } - - // skipping region functional tests for non s3 runs - if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { - ignoredLog(function, args, startTime, "Skipped region functional tests for non s3 runs").Info() - return - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - // Make a new bucket in 'eu-central-1'. - if err = c.MakeBucket(bucketName, region); err != nil { - failureLog(function, args, startTime, "", "MakeBucket Failed", err).Fatal() - } - if err = c.MakeBucket(bucketName, region); err == nil { - failureLog(function, args, startTime, "", "Bucket already exists", err).Fatal() - } - // Verify valid error response from server. - if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" && - minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { - failureLog(function, args, startTime, "", "Invalid error returned by server", err).Fatal() - } - if err = c.RemoveBucket(bucketName); err != nil { - failureLog(function, args, startTime, "", "Remove bucket failed", err).Fatal() - } - - successLogger(function, args, startTime).Info() -} - -func testMetadataSizeLimit() { - startTime := time.Now() - function := "PutObject(bucketName, objectName, reader, objectSize, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "opts.UserMetadata": "", - } - rand.Seed(startTime.Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client creation failed", err).Fatal() - } - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "Make bucket failed", err).Fatal() - } - - const HeaderSizeLimit = 8 * 1024 - const UserMetadataLimit = 2 * 1024 - - // Meta-data greater than the 2 KB limit of AWS - PUT calls with this meta-data should fail - metadata := make(map[string]string) - metadata["X-Amz-Meta-Mint-Test"] = string(bytes.Repeat([]byte("m"), 1+UserMetadataLimit-len("X-Amz-Meta-Mint-Test"))) - args["metadata"] = fmt.Sprint(metadata) - - _, err = c.PutObject(bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata}) - if err == nil { - failureLog(function, args, startTime, "", "Created object with user-defined metadata exceeding metadata size limits", nil).Fatal() - } - - // Meta-data (headers) greater than the 8 KB limit of AWS - PUT calls with this meta-data should fail - metadata = make(map[string]string) - metadata["X-Amz-Mint-Test"] = string(bytes.Repeat([]byte("m"), 1+HeaderSizeLimit-len("X-Amz-Mint-Test"))) - args["metadata"] = fmt.Sprint(metadata) - _, err = c.PutObject(bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata}) - if err == nil { - failureLog(function, args, startTime, "", "Created object with headers exceeding header size limits", nil).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Tests various bucket supported formats. -func testMakeBucketRegions() { - region := "eu-central-1" - // initialize logging params - startTime := time.Now() - function := "MakeBucket(bucketName, region)" - // initialize logging params - args := map[string]interface{}{ - "bucketName": "", - "region": region, - } - - // skipping region functional tests for non s3 runs - if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { - ignoredLog(function, args, startTime, "Skipped region functional tests for non s3 runs").Info() - return - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - // Make a new bucket in 'eu-central-1'. - if err = c.MakeBucket(bucketName, region); err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - if err = c.RemoveBucket(bucketName); err != nil { - failureLog(function, args, startTime, "", "Remove bucket failed", err).Fatal() - } - - // Make a new bucket with '.' in its name, in 'us-west-2'. This - // request is internally staged into a path style instead of - // virtual host style. - region = "us-west-2" - args["region"] = region - if err = c.MakeBucket(bucketName+".withperiod", region); err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Remove the newly created bucket. - if err = c.RemoveBucket(bucketName + ".withperiod"); err != nil { - failureLog(function, args, startTime, "", "Remove bucket failed", err).Fatal() - } - - successLogger(function, args, startTime).Info() -} - -// Test PutObject using a large data to trigger multipart readat -func testPutObjectReadAt() { - // initialize logging params - startTime := time.Now() - function := "PutObject(bucketName, objectName, reader, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "opts": "objectContentType", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "Make bucket failed", err).Fatal() - } - - // Generate data using 4 parts so that all 3 'workers' are utilized and a part is leftover. - // Use different data for each part for multipart tests to ensure part order at the end. - var reader = getDataReader("datafile-65-MB", sixtyFiveMiB) - defer reader.Close() - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - // Object content type - objectContentType := "binary/octet-stream" - args["objectContentType"] = objectContentType - - n, err := c.PutObject(bucketName, objectName, reader, int64(sixtyFiveMiB), minio.PutObjectOptions{ContentType: objectContentType}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - } - - if n != int64(sixtyFiveMiB) { - failureLog(function, args, startTime, "", "Number of bytes returned by PutObject does not match, expected "+string(sixtyFiveMiB)+" got "+string(n), err).Fatal() - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "Get Object failed", err).Fatal() - } - - st, err := r.Stat() - if err != nil { - failureLog(function, args, startTime, "", "Stat Object failed", err).Fatal() - } - if st.Size != int64(sixtyFiveMiB) { - failureLog(function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(sixtyFiveMiB)+" got "+string(st.Size), err).Fatal() - } - if st.ContentType != objectContentType { - failureLog(function, args, startTime, "", "Content types don't match", err).Fatal() - } - if err := r.Close(); err != nil { - failureLog(function, args, startTime, "", "Object Close failed", err).Fatal() - } - if err := r.Close(); err == nil { - failureLog(function, args, startTime, "", "Object is already closed, didn't return error on Close", err).Fatal() - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - err = c.RemoveBucket(bucketName) - - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - - successLogger(function, args, startTime).Info() -} - -// Test PutObject using a large data to trigger multipart readat -func testPutObjectWithMetadata() { - // initialize logging params - startTime := time.Now() - function := "PutObject(bucketName, objectName, reader,size, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}", - } - - if isQuickMode() { - ignoredLog(function, args, startTime, "Skipping functional tests for short runs").Info() - return - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "Make bucket failed", err).Fatal() - } - - // Generate data using 2 parts - // Use different data in each part for multipart tests to ensure part order at the end. - var reader = getDataReader("datafile-65-MB", sixtyFiveMiB) - defer reader.Close() - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - // Object custom metadata - customContentType := "custom/contenttype" - - args["metadata"] = map[string][]string{ - "Content-Type": {customContentType}, - } - - n, err := c.PutObject(bucketName, objectName, reader, int64(sixtyFiveMiB), minio.PutObjectOptions{ - ContentType: customContentType}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - } - - if n != int64(sixtyFiveMiB) { - failureLog(function, args, startTime, "", "Number of bytes returned by PutObject does not match, expected "+string(sixtyFiveMiB)+" got "+string(n), err).Fatal() - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "GetObject failed", err).Fatal() - } - - st, err := r.Stat() - if err != nil { - failureLog(function, args, startTime, "", "Stat failed", err).Fatal() - } - if st.Size != int64(sixtyFiveMiB) { - failureLog(function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(sixtyFiveMiB)+" got "+string(st.Size), err).Fatal() - } - if st.ContentType != customContentType { - failureLog(function, args, startTime, "", "ContentType does not match, expected "+customContentType+" got "+st.ContentType, err).Fatal() - } - if err := r.Close(); err != nil { - failureLog(function, args, startTime, "", "Object Close failed", err).Fatal() - } - if err := r.Close(); err == nil { - failureLog(function, args, startTime, "", "Object already closed, should respond with error", err).Fatal() - } - - if err = c.RemoveObject(bucketName, objectName); err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - - if err = c.RemoveBucket(bucketName); err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - - successLogger(function, args, startTime).Info() -} - -// Test put object with streaming signature. -func testPutObjectStreaming() { - // initialize logging params - objectName := "test-object" - startTime := time.Now() - function := "PutObject(bucketName, objectName, reader,size,opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": objectName, - "size": -1, - "opts": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), - "minio-go-test") - args["bucketName"] = bucketName - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Upload an object. - sizes := []int64{0, 64*1024 - 1, 64 * 1024} - - for _, size := range sizes { - data := bytes.Repeat([]byte("a"), int(size)) - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(data), int64(size), minio.PutObjectOptions{}) - if err != nil { - failureLog(function, args, startTime, "", "PutObjectStreaming failed", err).Fatal() - } - - if n != size { - failureLog(function, args, startTime, "", "Expected upload object size doesn't match with PutObjectStreaming return value", err).Fatal() - } - } - - // Remove the object. - err = c.RemoveObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - - // Remove the bucket. - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Test listing partially uploaded objects. -func testListPartiallyUploaded() { - // initialize logging params - startTime := time.Now() - function := "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "isRecursive": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Enable tracing, write to stdout. - // c.TraceOn(os.Stderr) - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - r := bytes.NewReader(bytes.Repeat([]byte("0"), sixtyFiveMiB*2)) - - reader, writer := io.Pipe() - go func() { - i := 0 - for i < 25 { - _, cerr := io.CopyN(writer, r, (sixtyFiveMiB*2)/25) - if cerr != nil { - failureLog(function, args, startTime, "", "Copy failed", err).Fatal() - } - i++ - r.Seek(0, 0) - } - writer.CloseWithError(errors.New("proactively closed to be verified later")) - }() - - objectName := bucketName + "-resumable" - args["objectName"] = objectName - - _, err = c.PutObject(bucketName, objectName, reader, int64(sixtyFiveMiB*2), minio.PutObjectOptions{ContentType: "application/octet-stream"}) - if err == nil { - failureLog(function, args, startTime, "", "PutObject should fail", err).Fatal() - } - if !strings.Contains(err.Error(), "proactively closed to be verified later") { - failureLog(function, args, startTime, "", "String not found in PutObject output", err).Fatal() - } - - doneCh := make(chan struct{}) - defer close(doneCh) - isRecursive := true - args["isRecursive"] = isRecursive - - multiPartObjectCh := c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) - for multiPartObject := range multiPartObjectCh { - if multiPartObject.Err != nil { - failureLog(function, args, startTime, "", "Multipart object error", multiPartObject.Err).Fatal() - } - } - - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Test get object seeker from the end, using whence set to '2'. -func testGetObjectSeekEnd() { - // initialize logging params - startTime := time.Now() - function := "GetObject(bucketName, objectName)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Generate 33K of data. - var reader = getDataReader("datafile-33-kB", thirtyThreeKiB) - defer reader.Close() - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - buf, err := ioutil.ReadAll(reader) - if err != nil { - failureLog(function, args, startTime, "", "ReadAll failed", err).Fatal() - } - - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - } - - if n != int64(thirtyThreeKiB) { - failureLog(function, args, startTime, "", "Number of bytes read does not match, expected "+string(int64(thirtyThreeKiB))+" got "+string(n), err).Fatal() - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "GetObject failed", err).Fatal() - } - - st, err := r.Stat() - if err != nil { - failureLog(function, args, startTime, "", "Stat failed", err).Fatal() - } - - if st.Size != int64(thirtyThreeKiB) { - failureLog(function, args, startTime, "", "Number of bytes read does not match, expected "+string(int64(thirtyThreeKiB))+" got "+string(st.Size), err).Fatal() - } - - pos, err := r.Seek(-100, 2) - if err != nil { - failureLog(function, args, startTime, "", "Object Seek failed", err).Fatal() - } - if pos != st.Size-100 { - failureLog(function, args, startTime, "", "Incorrect position", err).Fatal() - } - buf2 := make([]byte, 100) - m, err := io.ReadFull(r, buf2) - if err != nil { - failureLog(function, args, startTime, "", "Error reading through io.ReadFull", err).Fatal() - } - if m != len(buf2) { - failureLog(function, args, startTime, "", "Number of bytes dont match, expected "+string(len(buf2))+" got "+string(m), err).Fatal() - } - hexBuf1 := fmt.Sprintf("%02x", buf[len(buf)-100:]) - hexBuf2 := fmt.Sprintf("%02x", buf2[:m]) - if hexBuf1 != hexBuf2 { - failureLog(function, args, startTime, "", "Values at same index dont match", err).Fatal() - } - pos, err = r.Seek(-100, 2) - if err != nil { - failureLog(function, args, startTime, "", "Object Seek failed", err).Fatal() - } - if pos != st.Size-100 { - failureLog(function, args, startTime, "", "Incorrect position", err).Fatal() - } - if err = r.Close(); err != nil { - failureLog(function, args, startTime, "", "ObjectClose failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Test get object reader to not throw error on being closed twice. -func testGetObjectClosedTwice() { - // initialize logging params - startTime := time.Now() - function := "GetObject(bucketName, objectName)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Generate 33K of data. - var reader = getDataReader("datafile-33-kB", thirtyThreeKiB) - defer reader.Close() - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - n, err := c.PutObject(bucketName, objectName, reader, int64(thirtyThreeKiB), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - } - - if n != int64(thirtyThreeKiB) { - failureLog(function, args, startTime, "", "PutObject response doesn't match sent bytes, expected "+string(int64(thirtyThreeKiB))+" got "+string(n), err).Fatal() - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "GetObject failed", err).Fatal() - } - - st, err := r.Stat() - if err != nil { - failureLog(function, args, startTime, "", "Stat failed", err).Fatal() - } - if st.Size != int64(thirtyThreeKiB) { - failureLog(function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(thirtyThreeKiB))+" got "+string(st.Size), err).Fatal() - } - if err := r.Close(); err != nil { - failureLog(function, args, startTime, "", "Object Close failed", err).Fatal() - } - if err := r.Close(); err == nil { - failureLog(function, args, startTime, "", "Already closed object. No error returned", err).Fatal() - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Test removing multiple objects with Remove API -func testRemoveMultipleObjects() { - // initialize logging params - startTime := time.Now() - function := "RemoveObjects(bucketName, objectsCh)" - args := map[string]interface{}{ - "bucketName": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Enable tracing, write to stdout. - // c.TraceOn(os.Stderr) - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - r := bytes.NewReader(bytes.Repeat([]byte("a"), 8)) - - // Multi remove of 1100 objects - nrObjects := 1100 - - objectsCh := make(chan string) - - go func() { - defer close(objectsCh) - // Upload objects and send them to objectsCh - for i := 0; i < nrObjects; i++ { - objectName := "sample" + strconv.Itoa(i) + ".txt" - _, err = c.PutObject(bucketName, objectName, r, 8, minio.PutObjectOptions{ContentType: "application/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - continue - } - objectsCh <- objectName - } - }() - - // Call RemoveObjects API - errorCh := c.RemoveObjects(bucketName, objectsCh) - - // Check if errorCh doesn't receive any error - select { - case r, more := <-errorCh: - if more { - failureLog(function, args, startTime, "", "Unexpected error", r.Err).Fatal() - } - } - - // Clean the bucket created by the test - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Tests removing partially uploaded objects. -func testRemovePartiallyUploaded() { - // initialize logging params - startTime := time.Now() - function := "RemoveIncompleteUpload(bucketName, objectName)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Enable tracing, write to stdout. - // c.TraceOn(os.Stderr) - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - r := bytes.NewReader(bytes.Repeat([]byte("a"), 128*1024)) - - reader, writer := io.Pipe() - go func() { - i := 0 - for i < 25 { - _, cerr := io.CopyN(writer, r, 128*1024) - if cerr != nil { - failureLog(function, args, startTime, "", "Copy failed", err).Fatal() - } - i++ - r.Seek(0, 0) - } - writer.CloseWithError(errors.New("proactively closed to be verified later")) - }() - - objectName := bucketName + "-resumable" - args["objectName"] = objectName - - _, err = c.PutObject(bucketName, objectName, reader, 128*1024, minio.PutObjectOptions{ContentType: "application/octet-stream"}) - if err == nil { - failureLog(function, args, startTime, "", "PutObject should fail", err).Fatal() - } - if !strings.Contains(err.Error(), "proactively closed to be verified later") { - failureLog(function, args, startTime, "", "String not found", err).Fatal() - } - err = c.RemoveIncompleteUpload(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveIncompleteUpload failed", err).Fatal() - } - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Tests FPutObject of a big file to trigger multipart -func testFPutObjectMultipart() { - // initialize logging params - startTime := time.Now() - function := "FPutObject(bucketName, objectName, fileName, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "fileName": "", - "opts": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload. - var fileName = getFilePath("datafile-65-MB") - if os.Getenv("MINT_DATA_DIR") == "" { - // Make a temp file with minPartSize bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") - if err != nil { - failureLog(function, args, startTime, "", "TempFile creation failed", err).Fatal() - } - // Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload. - _, err = io.Copy(file, getDataReader("non-existent", sixtyFiveMiB)) - if err != nil { - failureLog(function, args, startTime, "", "Copy failed", err).Fatal() - } - err = file.Close() - if err != nil { - failureLog(function, args, startTime, "", "File Close failed", err).Fatal() - } - fileName = file.Name() - args["fileName"] = fileName - } - totalSize := sixtyFiveMiB * 1 - // Set base object name - objectName := bucketName + "FPutObject" + "-standard" - args["objectName"] = objectName - - objectContentType := "testapplication/octet-stream" - args["objectContentType"] = objectContentType - - // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) - n, err := c.FPutObject(bucketName, objectName, fileName, minio.PutObjectOptions{ContentType: objectContentType}) - if err != nil { - failureLog(function, args, startTime, "", "FPutObject failed", err).Fatal() - } - if n != int64(totalSize) { - failureLog(function, args, startTime, "", "FPutObject failed", err).Fatal() - } - - r, err := c.GetObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "GetObject failed", err).Fatal() - } - objInfo, err := r.Stat() - if err != nil { - failureLog(function, args, startTime, "", "Unexpected error", err).Fatal() - } - if objInfo.Size != int64(totalSize) { - failureLog(function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(totalSize))+" got "+string(objInfo.Size), err).Fatal() - } - if objInfo.ContentType != objectContentType { - failureLog(function, args, startTime, "", "ContentType doesn't match", err).Fatal() - } - - // Remove all objects and bucket and temp file - err = c.RemoveObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Tests FPutObject with null contentType (default = application/octet-stream) -func testFPutObject() { - // initialize logging params - startTime := time.Now() - function := "FPutObject(bucketName, objectName, fileName, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "opts": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Upload 3 parts worth of data to use all 3 of multiparts 'workers' and have an extra part. - // Use different data in part for multipart tests to check parts are uploaded in correct order. - var fName = getFilePath("datafile-65-MB") - if os.Getenv("MINT_DATA_DIR") == "" { - // Make a temp file with minPartSize bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") - if err != nil { - failureLog(function, args, startTime, "", "TempFile creation failed", err).Fatal() - } - - // Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload. - var buffer = bytes.Repeat([]byte(string('a')), sixtyFiveMiB) - if _, err = file.Write(buffer); err != nil { - failureLog(function, args, startTime, "", "File write failed", err).Fatal() - } - // Close the file pro-actively for windows. - err = file.Close() - if err != nil { - failureLog(function, args, startTime, "", "File close failed", err).Fatal() - } - fName = file.Name() - } - var totalSize = sixtyFiveMiB * 1 - - // Set base object name - objectName := bucketName + "FPutObject" - args["objectName"] = objectName - - // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) - n, err := c.FPutObject(bucketName, objectName+"-standard", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "FPutObject failed", err).Fatal() - } - if n != int64(totalSize) { - failureLog(function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err).Fatal() - } - - // Perform FPutObject with no contentType provided (Expecting application/octet-stream) - n, err = c.FPutObject(bucketName, objectName+"-Octet", fName, minio.PutObjectOptions{}) - if err != nil { - failureLog(function, args, startTime, "", "File close failed", err).Fatal() - } - if n != int64(totalSize) { - failureLog(function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err).Fatal() - } - srcFile, err := os.Open(fName) - if err != nil { - failureLog(function, args, startTime, "", "File open failed", err).Fatal() - } - defer srcFile.Close() - // Add extension to temp file name - tmpFile, err := os.Create(fName + ".gtar") - if err != nil { - failureLog(function, args, startTime, "", "File create failed", err).Fatal() - } - defer tmpFile.Close() - _, err = io.Copy(tmpFile, srcFile) - if err != nil { - failureLog(function, args, startTime, "", "File copy failed", err).Fatal() - } - - // Perform FPutObject with no contentType provided (Expecting application/x-gtar) - n, err = c.FPutObject(bucketName, objectName+"-GTar", fName+".gtar", minio.PutObjectOptions{}) - if err != nil { - failureLog(function, args, startTime, "", "FPutObject failed", err).Fatal() - } - if n != int64(totalSize) { - failureLog(function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err).Fatal() - } - - // Check headers - rStandard, err := c.StatObject(bucketName, objectName+"-standard") - if err != nil { - failureLog(function, args, startTime, "", "StatObject failed", err).Fatal() - } - if rStandard.ContentType != "application/octet-stream" { - failureLog(function, args, startTime, "", "ContentType does not match, expected application/octet-stream, got "+rStandard.ContentType, err).Fatal() - } - - rOctet, err := c.StatObject(bucketName, objectName+"-Octet") - if err != nil { - failureLog(function, args, startTime, "", "StatObject failed", err).Fatal() - } - if rOctet.ContentType != "application/octet-stream" { - failureLog(function, args, startTime, "", "ContentType does not match, expected application/octet-stream, got "+rStandard.ContentType, err).Fatal() - } - - rGTar, err := c.StatObject(bucketName, objectName+"-GTar") - if err != nil { - failureLog(function, args, startTime, "", "StatObject failed", err).Fatal() - } - if rGTar.ContentType != "application/x-gtar" { - failureLog(function, args, startTime, "", "ContentType does not match, expected application/x-gtar, got "+rStandard.ContentType, err).Fatal() - } - - // Remove all objects and bucket and temp file - err = c.RemoveObject(bucketName, objectName+"-standard") - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - - err = c.RemoveObject(bucketName, objectName+"-Octet") - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - - err = c.RemoveObject(bucketName, objectName+"-GTar") - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - - err = os.Remove(fName + ".gtar") - if err != nil { - failureLog(function, args, startTime, "", "File remove failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Tests FPutObjectWithContext request context cancels after timeout -func testFPutObjectWithContext() { - // initialize logging params - startTime := time.Now() - function := "FPutObject(bucketName, objectName, fileName, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "opts": "", - } - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Upload 1 parts worth of data to use multipart upload. - // Use different data in part for multipart tests to check parts are uploaded in correct order. - var fName = getFilePath("datafile-1-MB") - if os.Getenv("MINT_DATA_DIR") == "" { - // Make a temp file with 1 MiB bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectWithContextTest") - if err != nil { - failureLog(function, args, startTime, "", "TempFile creation failed", err).Fatal() - } - - // Upload 1 parts to trigger multipart upload - var buffer = bytes.Repeat([]byte(string('a')), 1024*1024*1) - if _, err = file.Write(buffer); err != nil { - failureLog(function, args, startTime, "", "File buffer write failed", err).Fatal() - } - // Close the file pro-actively for windows. - err = file.Close() - if err != nil { - failureLog(function, args, startTime, "", "File close failed", err).Fatal() - - } - fName = file.Name() - } - var totalSize = 1024 * 1024 * 1 - - // Set base object name - objectName := bucketName + "FPutObjectWithContext" - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) - defer cancel() - - // Perform standard FPutObjectWithContext with contentType provided (Expecting application/octet-stream) - _, err = c.FPutObjectWithContext(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) - if err == nil { - failureLog(function, args, startTime, "", "Request context cancellation failed", err).Fatal() - } - ctx, cancel = context.WithTimeout(context.Background(), 10*time.Minute) - defer cancel() - // Perform FPutObjectWithContext with a long timeout. Expect the put object to succeed - n, err := c.FPutObjectWithContext(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{}) - if err != nil { - failureLog(function, args, startTime, "", "FPutObjectWithContext failed", err).Fatal() - } - if n != int64(totalSize) { - failureLog(function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err).Fatal() - } - - _, err = c.StatObject(bucketName, objectName+"-Longtimeout") - if err != nil { - failureLog(function, args, startTime, "", "StatObject failed", err).Fatal() - } - err = c.RemoveObject(bucketName, objectName+"-Shorttimeout") - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - // Remove all objects and bucket and temp file - err = c.RemoveObject(bucketName, objectName+"-Longtimeout") - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - - err = os.Remove(fName) - if err != nil { - failureLog(function, args, startTime, "", "Remove file failed", err).Fatal() - } - successLogger(function, args, startTime).Info() - -} - -// Tests FPutObjectWithContext request context cancels after timeout -func testFPutObjectWithContextV2() { - // initialize logging params - startTime := time.Now() - function := "FPutObjectWithContext(ctx, bucketName, objectName, fileName, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "opts": "minio.PutObjectOptions{ContentType:objectContentType}", - } - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Upload 1 parts worth of data to use multipart upload. - // Use different data in part for multipart tests to check parts are uploaded in correct order. - var fName = getFilePath("datafile-1-MB") - if os.Getenv("MINT_DATA_DIR") == "" { - // Make a temp file with 1 MiB bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectWithContextTest") - if err != nil { - failureLog(function, args, startTime, "", "Temp file creation failed", err).Fatal() - - } - - // Upload 1 parts to trigger multipart upload - var buffer = bytes.Repeat([]byte(string('a')), 1024*1024*1) - if _, err = file.Write(buffer); err != nil { - failureLog(function, args, startTime, "", "Write buffer to file failed", err).Fatal() - - } - // Close the file pro-actively for windows. - err = file.Close() - if err != nil { - failureLog(function, args, startTime, "", "File close failed", err).Fatal() - - } - fName = file.Name() - } - var totalSize = 1024 * 1024 * 1 - - // Set base object name - objectName := bucketName + "FPutObjectWithContext" - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Millisecond) - defer cancel() - - // Perform standard FPutObjectWithContext with contentType provided (Expecting application/octet-stream) - _, err = c.FPutObjectWithContext(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) - if err == nil { - failureLog(function, args, startTime, "", "FPutObjectWithContext with short timeout failed", err).Fatal() - } - ctx, cancel = context.WithTimeout(context.Background(), 10*time.Minute) - defer cancel() - // Perform FPutObjectWithContext with a long timeout. Expect the put object to succeed - n, err := c.FPutObjectWithContext(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{}) - if err != nil { - failureLog(function, args, startTime, "", "FPutObjectWithContext with long timeout failed", err).Fatal() - } - if n != int64(totalSize) { - failureLog(function, args, startTime, "", "Number of bytes does not match:wanted"+string(totalSize)+" got "+string(n), err).Fatal() - } - - _, err = c.StatObject(bucketName, objectName+"-Longtimeout") - if err != nil { - failureLog(function, args, startTime, "", "StatObject failed", err).Fatal() - - } - - err = c.RemoveObject(bucketName, objectName+"-Shorttimeout") - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - err = c.RemoveObject(bucketName, objectName+"-Longtimeout") - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - - } - - err = os.Remove(fName) - if err != nil { - failureLog(function, args, startTime, "", "Remove file failed", err).Fatal() - - } - successLogger(function, args, startTime).Info() - -} - -// Test validates putObject with context to see if request cancellation is honored. -func testPutObjectWithContext() { - // initialize logging params - startTime := time.Now() - function := "PutObjectWithContext(ctx, bucketName, objectName, fileName, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "opts": "minio.PutObjectOptions{ContentType:objectContentType}", - } - // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Make a new bucket. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket call failed", err).Fatal() - } - defer c.RemoveBucket(bucketName) - bufSize := 1<<20 + 32*1024 - var reader = getDataReader("datafile-33-kB", bufSize) - defer reader.Close() - objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - _, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObjectWithContext with short timeout failed", err).Fatal() - } - - ctx, cancel = context.WithTimeout(context.Background(), 3*time.Minute) - defer cancel() - reader = getDataReader("datafile-33-kB", bufSize) - defer reader.Close() - _, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObjectWithContext with long timeout failed", err).Fatal() - } - - if err = c.RemoveObject(bucketName, objectName); err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - successLogger(function, args, startTime).Info() - -} - -// Tests get object ReaderSeeker interface methods. -func testGetObjectReadSeekFunctional() { - // initialize logging params - startTime := time.Now() - function := "GetObject(bucketName, objectName)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Generate 33K of data. - var reader = getDataReader("datafile-33-kB", thirtyThreeKiB) - defer reader.Close() - - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - buf, err := ioutil.ReadAll(reader) - if err != nil { - failureLog(function, args, startTime, "", "ReadAll failed", err).Fatal() - } - - // Save the data - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - } - - if n != int64(thirtyThreeKiB) { - failureLog(function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(thirtyThreeKiB))+", got "+string(n), err).Fatal() - } - - defer func() { - err = c.RemoveObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - }() - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "GetObject failed", err).Fatal() - } - - st, err := r.Stat() - if err != nil { - failureLog(function, args, startTime, "", "Stat object failed", err).Fatal() - } - - if st.Size != int64(thirtyThreeKiB) { - failureLog(function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(thirtyThreeKiB))+", got "+string(st.Size), err).Fatal() - } - - // This following function helps us to compare data from the reader after seek - // with the data from the original buffer - cmpData := func(r io.Reader, start, end int) { - if end-start == 0 { - return - } - buffer := bytes.NewBuffer([]byte{}) - if _, err := io.CopyN(buffer, r, int64(thirtyThreeKiB)); err != nil { - if err != io.EOF { - failureLog(function, args, startTime, "", "CopyN failed", err).Fatal() - } - } - if !bytes.Equal(buf[start:end], buffer.Bytes()) { - failureLog(function, args, startTime, "", "Incorrect read bytes v/s original buffer", err).Fatal() - } - } - - // Generic seek error for errors other than io.EOF - seekErr := errors.New("seek error") - - testCases := []struct { - offset int64 - whence int - pos int64 - err error - shouldCmp bool - start int - end int - }{ - // Start from offset 0, fetch data and compare - {0, 0, 0, nil, true, 0, 0}, - // Start from offset 2048, fetch data and compare - {2048, 0, 2048, nil, true, 2048, thirtyThreeKiB}, - // Start from offset larger than possible - {int64(thirtyThreeKiB) + 1024, 0, 0, seekErr, false, 0, 0}, - // Move to offset 0 without comparing - {0, 0, 0, nil, false, 0, 0}, - // Move one step forward and compare - {1, 1, 1, nil, true, 1, thirtyThreeKiB}, - // Move larger than possible - {int64(thirtyThreeKiB), 1, 0, seekErr, false, 0, 0}, - // Provide negative offset with CUR_SEEK - {int64(-1), 1, 0, seekErr, false, 0, 0}, - // Test with whence SEEK_END and with positive offset - {1024, 2, int64(thirtyThreeKiB) - 1024, io.EOF, true, 0, 0}, - // Test with whence SEEK_END and with negative offset - {-1024, 2, int64(thirtyThreeKiB) - 1024, nil, true, thirtyThreeKiB - 1024, thirtyThreeKiB}, - // Test with whence SEEK_END and with large negative offset - {-int64(thirtyThreeKiB) * 2, 2, 0, seekErr, true, 0, 0}, - } - - for i, testCase := range testCases { - // Perform seek operation - n, err := r.Seek(testCase.offset, testCase.whence) - // We expect an error - if testCase.err == seekErr && err == nil { - failureLog(function, args, startTime, "", "Test "+string(i+1)+", unexpected err value: expected: "+testCase.err.Error()+", found: "+err.Error(), err).Fatal() - } - // We expect a specific error - if testCase.err != seekErr && testCase.err != err { - failureLog(function, args, startTime, "", "Test "+string(i+1)+", unexpected err value: expected: "+testCase.err.Error()+", found: "+err.Error(), err).Fatal() - } - // If we expect an error go to the next loop - if testCase.err != nil { - continue - } - // Check the returned seek pos - if n != testCase.pos { - failureLog(function, args, startTime, "", "Test "+string(i+1)+", number of bytes seeked does not match, expected "+string(testCase.pos)+", got "+string(n), err).Fatal() - } - // Compare only if shouldCmp is activated - if testCase.shouldCmp { - cmpData(r, testCase.start, testCase.end) - } - } - successLogger(function, args, startTime).Info() -} - -// Tests get object ReaderAt interface methods. -func testGetObjectReadAtFunctional() { - // initialize logging params - startTime := time.Now() - function := "GetObject(bucketName, objectName)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Generate 33K of data. - var reader = getDataReader("datafile-33-kB", thirtyThreeKiB) - defer reader.Close() - - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - buf, err := ioutil.ReadAll(reader) - if err != nil { - failureLog(function, args, startTime, "", "ReadAll failed", err).Fatal() - } - - // Save the data - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - } - - if n != int64(thirtyThreeKiB) { - failureLog(function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(thirtyThreeKiB))+", got "+string(n), err).Fatal() - } - - // read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - } - offset := int64(2048) - - // read directly - buf1 := make([]byte, 512) - buf2 := make([]byte, 512) - buf3 := make([]byte, 512) - buf4 := make([]byte, 512) - - // Test readAt before stat is called. - m, err := r.ReadAt(buf1, offset) - if err != nil { - failureLog(function, args, startTime, "", "ReadAt failed", err).Fatal() - } - if m != len(buf1) { - failureLog(function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err).Fatal() - } - if !bytes.Equal(buf1, buf[offset:offset+512]) { - failureLog(function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err).Fatal() - } - offset += 512 - - st, err := r.Stat() - if err != nil { - failureLog(function, args, startTime, "", "Stat failed", err).Fatal() - } - - if st.Size != int64(thirtyThreeKiB) { - failureLog(function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(thirtyThreeKiB))+", got "+string(st.Size), err).Fatal() - } - - m, err = r.ReadAt(buf2, offset) - if err != nil { - failureLog(function, args, startTime, "", "ReadAt failed", err).Fatal() - } - if m != len(buf2) { - failureLog(function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err).Fatal() - } - if !bytes.Equal(buf2, buf[offset:offset+512]) { - failureLog(function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err).Fatal() - } - offset += 512 - m, err = r.ReadAt(buf3, offset) - if err != nil { - failureLog(function, args, startTime, "", "ReadAt failed", err).Fatal() - } - if m != len(buf3) { - failureLog(function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err).Fatal() - } - if !bytes.Equal(buf3, buf[offset:offset+512]) { - failureLog(function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err).Fatal() - } - offset += 512 - m, err = r.ReadAt(buf4, offset) - if err != nil { - failureLog(function, args, startTime, "", "ReadAt failed", err).Fatal() - } - if m != len(buf4) { - failureLog(function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err).Fatal() - } - if !bytes.Equal(buf4, buf[offset:offset+512]) { - failureLog(function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err).Fatal() - } - - buf5 := make([]byte, n) - // Read the whole object. - m, err = r.ReadAt(buf5, 0) - if err != nil { - if err != io.EOF { - failureLog(function, args, startTime, "", "ReadAt failed", err).Fatal() - } - } - if m != len(buf5) { - failureLog(function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err).Fatal() - } - if !bytes.Equal(buf, buf5) { - failureLog(function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err).Fatal() - } - - buf6 := make([]byte, n+1) - // Read the whole object and beyond. - _, err = r.ReadAt(buf6, 0) - if err != nil { - if err != io.EOF { - failureLog(function, args, startTime, "", "ReadAt failed", err).Fatal() - } - } - err = c.RemoveObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Test Presigned Post Policy -func testPresignedPostPolicy() { - // initialize logging params - startTime := time.Now() - function := "PresignedPostPolicy(policy)" - args := map[string]interface{}{ - "policy": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Generate 33K of data. - var reader = getDataReader("datafile-33-kB", thirtyThreeKiB) - defer reader.Close() - - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - - buf, err := ioutil.ReadAll(reader) - if err != nil { - failureLog(function, args, startTime, "", "ReadAll failed", err).Fatal() - } - - // Save the data - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - } - - if n != int64(thirtyThreeKiB) { - failureLog(function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(thirtyThreeKiB))+" got "+string(n), err).Fatal() - } - - policy := minio.NewPostPolicy() - - if err := policy.SetBucket(""); err == nil { - failureLog(function, args, startTime, "", "SetBucket did not fail for invalid conditions", err).Fatal() - } - if err := policy.SetKey(""); err == nil { - failureLog(function, args, startTime, "", "SetKey did not fail for invalid conditions", err).Fatal() - } - if err := policy.SetKeyStartsWith(""); err == nil { - failureLog(function, args, startTime, "", "SetKeyStartsWith did not fail for invalid conditions", err).Fatal() - } - if err := policy.SetExpires(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)); err == nil { - failureLog(function, args, startTime, "", "SetExpires did not fail for invalid conditions", err).Fatal() - } - if err := policy.SetContentType(""); err == nil { - failureLog(function, args, startTime, "", "SetContentType did not fail for invalid conditions", err).Fatal() - } - if err := policy.SetContentLengthRange(1024*1024, 1024); err == nil { - failureLog(function, args, startTime, "", "SetContentLengthRange did not fail for invalid conditions", err).Fatal() - } - - policy.SetBucket(bucketName) - policy.SetKey(objectName) - policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days - policy.SetContentType("image/png") - policy.SetContentLengthRange(1024, 1024*1024) - args["policy"] = policy - - _, _, err = c.PresignedPostPolicy(policy) - if err != nil { - failureLog(function, args, startTime, "", "PresignedPostPolicy failed", err).Fatal() - } - - policy = minio.NewPostPolicy() - - // Remove all objects and buckets - err = c.RemoveObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Tests copy object -func testCopyObject() { - // initialize logging params - startTime := time.Now() - function := "CopyObject(dst, src)" - args := map[string]interface{}{ - "dst": "", - "src": "", - } - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Make a new bucket in 'us-east-1' (destination bucket). - err = c.MakeBucket(bucketName+"-copy", "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Generate 33K of data. - var reader = getDataReader("datafile-33-kB", thirtyThreeKiB) - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, reader, int64(thirtyThreeKiB), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - } - - if n != int64(thirtyThreeKiB) { - failureLog(function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(thirtyThreeKiB))+", got "+string(n), err).Fatal() - } - - r, err := c.GetObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "GetObject failed", err).Fatal() - } - // Check the various fields of source object against destination object. - objInfo, err := r.Stat() - if err != nil { - failureLog(function, args, startTime, "", "Stat failed", err).Fatal() - } - - // Copy Source - src := minio.NewSourceInfo(bucketName, objectName, nil) - - // Set copy conditions. - - // All invalid conditions first. - err = src.SetModifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) - if err == nil { - failureLog(function, args, startTime, "", "SetModifiedSinceCond did not fail for invalid conditions", err).Fatal() - } - err = src.SetUnmodifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) - if err == nil { - failureLog(function, args, startTime, "", "SetUnmodifiedSinceCond did not fail for invalid conditions", err).Fatal() - } - err = src.SetMatchETagCond("") - if err == nil { - failureLog(function, args, startTime, "", "SetMatchETagCond did not fail for invalid conditions", err).Fatal() - } - err = src.SetMatchETagExceptCond("") - if err == nil { - failureLog(function, args, startTime, "", "SetMatchETagExceptCond did not fail for invalid conditions", err).Fatal() - } - - err = src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - if err != nil { - failureLog(function, args, startTime, "", "SetModifiedSinceCond failed", err).Fatal() - } - err = src.SetMatchETagCond(objInfo.ETag) - if err != nil { - failureLog(function, args, startTime, "", "SetMatchETagCond failed", err).Fatal() - } - args["src"] = src - - dst, err := minio.NewDestinationInfo(bucketName+"-copy", objectName+"-copy", nil, nil) - args["dst"] = dst - if err != nil { - failureLog(function, args, startTime, "", "NewDestinationInfo failed", err).Fatal() - } - - // Perform the Copy - err = c.CopyObject(dst, src) - if err != nil { - failureLog(function, args, startTime, "", "CopyObject failed", err).Fatal() - } - - // Source object - r, err = c.GetObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "GetObject failed", err).Fatal() - } - - // Destination object - readerCopy, err := c.GetObject(bucketName+"-copy", objectName+"-copy") - if err != nil { - failureLog(function, args, startTime, "", "GetObject failed", err).Fatal() - } - // Check the various fields of source object against destination object. - objInfo, err = r.Stat() - if err != nil { - failureLog(function, args, startTime, "", "Stat failed", err).Fatal() - } - objInfoCopy, err := readerCopy.Stat() - if err != nil { - failureLog(function, args, startTime, "", "Stat failed", err).Fatal() - } - if objInfo.Size != objInfoCopy.Size { - failureLog(function, args, startTime, "", "Number of bytes does not match, expected "+string(objInfoCopy.Size)+", got "+string(objInfo.Size), err).Fatal() - } - - // CopyObject again but with wrong conditions - src = minio.NewSourceInfo(bucketName, objectName, nil) - err = src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - if err != nil { - failureLog(function, args, startTime, "", "SetUnmodifiedSinceCond failed", err).Fatal() - } - err = src.SetMatchETagExceptCond(objInfo.ETag) - if err != nil { - failureLog(function, args, startTime, "", "SetMatchETagExceptCond failed", err).Fatal() - } - - // Perform the Copy which should fail - err = c.CopyObject(dst, src) - if err == nil { - failureLog(function, args, startTime, "", "CopyObject did not fail for invalid conditions", err).Fatal() - } - - // Remove all objects and buckets - err = c.RemoveObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - - err = c.RemoveObject(bucketName+"-copy", objectName+"-copy") - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - - err = c.RemoveBucket(bucketName + "-copy") - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// TestEncryptionPutGet tests client side encryption -func testEncryptionPutGet() { - // initialize logging params - startTime := time.Now() - function := "PutEncryptedObject(bucketName, objectName, reader, cbcMaterials, metadata, progress)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "cbcMaterials": "", - "metadata": "", - } - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Generate a symmetric key - symKey := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) - - // Generate an assymmetric key from predefine public and private certificates - privateKey, err := hex.DecodeString( - "30820277020100300d06092a864886f70d0101010500048202613082025d" + - "0201000281810087b42ea73243a3576dc4c0b6fa245d339582dfdbddc20c" + - "bb8ab666385034d997210c54ba79275c51162a1221c3fb1a4c7c61131ca6" + - "5563b319d83474ef5e803fbfa7e52b889e1893b02586b724250de7ac6351" + - "cc0b7c638c980acec0a07020a78eed7eaa471eca4b92071394e061346c06" + - "15ccce2f465dee2080a89e43f29b5702030100010281801dd5770c3af8b3" + - "c85cd18cacad81a11bde1acfac3eac92b00866e142301fee565365aa9af4" + - "57baebf8bb7711054d071319a51dd6869aef3848ce477a0dc5f0dbc0c336" + - "5814b24c820491ae2bb3c707229a654427e03307fec683e6b27856688f08" + - "bdaa88054c5eeeb773793ff7543ee0fb0e2ad716856f2777f809ef7e6fa4" + - "41024100ca6b1edf89e8a8f93cce4b98c76c6990a09eb0d32ad9d3d04fbf" + - "0b026fa935c44f0a1c05dd96df192143b7bda8b110ec8ace28927181fd8c" + - "d2f17330b9b63535024100aba0260afb41489451baaeba423bee39bcbd1e" + - "f63dd44ee2d466d2453e683bf46d019a8baead3a2c7fca987988eb4d565e" + - "27d6be34605953f5034e4faeec9bdb0241009db2cb00b8be8c36710aff96" + - "6d77a6dec86419baca9d9e09a2b761ea69f7d82db2ae5b9aae4246599bb2" + - "d849684d5ab40e8802cfe4a2b358ad56f2b939561d2902404e0ead9ecafd" + - "bb33f22414fa13cbcc22a86bdf9c212ce1a01af894e3f76952f36d6c904c" + - "bd6a7e0de52550c9ddf31f1e8bfe5495f79e66a25fca5c20b3af5b870241" + - "0083456232aa58a8c45e5b110494599bda8dbe6a094683a0539ddd24e19d" + - "47684263bbe285ad953d725942d670b8f290d50c0bca3d1dc9688569f1d5" + - "9945cb5c7d") - - if err != nil { - failureLog(function, args, startTime, "", "DecodeString for symmetric Key generation failed", err).Fatal() - } - - publicKey, err := hex.DecodeString("30819f300d06092a864886f70d010101050003818d003081890281810087" + - "b42ea73243a3576dc4c0b6fa245d339582dfdbddc20cbb8ab666385034d9" + - "97210c54ba79275c51162a1221c3fb1a4c7c61131ca65563b319d83474ef" + - "5e803fbfa7e52b889e1893b02586b724250de7ac6351cc0b7c638c980ace" + - "c0a07020a78eed7eaa471eca4b92071394e061346c0615ccce2f465dee20" + - "80a89e43f29b570203010001") - if err != nil { - failureLog(function, args, startTime, "", "DecodeString for symmetric Key generation failed", err).Fatal() - } - - // Generate an asymmetric key - asymKey, err := encrypt.NewAsymmetricKey(privateKey, publicKey) - if err != nil { - failureLog(function, args, startTime, "", "NewAsymmetricKey for symmetric Key generation failed", err).Fatal() - } - - testCases := []struct { - buf []byte - encKey encrypt.Key - }{ - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 0)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 15)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 16)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 17)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 31)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 32)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 33)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024*2)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024*1024)}, - - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 0)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 16)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 32)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1024)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1024*1024)}, - } - - for i, testCase := range testCases { - // Generate a random object name - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - // Secured object - cbcMaterials, err := encrypt.NewCBCSecureMaterials(testCase.encKey) - args["cbcMaterials"] = cbcMaterials - - if err != nil { - failureLog(function, args, startTime, "", "NewCBCSecureMaterials failed", err).Fatal() - } - - // Put encrypted data - _, err = c.PutEncryptedObject(bucketName, objectName, bytes.NewReader(testCase.buf), cbcMaterials) - if err != nil { - failureLog(function, args, startTime, "", "PutEncryptedObject failed", err).Fatal() - } - - // Read the data back - r, err := c.GetEncryptedObject(bucketName, objectName, cbcMaterials) - if err != nil { - failureLog(function, args, startTime, "", "GetEncryptedObject failed", err).Fatal() - } - defer r.Close() - - // Compare the sent object with the received one - recvBuffer := bytes.NewBuffer([]byte{}) - if _, err = io.Copy(recvBuffer, r); err != nil { - failureLog(function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err).Fatal() - } - if recvBuffer.Len() != len(testCase.buf) { - failureLog(function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err).Fatal() - } - if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { - failureLog(function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err).Fatal() - } - - // Remove test object - err = c.RemoveObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "Test "+string(i+1)+", RemoveObject failed with: "+err.Error(), err).Fatal() - } - successLogger(function, args, startTime).Info() - - } - - // Remove test bucket - err = c.RemoveBucket(bucketName) - if err != nil { - err = c.RemoveBucket(bucketName) - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// TestEncryptionFPut tests client side encryption -func testEncryptionFPut() { - // initialize logging params - startTime := time.Now() - function := "FPutEncryptedObject(bucketName, objectName, filePath, contentType, cbcMaterials)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "filePath": "", - "contentType": "", - "cbcMaterials": "", - } - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Generate a symmetric key - symKey := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) - - // Generate an assymmetric key from predefine public and private certificates - privateKey, err := hex.DecodeString( - "30820277020100300d06092a864886f70d0101010500048202613082025d" + - "0201000281810087b42ea73243a3576dc4c0b6fa245d339582dfdbddc20c" + - "bb8ab666385034d997210c54ba79275c51162a1221c3fb1a4c7c61131ca6" + - "5563b319d83474ef5e803fbfa7e52b889e1893b02586b724250de7ac6351" + - "cc0b7c638c980acec0a07020a78eed7eaa471eca4b92071394e061346c06" + - "15ccce2f465dee2080a89e43f29b5702030100010281801dd5770c3af8b3" + - "c85cd18cacad81a11bde1acfac3eac92b00866e142301fee565365aa9af4" + - "57baebf8bb7711054d071319a51dd6869aef3848ce477a0dc5f0dbc0c336" + - "5814b24c820491ae2bb3c707229a654427e03307fec683e6b27856688f08" + - "bdaa88054c5eeeb773793ff7543ee0fb0e2ad716856f2777f809ef7e6fa4" + - "41024100ca6b1edf89e8a8f93cce4b98c76c6990a09eb0d32ad9d3d04fbf" + - "0b026fa935c44f0a1c05dd96df192143b7bda8b110ec8ace28927181fd8c" + - "d2f17330b9b63535024100aba0260afb41489451baaeba423bee39bcbd1e" + - "f63dd44ee2d466d2453e683bf46d019a8baead3a2c7fca987988eb4d565e" + - "27d6be34605953f5034e4faeec9bdb0241009db2cb00b8be8c36710aff96" + - "6d77a6dec86419baca9d9e09a2b761ea69f7d82db2ae5b9aae4246599bb2" + - "d849684d5ab40e8802cfe4a2b358ad56f2b939561d2902404e0ead9ecafd" + - "bb33f22414fa13cbcc22a86bdf9c212ce1a01af894e3f76952f36d6c904c" + - "bd6a7e0de52550c9ddf31f1e8bfe5495f79e66a25fca5c20b3af5b870241" + - "0083456232aa58a8c45e5b110494599bda8dbe6a094683a0539ddd24e19d" + - "47684263bbe285ad953d725942d670b8f290d50c0bca3d1dc9688569f1d5" + - "9945cb5c7d") - - if err != nil { - failureLog(function, args, startTime, "", "DecodeString for symmetric Key generation failed", err).Fatal() - } - - publicKey, err := hex.DecodeString("30819f300d06092a864886f70d010101050003818d003081890281810087" + - "b42ea73243a3576dc4c0b6fa245d339582dfdbddc20cbb8ab666385034d9" + - "97210c54ba79275c51162a1221c3fb1a4c7c61131ca65563b319d83474ef" + - "5e803fbfa7e52b889e1893b02586b724250de7ac6351cc0b7c638c980ace" + - "c0a07020a78eed7eaa471eca4b92071394e061346c0615ccce2f465dee20" + - "80a89e43f29b570203010001") - if err != nil { - failureLog(function, args, startTime, "", "DecodeString for symmetric Key generation failed", err).Fatal() - } - - // Generate an asymmetric key - asymKey, err := encrypt.NewAsymmetricKey(privateKey, publicKey) - if err != nil { - failureLog(function, args, startTime, "", "NewAsymmetricKey for symmetric Key generation failed", err).Fatal() - } - - // Object custom metadata - customContentType := "custom/contenttype" - args["metadata"] = customContentType - - testCases := []struct { - buf []byte - encKey encrypt.Key - }{ - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 0)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 15)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 16)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 17)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 31)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 32)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 33)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024*2)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024*1024)}, - - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 0)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 16)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 32)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1024)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1024*1024)}, - } - - for i, testCase := range testCases { - // Generate a random object name - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - // Secured object - cbcMaterials, err := encrypt.NewCBCSecureMaterials(testCase.encKey) - args["cbcMaterials"] = cbcMaterials - - if err != nil { - failureLog(function, args, startTime, "", "NewCBCSecureMaterials failed", err).Fatal() - } - // Generate a random file name. - fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - file, err := os.Create(fileName) - if err != nil { - failureLog(function, args, startTime, "", "file create failed", err).Fatal() - } - _, err = file.Write(testCase.buf) - if err != nil { - failureLog(function, args, startTime, "", "file write failed", err).Fatal() - } - file.Close() - // Put encrypted data - if _, err = c.FPutEncryptedObject(bucketName, objectName, fileName, cbcMaterials); err != nil { - failureLog(function, args, startTime, "", "FPutEncryptedObject failed", err).Fatal() - } - - // Read the data back - r, err := c.GetEncryptedObject(bucketName, objectName, cbcMaterials) - if err != nil { - failureLog(function, args, startTime, "", "GetEncryptedObject failed", err).Fatal() - } - defer r.Close() - - // Compare the sent object with the received one - recvBuffer := bytes.NewBuffer([]byte{}) - if _, err = io.Copy(recvBuffer, r); err != nil { - failureLog(function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err).Fatal() - } - if recvBuffer.Len() != len(testCase.buf) { - failureLog(function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err).Fatal() - } - if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { - failureLog(function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err).Fatal() - } - - // Remove test object - err = c.RemoveObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "Test "+string(i+1)+", RemoveObject failed with: "+err.Error(), err).Fatal() - } - if err = os.Remove(fileName); err != nil { - failureLog(function, args, startTime, "", "File remove failed", err).Fatal() - } - } - - // Remove test bucket - err = c.RemoveBucket(bucketName) - if err != nil { - err = c.RemoveBucket(bucketName) - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} -func testBucketNotification() { - // initialize logging params - startTime := time.Now() - function := "SetBucketNotification(bucketName)" - args := map[string]interface{}{ - "bucketName": "", - } - - if os.Getenv("NOTIFY_BUCKET") == "" || - os.Getenv("NOTIFY_SERVICE") == "" || - os.Getenv("NOTIFY_REGION") == "" || - os.Getenv("NOTIFY_ACCOUNTID") == "" || - os.Getenv("NOTIFY_RESOURCE") == "" { - ignoredLog(function, args, startTime, "Skipped notification test as it is not configured").Info() - return - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Enable to debug - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - bucketName := os.Getenv("NOTIFY_BUCKET") - args["bucketName"] = bucketName - - topicArn := minio.NewArn("aws", os.Getenv("NOTIFY_SERVICE"), os.Getenv("NOTIFY_REGION"), os.Getenv("NOTIFY_ACCOUNTID"), os.Getenv("NOTIFY_RESOURCE")) - queueArn := minio.NewArn("aws", "dummy-service", "dummy-region", "dummy-accountid", "dummy-resource") - - topicConfig := minio.NewNotificationConfig(topicArn) - - topicConfig.AddEvents(minio.ObjectCreatedAll, minio.ObjectRemovedAll) - topicConfig.AddFilterSuffix("jpg") - - queueConfig := minio.NewNotificationConfig(queueArn) - queueConfig.AddEvents(minio.ObjectCreatedAll) - queueConfig.AddFilterPrefix("photos/") - - bNotification := minio.BucketNotification{} - bNotification.AddTopic(topicConfig) - - // Add the same topicConfig again, should have no effect - // because it is duplicated - bNotification.AddTopic(topicConfig) - if len(bNotification.TopicConfigs) != 1 { - failureLog(function, args, startTime, "", "Duplicate entry added", err).Fatal() - } - - // Add and remove a queue config - bNotification.AddQueue(queueConfig) - bNotification.RemoveQueueByArn(queueArn) - - err = c.SetBucketNotification(bucketName, bNotification) - if err != nil { - failureLog(function, args, startTime, "", "SetBucketNotification failed", err).Fatal() - } - - bNotification, err = c.GetBucketNotification(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "GetBucketNotification failed", err).Fatal() - } - - if len(bNotification.TopicConfigs) != 1 { - failureLog(function, args, startTime, "", "Topic config is empty", err).Fatal() - } - - if bNotification.TopicConfigs[0].Filter.S3Key.FilterRules[0].Value != "jpg" { - failureLog(function, args, startTime, "", "Couldn't get the suffix", err).Fatal() - } - - err = c.RemoveAllBucketNotification(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveAllBucketNotification failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Tests comprehensive list of all methods. -func testFunctional() { - // initialize logging params - startTime := time.Now() - function := "testFunctional()" - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, nil, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Enable to debug - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - function = "MakeBucket(bucketName, region)" - args := map[string]interface{}{ - "bucketName": bucketName, - } - - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Generate a random file name. - fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - file, err := os.Create(fileName) - if err != nil { - failureLog(function, args, startTime, "", "File creation failed", err).Fatal() - } - for i := 0; i < 3; i++ { - buf := make([]byte, rand.Intn(1<<19)) - _, err = file.Write(buf) - if err != nil { - failureLog(function, args, startTime, "", "File write failed", err).Fatal() - } - } - file.Close() - - // Verify if bucket exits and you have access. - var exists bool - exists, err = c.BucketExists(bucketName) - function = "BucketExists(bucketName)" - args = map[string]interface{}{ - "bucketName": bucketName, - } - - if err != nil { - failureLog(function, args, startTime, "", "BucketExists failed", err).Fatal() - } - if !exists { - failureLog(function, args, startTime, "", "Could not find the bucket", err).Fatal() - } - - // Asserting the default bucket policy. - policyAccess, err := c.GetBucketPolicy(bucketName, "") - function = "GetBucketPolicy(bucketName, objectPrefix)" - args = map[string]interface{}{ - "bucketName": bucketName, - "objectPrefix": "", - } - - if err != nil { - failureLog(function, args, startTime, "", "GetBucketPolicy failed", err).Fatal() - } - if policyAccess != "none" { - failureLog(function, args, startTime, "", "policy should be set to none", err).Fatal() - } - // Set the bucket policy to 'public readonly'. - err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyReadOnly) - function = "SetBucketPolicy(bucketName, objectPrefix, bucketPolicy)" - args = map[string]interface{}{ - "bucketName": bucketName, - "objectPrefix": "", - "bucketPolicy": policy.BucketPolicyReadOnly, - } - - if err != nil { - failureLog(function, args, startTime, "", "SetBucketPolicy failed", err).Fatal() - } - // should return policy `readonly`. - policyAccess, err = c.GetBucketPolicy(bucketName, "") - function = "GetBucketPolicy(bucketName, objectPrefix)" - args = map[string]interface{}{ - "bucketName": bucketName, - "objectPrefix": "", - } - - if err != nil { - failureLog(function, args, startTime, "", "GetBucketPolicy failed", err).Fatal() - } - if policyAccess != "readonly" { - failureLog(function, args, startTime, "", "policy should be set to readonly", err).Fatal() - } - - // Make the bucket 'public writeonly'. - err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyWriteOnly) - function = "SetBucketPolicy(bucketName, objectPrefix, bucketPolicy)" - args = map[string]interface{}{ - "bucketName": bucketName, - "objectPrefix": "", - "bucketPolicy": policy.BucketPolicyWriteOnly, - } - - if err != nil { - failureLog(function, args, startTime, "", "SetBucketPolicy failed", err).Fatal() - } - // should return policy `writeonly`. - policyAccess, err = c.GetBucketPolicy(bucketName, "") - function = "GetBucketPolicy(bucketName, objectPrefix)" - args = map[string]interface{}{ - "bucketName": bucketName, - "objectPrefix": "", - } - - if err != nil { - failureLog(function, args, startTime, "", "GetBucketPolicy failed", err).Fatal() - } - if policyAccess != "writeonly" { - failureLog(function, args, startTime, "", "policy should be set to writeonly", err).Fatal() - } - // Make the bucket 'public read/write'. - err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyReadWrite) - function = "SetBucketPolicy(bucketName, objectPrefix, bucketPolicy)" - args = map[string]interface{}{ - "bucketName": bucketName, - "objectPrefix": "", - "bucketPolicy": policy.BucketPolicyReadWrite, - } - - if err != nil { - failureLog(function, args, startTime, "", "SetBucketPolicy failed", err).Fatal() - } - // should return policy `readwrite`. - policyAccess, err = c.GetBucketPolicy(bucketName, "") - function = "GetBucketPolicy(bucketName, objectPrefix)" - args = map[string]interface{}{ - "bucketName": bucketName, - "objectPrefix": "", - } - - if err != nil { - failureLog(function, args, startTime, "", "GetBucketPolicy failed", err).Fatal() - } - if policyAccess != "readwrite" { - failureLog(function, args, startTime, "", "policy should be set to readwrite", err).Fatal() - } - // List all buckets. - buckets, err := c.ListBuckets() - function = "ListBuckets()" - args = nil - - if len(buckets) == 0 { - failureLog(function, args, startTime, "", "Found bucket list to be empty", err).Fatal() - } - if err != nil { - failureLog(function, args, startTime, "", "ListBuckets failed", err).Fatal() - } - - // Verify if previously created bucket is listed in list buckets. - bucketFound := false - for _, bucket := range buckets { - if bucket.Name == bucketName { - bucketFound = true - } - } - - // If bucket not found error out. - if !bucketFound { - failureLog(function, args, startTime, "", "Bucket: "+bucketName+" not found", err).Fatal() - } - - objectName := bucketName + "unique" - - // Generate data - buf := bytes.Repeat([]byte("f"), 1<<19) - - function = "PutObject(bucketName, objectName, reader, contentType)" - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "contentType": "", - } - - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - } - - if n != int64(len(buf)) { - failureLog(function, args, startTime, "", "Length doesn't match, expected "+string(int64(len(buf)))+" got "+string(n), err).Fatal() - } - - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName + "-nolength", - "contentType": "binary/octet-stream", - } - - n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - } - - if n != int64(len(buf)) { - failureLog(function, args, startTime, "", "Length doesn't match, expected "+string(int64(len(buf)))+" got "+string(n), err).Fatal() - } - - // Instantiate a done channel to close all listing. - doneCh := make(chan struct{}) - defer close(doneCh) - - objFound := false - isRecursive := true // Recursive is true. - - function = "ListObjects(bucketName, objectName, isRecursive, doneCh)" - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "isRecursive": isRecursive, - } - - for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { - if obj.Key == objectName { - objFound = true - break - } - } - if !objFound { - failureLog(function, args, startTime, "", "Object "+objectName+" not found", err).Fatal() - } - - objFound = false - isRecursive = true // Recursive is true. - function = "ListObjectsV2(bucketName, objectName, isRecursive, doneCh)" - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "isRecursive": isRecursive, - } - - for obj := range c.ListObjectsV2(bucketName, objectName, isRecursive, doneCh) { - if obj.Key == objectName { - objFound = true - break - } - } - if !objFound { - failureLog(function, args, startTime, "", "Object "+objectName+" not found", err).Fatal() - } - - incompObjNotFound := true - - function = "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)" - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "isRecursive": isRecursive, - } - - for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { - if objIncompl.Key != "" { - incompObjNotFound = false - break - } - } - if !incompObjNotFound { - failureLog(function, args, startTime, "", "Unexpected dangling incomplete upload found", err).Fatal() - } - - newReader, err := c.GetObject(bucketName, objectName) - function = "GetObject(bucketName, objectName)" - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - } - - if err != nil { - failureLog(function, args, startTime, "", "GetObject failed", err).Fatal() - } - - newReadBytes, err := ioutil.ReadAll(newReader) - if err != nil { - failureLog(function, args, startTime, "", "ReadAll failed", err).Fatal() - } - - if !bytes.Equal(newReadBytes, buf) { - failureLog(function, args, startTime, "", "GetObject bytes mismatch", err).Fatal() - } - - err = c.FGetObject(bucketName, objectName, fileName+"-f") - function = "FGetObject(bucketName, objectName, fileName)" - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "fileName": fileName + "-f", - } - - if err != nil { - failureLog(function, args, startTime, "", "FGetObject failed", err).Fatal() - } - - // Generate presigned HEAD object url. - presignedHeadURL, err := c.PresignedHeadObject(bucketName, objectName, 3600*time.Second, nil) - function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)" - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "expires": 3600 * time.Second, - } - - if err != nil { - failureLog(function, args, startTime, "", "PresignedHeadObject failed", err).Fatal() - } - // Verify if presigned url works. - resp, err := http.Head(presignedHeadURL.String()) - if err != nil { - failureLog(function, args, startTime, "", "PresignedHeadObject response incorrect", err).Fatal() - } - if resp.StatusCode != http.StatusOK { - failureLog(function, args, startTime, "", "PresignedHeadObject response incorrect, status "+string(resp.StatusCode), err).Fatal() - } - if resp.Header.Get("ETag") == "" { - failureLog(function, args, startTime, "", "PresignedHeadObject response incorrect", err).Fatal() - } - resp.Body.Close() - - // Generate presigned GET object url. - presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second, nil) - function = "PresignedGetObject(bucketName, objectName, expires, reqParams)" - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "expires": 3600 * time.Second, - } - - if err != nil { - failureLog(function, args, startTime, "", "PresignedGetObject failed", err).Fatal() - } - - // Verify if presigned url works. - resp, err = http.Get(presignedGetURL.String()) - if err != nil { - failureLog(function, args, startTime, "", "PresignedGetObject response incorrect", err).Fatal() - } - if resp.StatusCode != http.StatusOK { - failureLog(function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err).Fatal() - } - newPresignedBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - failureLog(function, args, startTime, "", "PresignedGetObject response incorrect", err).Fatal() - } - resp.Body.Close() - if !bytes.Equal(newPresignedBytes, buf) { - failureLog(function, args, startTime, "", "PresignedGetObject response incorrect", err).Fatal() - } - - // Set request parameters. - reqParams := make(url.Values) - reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") - presignedGetURL, err = c.PresignedGetObject(bucketName, objectName, 3600*time.Second, reqParams) - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "expires": 3600 * time.Second, - "reqParams": reqParams, - } - - if err != nil { - failureLog(function, args, startTime, "", "PresignedGetObject failed", err).Fatal() - } - // Verify if presigned url works. - resp, err = http.Get(presignedGetURL.String()) - if err != nil { - failureLog(function, args, startTime, "", "PresignedGetObject response incorrect", err).Fatal() - } - if resp.StatusCode != http.StatusOK { - failureLog(function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err).Fatal() - } - newPresignedBytes, err = ioutil.ReadAll(resp.Body) - if err != nil { - failureLog(function, args, startTime, "", "PresignedGetObject response incorrect", err).Fatal() - } - if !bytes.Equal(newPresignedBytes, buf) { - failureLog(function, args, startTime, "", "Bytes mismatch for presigned GET URL", err).Fatal() - } - if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { - failureLog(function, args, startTime, "", "wrong Content-Disposition received "+string(resp.Header.Get("Content-Disposition")), err).Fatal() - } - - presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) - - function = "PresignedPutObject(bucketName, objectName, expires)" - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "expires": 3600 * time.Second, - } - - if err != nil { - failureLog(function, args, startTime, "", "PresignedPutObject failed", err).Fatal() - } - - buf = bytes.Repeat([]byte("g"), 1<<19) - - req, err := http.NewRequest("PUT", presignedPutURL.String(), bytes.NewReader(buf)) - if err != nil { - failureLog(function, args, startTime, "", "Couldn't make HTTP request with PresignedPutObject URL", err).Fatal() - } - httpClient := &http.Client{ - // Setting a sensible time out of 30secs to wait for response - // headers. Request is pro-actively cancelled after 30secs - // with no response. - Timeout: 30 * time.Second, - Transport: http.DefaultTransport, - } - resp, err = httpClient.Do(req) - if err != nil { - failureLog(function, args, startTime, "", "PresignedPutObject failed", err).Fatal() - } - - newReader, err = c.GetObject(bucketName, objectName+"-presigned") - if err != nil { - failureLog(function, args, startTime, "", "GetObject after PresignedPutObject failed", err).Fatal() - } - - newReadBytes, err = ioutil.ReadAll(newReader) - if err != nil { - failureLog(function, args, startTime, "", "ReadAll after GetObject failed", err).Fatal() - } - - if !bytes.Equal(newReadBytes, buf) { - failureLog(function, args, startTime, "", "Bytes mismatch", err).Fatal() - } - - err = c.RemoveObject(bucketName, objectName) - function = "RemoveObject(bucketName, objectName)" - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - } - - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - err = c.RemoveObject(bucketName, objectName+"-f") - args["objectName"] = objectName + "-f" - - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - - err = c.RemoveObject(bucketName, objectName+"-nolength") - args["objectName"] = objectName + "-nolength" - - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - - err = c.RemoveObject(bucketName, objectName+"-presigned") - args["objectName"] = objectName + "-presigned" - - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - - err = c.RemoveBucket(bucketName) - function = "RemoveBucket(bucketName)" - args = map[string]interface{}{ - "bucketName": bucketName, - } - - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - err = c.RemoveBucket(bucketName) - if err == nil { - failureLog(function, args, startTime, "", "RemoveBucket did not fail for invalid bucket name", err).Fatal() - } - if err.Error() != "The specified bucket does not exist" { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - if err = os.Remove(fileName); err != nil { - failureLog(function, args, startTime, "", "File Remove failed", err).Fatal() - } - if err = os.Remove(fileName + "-f"); err != nil { - failureLog(function, args, startTime, "", "File Remove failed", err).Fatal() - } - function = "testFunctional()" - successLogger(function, args, startTime).Info() -} - -// Test for validating GetObject Reader* methods functioning when the -// object is modified in the object store. -func testGetObjectObjectModified() { - // initialize logging params - startTime := time.Now() - function := "GetObject(bucketName, objectName)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - } - - // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Make a new bucket. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - defer c.RemoveBucket(bucketName) - - // Upload an object. - objectName := "myobject" - content := "helloworld" - _, err = c.PutObject(bucketName, objectName, strings.NewReader(content), int64(len(content)), minio.PutObjectOptions{ContentType: "application/text"}) - if err != nil { - failureLog(function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err).Fatal() - } - - defer c.RemoveObject(bucketName, objectName) - - reader, err := c.GetObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "Failed to GetObject "+objectName+", from bucket "+bucketName, err).Fatal() - } - defer reader.Close() - - // Read a few bytes of the object. - b := make([]byte, 5) - n, err := reader.ReadAt(b, 0) - if err != nil { - failureLog(function, args, startTime, "", "Failed to read object "+objectName+", from bucket "+bucketName+" at an offset", err).Fatal() - } - - // Upload different contents to the same object while object is being read. - newContent := "goodbyeworld" - _, err = c.PutObject(bucketName, objectName, strings.NewReader(newContent), int64(len(newContent)), minio.PutObjectOptions{ContentType: "application/text"}) - if err != nil { - failureLog(function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err).Fatal() - } - - // Confirm that a Stat() call in between doesn't change the Object's cached etag. - _, err = reader.Stat() - expectedError := "At least one of the pre-conditions you specified did not hold" - if err.Error() != expectedError { - failureLog(function, args, startTime, "", "Expected Stat to fail with error "+expectedError+", but received "+err.Error(), err).Fatal() - } - - // Read again only to find object contents have been modified since last read. - _, err = reader.ReadAt(b, int64(n)) - if err.Error() != expectedError { - failureLog(function, args, startTime, "", "Expected ReadAt to fail with error "+expectedError+", but received "+err.Error(), err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Test validates putObject to upload a file seeked at a given offset. -func testPutObjectUploadSeekedObject() { - // initialize logging params - startTime := time.Now() - function := "PutObject(bucketName, objectName, fileToUpload, contentType)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "fileToUpload": "", - "contentType": "binary/octet-stream", - } - - // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Make a new bucket. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - defer c.RemoveBucket(bucketName) - - tempfile, err := ioutil.TempFile("", "minio-go-upload-test-") - args["fileToUpload"] = tempfile - - if err != nil { - failureLog(function, args, startTime, "", "TempFile create failed", err).Fatal() - } - - var data []byte - if fileName := getFilePath("datafile-100-kB"); fileName != "" { - data, _ = ioutil.ReadFile(fileName) - } else { - // Generate 100kB data - data = bytes.Repeat([]byte("1"), 120000) - } - var length = len(data) - if _, err = tempfile.Write(data); err != nil { - failureLog(function, args, startTime, "", "TempFile write failed", err).Fatal() - } - - objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) - args["objectName"] = objectName - - offset := length / 2 - if _, err := tempfile.Seek(int64(offset), 0); err != nil { - failureLog(function, args, startTime, "", "TempFile seek failed", err).Fatal() - } - - n, err := c.PutObject(bucketName, objectName, tempfile, int64(length), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - } - if n != int64(length-offset) { - failureLog(function, args, startTime, "", "Invalid length returned, expected "+string(int64(length-offset))+" got "+string(n), err).Fatal() - } - tempfile.Close() - if err = os.Remove(tempfile.Name()); err != nil { - failureLog(function, args, startTime, "", "File remove failed", err).Fatal() - } - - length = int(n) - - obj, err := c.GetObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "GetObject failed", err).Fatal() - } - - n, err = obj.Seek(int64(offset), 0) - if err != nil { - failureLog(function, args, startTime, "", "Seek failed", err).Fatal() - } - if n != int64(offset) { - failureLog(function, args, startTime, "", "Invalid offset returned, expected "+string(int64(offset))+" got "+string(n), err).Fatal() - } - - n, err = c.PutObject(bucketName, objectName+"getobject", obj, int64(length), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "GetObject failed", err).Fatal() - } - if n != int64(length-offset) { - failureLog(function, args, startTime, "", "Invalid offset returned, expected "+string(int64(length-offset))+" got "+string(n), err).Fatal() - } - - if err = c.RemoveObject(bucketName, objectName); err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - - if err = c.RemoveObject(bucketName, objectName+"getobject"); err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - - if err = c.RemoveBucket(bucketName); err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Tests bucket re-create errors. -func testMakeBucketErrorV2() { - // initialize logging params - startTime := time.Now() - function := "MakeBucket(bucketName, region)" - args := map[string]interface{}{ - "bucketName": "", - "region": "eu-west-1", - } - - if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { - ignoredLog(function, args, startTime, "Skipped region functional tests for non s3 runs").Info() - return - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio v2 client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - // Make a new bucket in 'eu-west-1'. - if err = c.MakeBucket(bucketName, "eu-west-1"); err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - if err = c.MakeBucket(bucketName, "eu-west-1"); err == nil { - failureLog(function, args, startTime, "", "MakeBucket did not fail for existing bucket name", err).Fatal() - } - // Verify valid error response from server. - if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" && - minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { - failureLog(function, args, startTime, "", "Invalid error returned by server", err).Fatal() - } - if err = c.RemoveBucket(bucketName); err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Test get object reader to not throw error on being closed twice. -func testGetObjectClosedTwiceV2() { - // initialize logging params - startTime := time.Now() - function := "MakeBucket(bucketName, region)" - args := map[string]interface{}{ - "bucketName": "", - "region": "eu-west-1", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio v2 client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Generate 33K of data. - var reader = getDataReader("datafile-33-kB", thirtyThreeKiB) - defer reader.Close() - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - n, err := c.PutObject(bucketName, objectName, reader, int64(thirtyThreeKiB), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - } - - if n != int64(thirtyThreeKiB) { - failureLog(function, args, startTime, "", "Number of bytes does not match, expected "+string(thirtyThreeKiB)+" got "+string(n), err).Fatal() - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "GetObject failed", err).Fatal() - } - - st, err := r.Stat() - if err != nil { - failureLog(function, args, startTime, "", "Stat failed", err).Fatal() - } - - if st.Size != int64(thirtyThreeKiB) { - failureLog(function, args, startTime, "", "Number of bytes does not match, expected "+string(thirtyThreeKiB)+" got "+string(st.Size), err).Fatal() - } - if err := r.Close(); err != nil { - failureLog(function, args, startTime, "", "Stat failed", err).Fatal() - } - if err := r.Close(); err == nil { - failureLog(function, args, startTime, "", "Object is already closed, should return error", err).Fatal() - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Tests removing partially uploaded objects. -func testRemovePartiallyUploadedV2() { - // initialize logging params - startTime := time.Now() - function := "RemoveIncompleteUpload(bucketName, objectName)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio v2 client object creation failed", err).Fatal() - } - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Enable tracing, write to stdout. - // c.TraceOn(os.Stderr) - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - // make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - r := bytes.NewReader(bytes.Repeat([]byte("a"), 128*1024)) - - reader, writer := io.Pipe() - go func() { - i := 0 - for i < 25 { - _, cerr := io.CopyN(writer, r, 128*1024) - if cerr != nil { - failureLog(function, args, startTime, "", "Copy failed", cerr).Fatal() - } - i++ - r.Seek(0, 0) - } - writer.CloseWithError(errors.New("proactively closed to be verified later")) - }() - - objectName := bucketName + "-resumable" - args["objectName"] = objectName - - _, err = c.PutObject(bucketName, objectName, reader, -1, minio.PutObjectOptions{ContentType: "application/octet-stream"}) - if err == nil { - failureLog(function, args, startTime, "", "PutObject should fail", err).Fatal() - } - if err.Error() != "proactively closed to be verified later" { - failureLog(function, args, startTime, "", "Unexpected error, expected : proactively closed to be verified later", err).Fatal() - } - err = c.RemoveIncompleteUpload(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveIncompleteUpload failed", err).Fatal() - } - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Tests FPutObject hidden contentType setting -func testFPutObjectV2() { - // initialize logging params - startTime := time.Now() - function := "FPutObject(bucketName, objectName, fileName, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "fileName": "", - "opts": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio v2 client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Make a temp file with 11*1024*1024 bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") - if err != nil { - failureLog(function, args, startTime, "", "TempFile creation failed", err).Fatal() - } - - r := bytes.NewReader(bytes.Repeat([]byte("b"), 11*1024*1024)) - n, err := io.CopyN(file, r, 11*1024*1024) - if err != nil { - failureLog(function, args, startTime, "", "Copy failed", err).Fatal() - } - if n != int64(11*1024*1024) { - failureLog(function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err).Fatal() - } - - // Close the file pro-actively for windows. - err = file.Close() - if err != nil { - failureLog(function, args, startTime, "", "File close failed", err).Fatal() - } - - // Set base object name - objectName := bucketName + "FPutObject" - args["objectName"] = objectName - args["fileName"] = file.Name() - - // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) - n, err = c.FPutObject(bucketName, objectName+"-standard", file.Name(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "FPutObject failed", err).Fatal() - } - if n != int64(11*1024*1024) { - failureLog(function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err).Fatal() - } - - // Perform FPutObject with no contentType provided (Expecting application/octet-stream) - args["objectName"] = objectName + "-Octet" - args["contentType"] = "" - - n, err = c.FPutObject(bucketName, objectName+"-Octet", file.Name(), minio.PutObjectOptions{}) - if err != nil { - failureLog(function, args, startTime, "", "FPutObject failed", err).Fatal() - } - if n != int64(11*1024*1024) { - failureLog(function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err).Fatal() - } - - // Add extension to temp file name - fileName := file.Name() - err = os.Rename(file.Name(), fileName+".gtar") - if err != nil { - failureLog(function, args, startTime, "", "Rename failed", err).Fatal() - } - - // Perform FPutObject with no contentType provided (Expecting application/x-gtar) - args["objectName"] = objectName + "-Octet" - args["contentType"] = "" - args["fileName"] = fileName + ".gtar" - - n, err = c.FPutObject(bucketName, objectName+"-GTar", fileName+".gtar", minio.PutObjectOptions{}) - if err != nil { - failureLog(function, args, startTime, "", "FPutObject failed", err).Fatal() - } - if n != int64(11*1024*1024) { - failureLog(function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err).Fatal() - } - - // Check headers - rStandard, err := c.StatObject(bucketName, objectName+"-standard") - if err != nil { - failureLog(function, args, startTime, "", "StatObject failed", err).Fatal() - } - if rStandard.ContentType != "application/octet-stream" { - failureLog(function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rStandard.ContentType, err).Fatal() - } - - rOctet, err := c.StatObject(bucketName, objectName+"-Octet") - if err != nil { - failureLog(function, args, startTime, "", "StatObject failed", err).Fatal() - } - if rOctet.ContentType != "application/octet-stream" { - failureLog(function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rOctet.ContentType, err).Fatal() - } - - rGTar, err := c.StatObject(bucketName, objectName+"-GTar") - if err != nil { - failureLog(function, args, startTime, "", "StatObject failed", err).Fatal() - } - if rGTar.ContentType != "application/x-gtar" { - failureLog(function, args, startTime, "", "Content-Type headers mismatched, expected: application/x-gtar , got "+rGTar.ContentType, err).Fatal() - } - - // Remove all objects and bucket and temp file - err = c.RemoveObject(bucketName, objectName+"-standard") - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - - err = c.RemoveObject(bucketName, objectName+"-Octet") - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - - err = c.RemoveObject(bucketName, objectName+"-GTar") - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - - err = os.Remove(fileName + ".gtar") - if err != nil { - failureLog(function, args, startTime, "", "File remove failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Tests various bucket supported formats. -func testMakeBucketRegionsV2() { - // initialize logging params - startTime := time.Now() - function := "MakeBucket(bucketName, region)" - args := map[string]interface{}{ - "bucketName": "", - "region": "eu-west-1", - } - - if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { - ignoredLog(function, args, startTime, "Skipped region functional tests for non s3 runs").Info() - return - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio v2 client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - // Make a new bucket in 'eu-central-1'. - if err = c.MakeBucket(bucketName, "eu-west-1"); err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - if err = c.RemoveBucket(bucketName); err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - - // Make a new bucket with '.' in its name, in 'us-west-2'. This - // request is internally staged into a path style instead of - // virtual host style. - if err = c.MakeBucket(bucketName+".withperiod", "us-west-2"); err != nil { - args["bucketName"] = bucketName + ".withperiod" - args["region"] = "us-west-2" - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Remove the newly created bucket. - if err = c.RemoveBucket(bucketName + ".withperiod"); err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Tests get object ReaderSeeker interface methods. -func testGetObjectReadSeekFunctionalV2() { - // initialize logging params - startTime := time.Now() - function := "GetObject(bucketName, objectName)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio v2 client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Generate 33K of data. - var reader = getDataReader("datafile-33-kB", thirtyThreeKiB) - defer reader.Close() - - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - buf, err := ioutil.ReadAll(reader) - if err != nil { - failureLog(function, args, startTime, "", "ReadAll failed", err).Fatal() - } - - // Save the data. - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(thirtyThreeKiB), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - } - - if n != int64(thirtyThreeKiB) { - failureLog(function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(thirtyThreeKiB))+" got "+string(n), err).Fatal() - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "GetObject failed", err).Fatal() - } - - st, err := r.Stat() - if err != nil { - failureLog(function, args, startTime, "", "Stat failed", err).Fatal() - } - - if st.Size != int64(thirtyThreeKiB) { - failureLog(function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(thirtyThreeKiB))+" got "+string(st.Size), err).Fatal() - } - - offset := int64(2048) - n, err = r.Seek(offset, 0) - if err != nil { - failureLog(function, args, startTime, "", "Seek failed", err).Fatal() - } - if n != offset { - failureLog(function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err).Fatal() - } - n, err = r.Seek(0, 1) - if err != nil { - failureLog(function, args, startTime, "", "Seek failed", err).Fatal() - } - if n != offset { - failureLog(function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err).Fatal() - } - _, err = r.Seek(offset, 2) - if err == nil { - failureLog(function, args, startTime, "", "Seek on positive offset for whence '2' should error out", err).Fatal() - } - n, err = r.Seek(-offset, 2) - if err != nil { - failureLog(function, args, startTime, "", "Seek failed", err).Fatal() - } - if n != st.Size-offset { - failureLog(function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(st.Size-offset)+" got "+string(n), err).Fatal() - } - - var buffer1 bytes.Buffer - if _, err = io.CopyN(&buffer1, r, st.Size); err != nil { - if err != io.EOF { - failureLog(function, args, startTime, "", "Copy failed", err).Fatal() - } - } - if !bytes.Equal(buf[len(buf)-int(offset):], buffer1.Bytes()) { - failureLog(function, args, startTime, "", "Incorrect read bytes v/s original buffer", err).Fatal() - } - - // Seek again and read again. - n, err = r.Seek(offset-1, 0) - if err != nil { - failureLog(function, args, startTime, "", "Seek failed", err).Fatal() - } - if n != (offset - 1) { - failureLog(function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset-1)+" got "+string(n), err).Fatal() - } - - var buffer2 bytes.Buffer - if _, err = io.CopyN(&buffer2, r, st.Size); err != nil { - if err != io.EOF { - failureLog(function, args, startTime, "", "Copy failed", err).Fatal() - } - } - // Verify now lesser bytes. - if !bytes.Equal(buf[2047:], buffer2.Bytes()) { - failureLog(function, args, startTime, "", "Incorrect read bytes v/s original buffer", err).Fatal() - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Tests get object ReaderAt interface methods. -func testGetObjectReadAtFunctionalV2() { - // initialize logging params - startTime := time.Now() - function := "GetObject(bucketName, objectName)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio v2 client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Generate 33K of data. - var reader = getDataReader("datafile-33-kB", thirtyThreeKiB) - defer reader.Close() - - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - buf, err := ioutil.ReadAll(reader) - if err != nil { - failureLog(function, args, startTime, "", "ReadAll failed", err).Fatal() - } - - // Save the data - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(thirtyThreeKiB), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - } - - if n != int64(thirtyThreeKiB) { - failureLog(function, args, startTime, "", "Number of bytes does not match, expected "+string(thirtyThreeKiB)+" got "+string(n), err).Fatal() - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "GetObject failed", err).Fatal() - } - - st, err := r.Stat() - if err != nil { - failureLog(function, args, startTime, "", "Stat failed", err).Fatal() - } - - if st.Size != int64(thirtyThreeKiB) { - failureLog(function, args, startTime, "", "Number of bytes does not match, expected "+string(thirtyThreeKiB)+" got "+string(st.Size), err).Fatal() - } - - offset := int64(2048) - - // Read directly - buf2 := make([]byte, 512) - buf3 := make([]byte, 512) - buf4 := make([]byte, 512) - - m, err := r.ReadAt(buf2, offset) - if err != nil { - failureLog(function, args, startTime, "", "ReadAt failed", err).Fatal() - } - if m != len(buf2) { - failureLog(function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+" got "+string(m), err).Fatal() - } - if !bytes.Equal(buf2, buf[offset:offset+512]) { - failureLog(function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err).Fatal() - } - offset += 512 - m, err = r.ReadAt(buf3, offset) - if err != nil { - failureLog(function, args, startTime, "", "ReadAt failed", err).Fatal() - } - if m != len(buf3) { - failureLog(function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+" got "+string(m), err).Fatal() - } - if !bytes.Equal(buf3, buf[offset:offset+512]) { - failureLog(function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err).Fatal() - } - offset += 512 - m, err = r.ReadAt(buf4, offset) - if err != nil { - failureLog(function, args, startTime, "", "ReadAt failed", err).Fatal() - } - if m != len(buf4) { - failureLog(function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+" got "+string(m), err).Fatal() - } - if !bytes.Equal(buf4, buf[offset:offset+512]) { - failureLog(function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err).Fatal() - } - - buf5 := make([]byte, n) - // Read the whole object. - m, err = r.ReadAt(buf5, 0) - if err != nil { - if err != io.EOF { - failureLog(function, args, startTime, "", "ReadAt failed", err).Fatal() - } - } - if m != len(buf5) { - failureLog(function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+" got "+string(m), err).Fatal() - } - if !bytes.Equal(buf, buf5) { - failureLog(function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err).Fatal() - } - - buf6 := make([]byte, n+1) - // Read the whole object and beyond. - _, err = r.ReadAt(buf6, 0) - if err != nil { - if err != io.EOF { - failureLog(function, args, startTime, "", "ReadAt failed", err).Fatal() - } - } - err = c.RemoveObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Tests copy object -func testCopyObjectV2() { - // initialize logging params - startTime := time.Now() - function := "CopyObject(destination, source)" - args := map[string]interface{}{ - "destination": "", - "source": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio v2 client object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Make a new bucket in 'us-east-1' (destination bucket). - err = c.MakeBucket(bucketName+"-copy", "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Generate 33K of data. - var reader = getDataReader("datafile-33-kB", thirtyThreeKiB) - defer reader.Close() - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, reader, int64(thirtyThreeKiB), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - } - - if n != int64(thirtyThreeKiB) { - failureLog(function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(thirtyThreeKiB))+" got "+string(n), err).Fatal() - } - - r, err := c.GetObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "GetObject failed", err).Fatal() - } - // Check the various fields of source object against destination object. - objInfo, err := r.Stat() - if err != nil { - failureLog(function, args, startTime, "", "Stat failed", err).Fatal() - } - - // Copy Source - src := minio.NewSourceInfo(bucketName, objectName, nil) - - // Set copy conditions. - - // All invalid conditions first. - err = src.SetModifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) - if err == nil { - failureLog(function, args, startTime, "", "SetModifiedSinceCond did not fail for invalid conditions", err).Fatal() - } - err = src.SetUnmodifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) - if err == nil { - failureLog(function, args, startTime, "", "SetUnmodifiedSinceCond did not fail for invalid conditions", err).Fatal() - } - err = src.SetMatchETagCond("") - if err == nil { - failureLog(function, args, startTime, "", "SetMatchETagCond did not fail for invalid conditions", err).Fatal() - } - err = src.SetMatchETagExceptCond("") - if err == nil { - failureLog(function, args, startTime, "", "SetMatchETagExceptCond did not fail for invalid conditions", err).Fatal() - } - - err = src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - if err != nil { - failureLog(function, args, startTime, "", "SetModifiedSinceCond failed", err).Fatal() - } - err = src.SetMatchETagCond(objInfo.ETag) - if err != nil { - failureLog(function, args, startTime, "", "SetMatchETagCond failed", err).Fatal() - } - args["source"] = src - - dst, err := minio.NewDestinationInfo(bucketName+"-copy", objectName+"-copy", nil, nil) - if err != nil { - failureLog(function, args, startTime, "", "NewDestinationInfo failed", err).Fatal() - } - args["destination"] = dst - - // Perform the Copy - err = c.CopyObject(dst, src) - if err != nil { - failureLog(function, args, startTime, "", "CopyObject failed", err).Fatal() - } - - // Source object - r, err = c.GetObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "GetObject failed", err).Fatal() - } - // Destination object - readerCopy, err := c.GetObject(bucketName+"-copy", objectName+"-copy") - if err != nil { - failureLog(function, args, startTime, "", "GetObject failed", err).Fatal() - } - // Check the various fields of source object against destination object. - objInfo, err = r.Stat() - if err != nil { - failureLog(function, args, startTime, "", "Stat failed", err).Fatal() - } - objInfoCopy, err := readerCopy.Stat() - if err != nil { - failureLog(function, args, startTime, "", "Stat failed", err).Fatal() - } - if objInfo.Size != objInfoCopy.Size { - failureLog(function, args, startTime, "", "Number of bytes does not match, expected "+string(objInfoCopy.Size)+" got "+string(objInfo.Size), err).Fatal() - } - - // CopyObject again but with wrong conditions - src = minio.NewSourceInfo(bucketName, objectName, nil) - err = src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - if err != nil { - failureLog(function, args, startTime, "", "SetUnmodifiedSinceCond failed", err).Fatal() - } - err = src.SetMatchETagExceptCond(objInfo.ETag) - if err != nil { - failureLog(function, args, startTime, "", "SetMatchETagExceptCond failed", err).Fatal() - } - - // Perform the Copy which should fail - err = c.CopyObject(dst, src) - if err == nil { - failureLog(function, args, startTime, "", "CopyObject did not fail for invalid conditions", err).Fatal() - } - - // Remove all objects and buckets - err = c.RemoveObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - - err = c.RemoveObject(bucketName+"-copy", objectName+"-copy") - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - - err = c.RemoveBucket(bucketName + "-copy") - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -func testComposeObjectErrorCasesWrapper(c *minio.Client) { - // initialize logging params - startTime := time.Now() - function := "testComposeObjectErrorCasesWrapper(minioClient)" - args := map[string]interface{}{} - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(bucketName, "us-east-1") - - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Test that more than 10K source objects cannot be - // concatenated. - srcArr := [10001]minio.SourceInfo{} - srcSlice := srcArr[:] - dst, err := minio.NewDestinationInfo(bucketName, "object", nil, nil) - if err != nil { - failureLog(function, args, startTime, "", "NewDestinationInfo failed", err).Fatal() - } - - if err := c.ComposeObject(dst, srcSlice); err == nil { - failureLog(function, args, startTime, "", "Expected error in ComposeObject", err).Fatal() - } else if err.Error() != "There must be as least one and up to 10000 source objects." { - failureLog(function, args, startTime, "", "Got unexpected error", err).Fatal() - } - - // Create a source with invalid offset spec and check that - // error is returned: - // 1. Create the source object. - const badSrcSize = 5 * 1024 * 1024 - buf := bytes.Repeat([]byte("1"), badSrcSize) - _, err = c.PutObject(bucketName, "badObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - } - // 2. Set invalid range spec on the object (going beyond - // object size) - badSrc := minio.NewSourceInfo(bucketName, "badObject", nil) - err = badSrc.SetRange(1, badSrcSize) - if err != nil { - failureLog(function, args, startTime, "", "Setting NewSourceInfo failed", err).Fatal() - } - // 3. ComposeObject call should fail. - if err := c.ComposeObject(dst, []minio.SourceInfo{badSrc}); err == nil { - failureLog(function, args, startTime, "", "ComposeObject expected to fail", err).Fatal() - } else if !strings.Contains(err.Error(), "has invalid segment-to-copy") { - failureLog(function, args, startTime, "", "Got invalid error", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Test expected error cases -func testComposeObjectErrorCasesV2() { - // initialize logging params - startTime := time.Now() - function := "testComposeObjectErrorCasesV2()" - args := map[string]interface{}{} - - // Instantiate new minio client object - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio v2 client object creation failed", err).Fatal() - } - - testComposeObjectErrorCasesWrapper(c) -} - -func testComposeMultipleSources(c *minio.Client) { - // initialize logging params - startTime := time.Now() - function := "ComposeObject(destination, sources)" - args := map[string]interface{}{ - "destination": "", - "sources": "", - } - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Upload a small source object - const srcSize = 1024 * 1024 * 5 - buf := bytes.Repeat([]byte("1"), srcSize) - _, err = c.PutObject(bucketName, "srcObject", bytes.NewReader(buf), int64(srcSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - } - - // We will append 10 copies of the object. - srcs := []minio.SourceInfo{} - for i := 0; i < 10; i++ { - srcs = append(srcs, minio.NewSourceInfo(bucketName, "srcObject", nil)) - } - // make the last part very small - err = srcs[9].SetRange(0, 0) - if err != nil { - failureLog(function, args, startTime, "", "SetRange failed", err).Fatal() - } - args["sources"] = srcs - - dst, err := minio.NewDestinationInfo(bucketName, "dstObject", nil, nil) - args["destination"] = dst - - if err != nil { - failureLog(function, args, startTime, "", "NewDestinationInfo failed", err).Fatal() - } - err = c.ComposeObject(dst, srcs) - if err != nil { - failureLog(function, args, startTime, "", "ComposeObject failed", err).Fatal() - } - - objProps, err := c.StatObject(bucketName, "dstObject") - if err != nil { - failureLog(function, args, startTime, "", "StatObject failed", err).Fatal() - } - - if objProps.Size != 9*srcSize+1 { - failureLog(function, args, startTime, "", "Size mismatched! Expected "+string(10000*srcSize)+" got "+string(objProps.Size), err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Test concatenating multiple objects objects -func testCompose10KSourcesV2() { - // initialize logging params - startTime := time.Now() - function := "testCompose10KSourcesV2(minioClient)" - args := map[string]interface{}{} - - // Instantiate new minio client object - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio v2 client object creation failed", err).Fatal() - } - - testComposeMultipleSources(c) -} - -func testEncryptedCopyObjectWrapper(c *minio.Client) { - // initialize logging params - startTime := time.Now() - function := "testEncryptedCopyObjectWrapper(minioClient)" - args := map[string]interface{}{} - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - key1 := minio.NewSSEInfo([]byte("32byteslongsecretkeymustbegiven1"), "AES256") - key2 := minio.NewSSEInfo([]byte("32byteslongsecretkeymustbegiven2"), "AES256") - - // 1. create an sse-c encrypted object to copy by uploading - const srcSize = 1024 * 1024 - buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB - metadata := make(map[string]string) - for k, v := range key1.GetSSEHeaders() { - metadata[k] = v - } - _, err = c.PutObject(bucketName, "srcObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{UserMetadata: metadata, Progress: nil}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject call failed", err).Fatal() - } - - // 2. copy object and change encryption key - src := minio.NewSourceInfo(bucketName, "srcObject", &key1) - dst, err := minio.NewDestinationInfo(bucketName, "dstObject", &key2, nil) - if err != nil { - failureLog(function, args, startTime, "", "NewDestinationInfo failed", err).Fatal() - } - - err = c.CopyObject(dst, src) - if err != nil { - failureLog(function, args, startTime, "", "CopyObject failed", err).Fatal() - } - - // 3. get copied object and check if content is equal - reqH := minio.NewGetReqHeaders() - for k, v := range key2.GetSSEHeaders() { - reqH.Set(k, v) - } - coreClient := minio.Core{c} - reader, _, err := coreClient.GetObject(bucketName, "dstObject", reqH) - if err != nil { - failureLog(function, args, startTime, "", "GetObject failed", err).Fatal() - } - defer reader.Close() - - decBytes, err := ioutil.ReadAll(reader) - if err != nil { - failureLog(function, args, startTime, "", "ReadAll failed", err).Fatal() - } - if !bytes.Equal(decBytes, buf) { - failureLog(function, args, startTime, "", "Downloaded object mismatched for encrypted object", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Test encrypted copy object -func testEncryptedCopyObject() { - // initialize logging params - startTime := time.Now() - function := "testEncryptedCopyObject()" - args := map[string]interface{}{} - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio v2 client object creation failed", err).Fatal() - } - - // c.TraceOn(os.Stderr) - testEncryptedCopyObjectWrapper(c) -} - -// Test encrypted copy object -func testEncryptedCopyObjectV2() { - // initialize logging params - startTime := time.Now() - function := "testEncryptedCopyObjectV2()" - args := map[string]interface{}{} - - // Instantiate new minio client object - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio v2 client object creation failed", err).Fatal() - } - - testEncryptedCopyObjectWrapper(c) -} - -func testUserMetadataCopying() { - // initialize logging params - startTime := time.Now() - function := "testUserMetadataCopying()" - args := map[string]interface{}{} - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - // c.TraceOn(os.Stderr) - testUserMetadataCopyingWrapper(c) -} - -func testUserMetadataCopyingWrapper(c *minio.Client) { - // initialize logging params - startTime := time.Now() - function := "CopyObject(destination, source)" - args := map[string]interface{}{ - "destination": "", - "source": "", - } - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - fetchMeta := func(object string) (h http.Header) { - objInfo, err := c.StatObject(bucketName, object) - if err != nil { - failureLog(function, args, startTime, "", "Stat failed", err).Fatal() - } - h = make(http.Header) - for k, vs := range objInfo.Metadata { - if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") { - for _, v := range vs { - h.Add(k, v) - } - } - } - return h - } - - // 1. create a client encrypted object to copy by uploading - const srcSize = 1024 * 1024 - buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB - metadata := make(http.Header) - metadata.Set("x-amz-meta-myheader", "myvalue") - m := make(map[string]string) - m["x-amz-meta-myheader"] = "myvalue" - _, err = c.PutObject(bucketName, "srcObject", - bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{UserMetadata: m}) - if err != nil { - failureLog(function, args, startTime, "", "PutObjectWithMetadata failed", err).Fatal() - } - if !reflect.DeepEqual(metadata, fetchMeta("srcObject")) { - failureLog(function, args, startTime, "", "Metadata match failed", err).Fatal() - } - - // 2. create source - src := minio.NewSourceInfo(bucketName, "srcObject", nil) - // 2.1 create destination with metadata set - dst1, err := minio.NewDestinationInfo(bucketName, "dstObject-1", nil, map[string]string{"notmyheader": "notmyvalue"}) - if err != nil { - failureLog(function, args, startTime, "", "NewDestinationInfo failed", err).Fatal() - } - - // 3. Check that copying to an object with metadata set resets - // the headers on the copy. - err = c.CopyObject(dst1, src) - args["destination"] = dst1 - args["source"] = src - - if err != nil { - failureLog(function, args, startTime, "", "CopyObject failed", err).Fatal() - } - - expectedHeaders := make(http.Header) - expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue") - if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-1")) { - failureLog(function, args, startTime, "", "Metadata match failed", err).Fatal() - } - - // 4. create destination with no metadata set and same source - dst2, err := minio.NewDestinationInfo(bucketName, "dstObject-2", nil, nil) - if err != nil { - failureLog(function, args, startTime, "", "NewDestinationInfo failed", err).Fatal() - - } - src = minio.NewSourceInfo(bucketName, "srcObject", nil) - - // 5. Check that copying to an object with no metadata set, - // copies metadata. - err = c.CopyObject(dst2, src) - args["destination"] = dst2 - args["source"] = src - - if err != nil { - failureLog(function, args, startTime, "", "CopyObject failed", err).Fatal() - } - - expectedHeaders = metadata - if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-2")) { - failureLog(function, args, startTime, "", "Metadata match failed", err).Fatal() - } - - // 6. Compose a pair of sources. - srcs := []minio.SourceInfo{ - minio.NewSourceInfo(bucketName, "srcObject", nil), - minio.NewSourceInfo(bucketName, "srcObject", nil), - } - dst3, err := minio.NewDestinationInfo(bucketName, "dstObject-3", nil, nil) - if err != nil { - failureLog(function, args, startTime, "", "NewDestinationInfo failed", err).Fatal() - } - - err = c.ComposeObject(dst3, srcs) - function = "ComposeObject(destination, sources)" - args["destination"] = dst3 - args["source"] = srcs - - if err != nil { - failureLog(function, args, startTime, "", "ComposeObject failed", err).Fatal() - } - - // Check that no headers are copied in this case - if !reflect.DeepEqual(make(http.Header), fetchMeta("dstObject-3")) { - failureLog(function, args, startTime, "", "Metadata match failed", err).Fatal() - } - - // 7. Compose a pair of sources with dest user metadata set. - srcs = []minio.SourceInfo{ - minio.NewSourceInfo(bucketName, "srcObject", nil), - minio.NewSourceInfo(bucketName, "srcObject", nil), - } - dst4, err := minio.NewDestinationInfo(bucketName, "dstObject-4", nil, map[string]string{"notmyheader": "notmyvalue"}) - if err != nil { - failureLog(function, args, startTime, "", "NewDestinationInfo failed", err).Fatal() - } - - err = c.ComposeObject(dst4, srcs) - function = "ComposeObject(destination, sources)" - args["destination"] = dst4 - args["source"] = srcs - - if err != nil { - failureLog(function, args, startTime, "", "ComposeObject failed", err).Fatal() - } - - // Check that no headers are copied in this case - expectedHeaders = make(http.Header) - expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue") - if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-4")) { - failureLog(function, args, startTime, "", "Metadata match failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -func testUserMetadataCopyingV2() { - // initialize logging params - startTime := time.Now() - function := "testUserMetadataCopyingV2()" - args := map[string]interface{}{} - - // Instantiate new minio client object - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client v2 object creation failed", err).Fatal() - } - - // c.TraceOn(os.Stderr) - testUserMetadataCopyingWrapper(c) -} - -// Test put object with size -1 byte object. -func testPutObjectNoLengthV2() { - // initialize logging params - startTime := time.Now() - function := "PutObject(bucketName, objectName, reader, size, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "size": -1, - "opts": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client v2 object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), - "minio-go-test") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - objectName := bucketName + "unique" - args["objectName"] = objectName - - // Generate data using 4 parts so that all 3 'workers' are utilized and a part is leftover. - // Use different data for each part for multipart tests to ensure part order at the end. - var reader = getDataReader("datafile-65-MB", sixtyFiveMiB) - defer reader.Close() - - // Upload an object. - n, err := c.PutObject(bucketName, objectName, reader, -1, minio.PutObjectOptions{}) - - if err != nil { - failureLog(function, args, startTime, "", "PutObjectWithSize failed", err).Fatal() - } - if n != int64(sixtyFiveMiB) { - failureLog(function, args, startTime, "", "Expected upload object size "+string(sixtyFiveMiB)+" got "+string(n), err).Fatal() - } - - // Remove the object. - err = c.RemoveObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - - // Remove the bucket. - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Test put objects of unknown size. -func testPutObjectsUnknownV2() { - // initialize logging params - startTime := time.Now() - function := "PutObject(bucketName, objectName, reader,size,opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "size": "", - "opts": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client v2 object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), - "minio-go-test") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Issues are revealed by trying to upload multiple files of unknown size - // sequentially (on 4GB machines) - for i := 1; i <= 4; i++ { - // Simulate that we could be receiving byte slices of data that we want - // to upload as a file - rpipe, wpipe := io.Pipe() - defer rpipe.Close() - go func() { - b := []byte("test") - wpipe.Write(b) - wpipe.Close() - }() - - // Upload the object. - objectName := fmt.Sprintf("%sunique%d", bucketName, i) - args["objectName"] = objectName - - n, err := c.PutObject(bucketName, objectName, rpipe, -1, minio.PutObjectOptions{}) - if err != nil { - failureLog(function, args, startTime, "", "PutObjectStreaming failed", err).Fatal() - } - if n != int64(4) { - failureLog(function, args, startTime, "", "Expected upload object size "+string(4)+" got "+string(n), err).Fatal() - } - - // Remove the object. - err = c.RemoveObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - } - - // Remove the bucket. - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Test put object with 0 byte object. -func testPutObject0ByteV2() { - // initialize logging params - startTime := time.Now() - function := "PutObject(bucketName, objectName, reader, size, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "size": 0, - "opts": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client v2 object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), - "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - objectName := bucketName + "unique" - - // Upload an object. - n, err := c.PutObject(bucketName, objectName, bytes.NewReader([]byte("")), 0, minio.PutObjectOptions{}) - - if err != nil { - failureLog(function, args, startTime, "", "PutObjectWithSize failed", err).Fatal() - } - if n != 0 { - failureLog(function, args, startTime, "", "Expected upload object size 0 but got "+string(n), err).Fatal() - } - - // Remove the object. - err = c.RemoveObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - - // Remove the bucket. - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Test expected error cases -func testComposeObjectErrorCases() { - // initialize logging params - startTime := time.Now() - function := "testComposeObjectErrorCases()" - args := map[string]interface{}{} - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - testComposeObjectErrorCasesWrapper(c) -} - -// Test concatenating 10K objects -func testCompose10KSources() { - // initialize logging params - startTime := time.Now() - function := "testCompose10KSources()" - args := map[string]interface{}{} - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client object creation failed", err).Fatal() - } - - testComposeMultipleSources(c) -} - -// Tests comprehensive list of all methods. -func testFunctionalV2() { - // initialize logging params - startTime := time.Now() - function := "testFunctionalV2()" - args := map[string]interface{}{} - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client v2 object creation failed", err).Fatal() - } - - // Enable to debug - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Generate a random file name. - fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - file, err := os.Create(fileName) - if err != nil { - failureLog(function, args, startTime, "", "file create failed", err).Fatal() - } - for i := 0; i < 3; i++ { - buf := make([]byte, rand.Intn(1<<19)) - _, err = file.Write(buf) - if err != nil { - failureLog(function, args, startTime, "", "file write failed", err).Fatal() - } - } - file.Close() - - // Verify if bucket exits and you have access. - var exists bool - exists, err = c.BucketExists(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "BucketExists failed", err).Fatal() - } - if !exists { - failureLog(function, args, startTime, "", "Could not find existing bucket "+bucketName, err).Fatal() - } - - // Make the bucket 'public read/write'. - err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyReadWrite) - if err != nil { - failureLog(function, args, startTime, "", "SetBucketPolicy failed", err).Fatal() - } - - // List all buckets. - buckets, err := c.ListBuckets() - if len(buckets) == 0 { - failureLog(function, args, startTime, "", "List buckets cannot be empty", err).Fatal() - } - if err != nil { - failureLog(function, args, startTime, "", "ListBuckets failed", err).Fatal() - } - - // Verify if previously created bucket is listed in list buckets. - bucketFound := false - for _, bucket := range buckets { - if bucket.Name == bucketName { - bucketFound = true - } - } - - // If bucket not found error out. - if !bucketFound { - failureLog(function, args, startTime, "", "Bucket "+bucketName+"not found", err).Fatal() - } - - objectName := bucketName + "unique" - - // Generate data - buf := bytes.Repeat([]byte("n"), rand.Intn(1<<19)) - - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - } - if n != int64(len(buf)) { - failureLog(function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(n), err).Fatal() - } - - n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - } - - if n != int64(len(buf)) { - failureLog(function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(n), err).Fatal() - } - - // Instantiate a done channel to close all listing. - doneCh := make(chan struct{}) - defer close(doneCh) - - objFound := false - isRecursive := true // Recursive is true. - for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { - if obj.Key == objectName { - objFound = true - break - } - } - if !objFound { - failureLog(function, args, startTime, "", "Could not find existing object "+objectName, err).Fatal() - } - - objFound = false - isRecursive = true // Recursive is true. - for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { - if obj.Key == objectName { - objFound = true - break - } - } - if !objFound { - failureLog(function, args, startTime, "", "Could not find existing object "+objectName, err).Fatal() - } - - incompObjNotFound := true - for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { - if objIncompl.Key != "" { - incompObjNotFound = false - break - } - } - if !incompObjNotFound { - failureLog(function, args, startTime, "", "Unexpected dangling incomplete upload found", err).Fatal() - } - - newReader, err := c.GetObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "GetObject failed", err).Fatal() - } - - newReadBytes, err := ioutil.ReadAll(newReader) - if err != nil { - failureLog(function, args, startTime, "", "ReadAll failed", err).Fatal() - } - - if !bytes.Equal(newReadBytes, buf) { - failureLog(function, args, startTime, "", "Bytes mismatch", err).Fatal() - } - - err = c.FGetObject(bucketName, objectName, fileName+"-f") - if err != nil { - failureLog(function, args, startTime, "", "FgetObject failed", err).Fatal() - } - - // Generate presigned HEAD object url. - presignedHeadURL, err := c.PresignedHeadObject(bucketName, objectName, 3600*time.Second, nil) - if err != nil { - failureLog(function, args, startTime, "", "PresignedHeadObject failed", err).Fatal() - } - // Verify if presigned url works. - resp, err := http.Head(presignedHeadURL.String()) - if err != nil { - failureLog(function, args, startTime, "", "PresignedHeadObject URL head request failed", err).Fatal() - } - if resp.StatusCode != http.StatusOK { - failureLog(function, args, startTime, "", "PresignedHeadObject URL returns status "+string(resp.StatusCode), err).Fatal() - } - if resp.Header.Get("ETag") == "" { - failureLog(function, args, startTime, "", "Got empty ETag", err).Fatal() - } - resp.Body.Close() - - // Generate presigned GET object url. - presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second, nil) - if err != nil { - failureLog(function, args, startTime, "", "PresignedGetObject failed", err).Fatal() - } - // Verify if presigned url works. - resp, err = http.Get(presignedGetURL.String()) - if err != nil { - failureLog(function, args, startTime, "", "PresignedGetObject URL GET request failed", err).Fatal() - } - if resp.StatusCode != http.StatusOK { - failureLog(function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err).Fatal() - } - newPresignedBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - failureLog(function, args, startTime, "", "ReadAll failed", err).Fatal() - } - resp.Body.Close() - if !bytes.Equal(newPresignedBytes, buf) { - failureLog(function, args, startTime, "", "Bytes mismatch", err).Fatal() - } - - // Set request parameters. - reqParams := make(url.Values) - reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") - // Generate presigned GET object url. - presignedGetURL, err = c.PresignedGetObject(bucketName, objectName, 3600*time.Second, reqParams) - if err != nil { - failureLog(function, args, startTime, "", "PresignedGetObject failed", err).Fatal() - } - // Verify if presigned url works. - resp, err = http.Get(presignedGetURL.String()) - if err != nil { - failureLog(function, args, startTime, "", "PresignedGetObject URL GET request failed", err).Fatal() - } - if resp.StatusCode != http.StatusOK { - failureLog(function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err).Fatal() - } - newPresignedBytes, err = ioutil.ReadAll(resp.Body) - if err != nil { - failureLog(function, args, startTime, "", "ReadAll failed", err).Fatal() - } - if !bytes.Equal(newPresignedBytes, buf) { - failureLog(function, args, startTime, "", "Bytes mismatch", err).Fatal() - } - // Verify content disposition. - if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { - failureLog(function, args, startTime, "", "wrong Content-Disposition received ", err).Fatal() - } - - presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) - if err != nil { - failureLog(function, args, startTime, "", "PresignedPutObject failed", err).Fatal() - } - // Generate data more than 32K - buf = bytes.Repeat([]byte("1"), rand.Intn(1<<10)+32*1024) - - req, err := http.NewRequest("PUT", presignedPutURL.String(), bytes.NewReader(buf)) - if err != nil { - failureLog(function, args, startTime, "", "HTTP request to PresignedPutObject URL failed", err).Fatal() - } - httpClient := &http.Client{ - // Setting a sensible time out of 30secs to wait for response - // headers. Request is pro-actively cancelled after 30secs - // with no response. - Timeout: 30 * time.Second, - Transport: http.DefaultTransport, - } - resp, err = httpClient.Do(req) - if err != nil { - failureLog(function, args, startTime, "", "HTTP request to PresignedPutObject URL failed", err).Fatal() - } - - newReader, err = c.GetObject(bucketName, objectName+"-presigned") - if err != nil { - failureLog(function, args, startTime, "", "GetObject failed", err).Fatal() - } - - newReadBytes, err = ioutil.ReadAll(newReader) - if err != nil { - failureLog(function, args, startTime, "", "ReadAll failed", err).Fatal() - } - - if !bytes.Equal(newReadBytes, buf) { - failureLog(function, args, startTime, "", "Bytes mismatch", err).Fatal() - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - err = c.RemoveObject(bucketName, objectName+"-f") - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - err = c.RemoveObject(bucketName, objectName+"-nolength") - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - err = c.RemoveObject(bucketName, objectName+"-presigned") - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject failed", err).Fatal() - } - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket failed", err).Fatal() - } - err = c.RemoveBucket(bucketName) - if err == nil { - failureLog(function, args, startTime, "", "RemoveBucket should fail as bucket does not exist", err).Fatal() - } - if err.Error() != "The specified bucket does not exist" { - failureLog(function, args, startTime, "", "RemoveBucket failed with wrong error message", err).Fatal() - } - if err = os.Remove(fileName); err != nil { - failureLog(function, args, startTime, "", "File remove failed", err).Fatal() - } - if err = os.Remove(fileName + "-f"); err != nil { - failureLog(function, args, startTime, "", "File removes failed", err).Fatal() - } - successLogger(function, args, startTime).Info() -} - -// Test get object with GetObjectWithContext -func testGetObjectWithContext() { - // initialize logging params - startTime := time.Now() - function := "GetObjectWithContext(ctx, bucketName, objectName)" - args := map[string]interface{}{ - "ctx": "", - "bucketName": "", - "objectName": "", - } - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client v4 object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Generate data more than 32K. - bufSize := 1<<20 + 32*1024 - var reader = getDataReader("datafile-33-kB", bufSize) - defer reader.Close() - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - - _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - - } - - ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) - defer cancel() - - // Read the data back - r, err := c.GetObjectWithContext(ctx, bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "GetObjectWithContext failed - request timeout not honored", err).Fatal() - - } - ctx, cancel = context.WithTimeout(context.Background(), 3*time.Minute) - defer cancel() - - // Read the data back - r, err = c.GetObjectWithContext(ctx, bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "GetObjectWithContext failed", err).Fatal() - - } - - st, err := r.Stat() - if err != nil { - failureLog(function, args, startTime, "", "object Stat call failed", err).Fatal() - } - if st.Size != int64(bufSize) { - failureLog(function, args, startTime, "", "Number of bytes in stat does not match: want "+string(bufSize)+", got"+string(st.Size), err).Fatal() - } - if err := r.Close(); err != nil { - failureLog(function, args, startTime, "", "object Close() call failed", err).Fatal() - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject call failed", err).Fatal() - } - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket call failed", err).Fatal() - } - successLogger(function, args, startTime).Info() - -} - -// Test get object with FGetObjectWithContext -func testFGetObjectWithContext() { - // initialize logging params - startTime := time.Now() - function := "FGetObjectWithContext(ctx, bucketName, objectName, fileName)" - args := map[string]interface{}{ - "ctx": "", - "bucketName": "", - "objectName": "", - } - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client v4 object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Generate data more than 32K. - var reader = getDataReader("datafile-1-MiB", oneMiB) - defer reader.Close() - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - - _, err = c.PutObject(bucketName, objectName, reader, int64(oneMiB), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject failed", err).Fatal() - } - - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Millisecond) - defer cancel() - - fileName := "tempfile-context" - // Read the data back - err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-f") - if err == nil { - failureLog(function, args, startTime, "", "FGetObjectWithContext with short timeout failed", err).Fatal() - } - ctx, cancel = context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - - // Read the data back - err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-fcontext") - if err != nil { - failureLog(function, args, startTime, "", "FGetObjectWithContext with long timeout failed", err).Fatal() - } - if err = os.Remove(fileName + "-fcontext"); err != nil { - failureLog(function, args, startTime, "", "Remove file failed", err).Fatal() - - } - err = c.RemoveObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject call failed", err).Fatal() - - } - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket call failed", err).Fatal() - - } - successLogger(function, args, startTime).Info() - -} - -// Test validates putObject with context to see if request cancellation is honored for V2. -func testPutObjectWithContextV2() { - // initialize logging params - startTime := time.Now() - function := "PutObjectWithContext(ctx, bucketName, objectName, fileName, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "opts": "minio.PutObjectOptions{ContentType:objectContentType}", - } - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client v2 object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Make a new bucket. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - - } - defer c.RemoveBucket(bucketName) - bufSize := 1<<20 + 32*1024 - var reader = getDataReader("datafile-33-kB", bufSize) - defer reader.Close() - objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - _, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObjectWithContext with short timeout failed", err).Fatal() - } - - ctx, cancel = context.WithTimeout(context.Background(), 3*time.Minute) - defer cancel() - reader = getDataReader("datafile-33-kB", bufSize) - defer reader.Close() - _, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObjectWithContext with long timeout failed", err).Fatal() - } - - if err = c.RemoveObject(bucketName, objectName); err != nil { - failureLog(function, args, startTime, "", "RemoveObject call failed", err).Fatal() - } - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket call failed", err).Fatal() - } - successLogger(function, args, startTime).Info() - -} - -// Test get object with GetObjectWithContext -func testGetObjectWithContextV2() { - // initialize logging params - startTime := time.Now() - function := "GetObjectWithContext(ctx, bucketName, objectName)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - } - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client v2 object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket failed", err).Fatal() - } - - // Generate data more than 32K. - bufSize := 1<<20 + 32*1024 - var reader = getDataReader("datafile-33-kB", bufSize) - defer reader.Close() - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - - _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject call failed", err).Fatal() - } - - ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) - defer cancel() - - // Read the data back - r, err := c.GetObjectWithContext(ctx, bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "GetObjectWithContext failed due to non-cancellation upon short timeout", err).Fatal() - - } - ctx, cancel = context.WithTimeout(context.Background(), 3*time.Minute) - defer cancel() - - // Read the data back - r, err = c.GetObjectWithContext(ctx, bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "GetObjectWithContext failed due to non-cancellation upon long timeout", err).Fatal() - - } - - st, err := r.Stat() - if err != nil { - failureLog(function, args, startTime, "", "object Stat call failed", err).Fatal() - } - if st.Size != int64(bufSize) { - failureLog(function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(bufSize)+" got "+string(st.Size), err).Fatal() - } - if err := r.Close(); err != nil { - failureLog(function, args, startTime, "", " object Close() call failed", err).Fatal() - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject call failed", err).Fatal() - - } - - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket call failed", err).Fatal() - } - successLogger(function, args, startTime).Info() - -} - -// Test get object with FGetObjectWithContext -func testFGetObjectWithContextV2() { - // initialize logging params - startTime := time.Now() - function := "FGetObjectWithContext(ctx, bucketName, objectName,fileName)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "fileName": "", - } - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - failureLog(function, args, startTime, "", "Minio client v2 object creation failed", err).Fatal() - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - failureLog(function, args, startTime, "", "MakeBucket call failed", err).Fatal() - } - - // Generate data more than 32K. - - var reader = getDataReader("datafile-1-MiB", oneMiB) - defer reader.Close() - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - - _, err = c.PutObject(bucketName, objectName, reader, int64(oneMiB), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - failureLog(function, args, startTime, "", "PutObject call failed", err).Fatal() - } - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) - defer cancel() - - fileName := "tempfile-context" - // Read the data back - err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-f") - if err == nil { - failureLog(function, args, startTime, "", "FGetObjectWithContext call with short request timeout failed", err).Fatal() - - } - ctx, cancel = context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - - // Read the data back - err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-fcontext") - if err != nil { - failureLog(function, args, startTime, "", "FGetObjectWithContext call with long request timeout failed", err).Fatal() - } - - if err = os.Remove(fileName + "-fcontext"); err != nil { - failureLog(function, args, startTime, "", "Remove file failed", err).Fatal() - } - err = c.RemoveObject(bucketName, objectName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveObject call failed", err).Fatal() - } - err = c.RemoveBucket(bucketName) - if err != nil { - failureLog(function, args, startTime, "", "RemoveBucket call failed", err).Fatal() - } - successLogger(function, args, startTime).Info() - -} - -// Convert string to bool and always return false if any error -func mustParseBool(str string) bool { - b, err := strconv.ParseBool(str) - if err != nil { - return false - } - return b -} - -func main() { - // Output to stdout instead of the default stderr - log.SetOutput(os.Stdout) - // create custom formatter - mintFormatter := mintJSONFormatter{} - // set custom formatter - log.SetFormatter(&mintFormatter) - // log Info or above -- success cases are Info level, failures are Fatal level - log.SetLevel(log.InfoLevel) - // execute tests - if !isQuickMode() { - testMakeBucketErrorV2() - testGetObjectClosedTwiceV2() - testRemovePartiallyUploadedV2() - testFPutObjectV2() - testMakeBucketRegionsV2() - testGetObjectReadSeekFunctionalV2() - testGetObjectReadAtFunctionalV2() - testCopyObjectV2() - testFunctionalV2() - testComposeObjectErrorCasesV2() - testCompose10KSourcesV2() - testEncryptedCopyObjectV2() - testUserMetadataCopyingV2() - testPutObject0ByteV2() - testPutObjectNoLengthV2() - testPutObjectsUnknownV2() - testGetObjectWithContextV2() - testFPutObjectWithContextV2() - testFGetObjectWithContextV2() - testPutObjectWithContextV2() - testMakeBucketError() - testMakeBucketRegions() - testPutObjectWithMetadata() - testPutObjectReadAt() - testPutObjectStreaming() - testListPartiallyUploaded() - testGetObjectSeekEnd() - testGetObjectClosedTwice() - testRemoveMultipleObjects() - testRemovePartiallyUploaded() - testFPutObjectMultipart() - testFPutObject() - testGetObjectReadSeekFunctional() - testGetObjectReadAtFunctional() - testPresignedPostPolicy() - testCopyObject() - testEncryptionPutGet() - testEncryptionFPut() - testComposeObjectErrorCases() - testCompose10KSources() - testUserMetadataCopying() - testEncryptedCopyObject() - testBucketNotification() - testFunctional() - testGetObjectObjectModified() - testPutObjectUploadSeekedObject() - testGetObjectWithContext() - testFPutObjectWithContext() - testFGetObjectWithContext() - testPutObjectWithContext() - } else { - testFunctional() - testFunctionalV2() - } -} diff --git a/vendor/vendor.json b/vendor/vendor.json index 5b4a8f74d..ee4d58dda 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -336,10 +336,10 @@ "revisionTime": "2016-02-29T08:42:30-08:00" }, { - "checksumSHA1": "mqxOM3CsubB09O0nDEe4efu0JLQ=", + "checksumSHA1": "EkdIh5Mk2bRiARtdoqUfnBuyndk=", "path": "github.com/minio/minio-go", - "revision": "414c6b6a2e97428776cd831d9745589ebcf873e5", - "revisionTime": "2017-09-27T19:03:45Z" + "revision": "9690dc6c40e6ef271727848c04f974e801212ac1", + "revisionTime": "2017-10-02T19:34:27Z" }, { "checksumSHA1": "5juljGXPkBWENR2Os7dlnPQER48=",