2021-04-18 15:41:13 -04:00
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
2016-06-27 13:01:09 -04:00
2016-08-18 19:23:42 -04:00
package cmd
2016-06-27 13:01:09 -04:00
import (
"bytes"
2018-03-15 16:27:16 -04:00
"context"
2016-06-29 05:28:46 -04:00
"crypto/md5"
"encoding/hex"
2020-11-23 12:12:17 -05:00
"errors"
2016-06-29 05:28:46 -04:00
"io/ioutil"
"os"
"path"
2016-06-27 13:01:09 -04:00
"testing"
2016-11-22 21:18:22 -05:00
humanize "github.com/dustin/go-humanize"
2021-06-01 17:59:40 -04:00
"github.com/minio/minio/internal/hash"
2016-06-27 13:01:09 -04:00
)
2016-07-21 16:15:54 -04:00
func md5Header ( data [ ] byte ) map [ string ] string {
2021-05-24 12:28:19 -04:00
return map [ string ] string { "etag" : getMD5Hash ( data ) }
2016-07-21 16:15:54 -04:00
}
2020-06-12 23:04:01 -04:00
// Wrapper for calling PutObject tests for both Erasure multiple disks and single node setup.
2017-10-22 01:30:34 -04:00
func TestObjectAPIPutObjectSingle ( t * testing . T ) {
2021-01-05 23:08:35 -05:00
ExecExtendedObjectLayerTest ( t , testObjectAPIPutObject )
2016-06-27 13:01:09 -04:00
}
// Tests validate correctness of PutObject.
2016-07-07 18:05:51 -04:00
func testObjectAPIPutObject ( obj ObjectLayer , instanceType string , t TestErrHandler ) {
2016-06-27 13:01:09 -04:00
// Generating cases for which the PutObject fails.
bucket := "minio-bucket"
object := "minio-object"
// Create bucket.
2020-06-12 23:04:01 -04:00
err := obj . MakeBucketWithLocation ( context . Background ( ) , bucket , BucketOptions { } )
2016-06-27 13:01:09 -04:00
if err != nil {
// Failed to create newbucket, abort.
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
// Creating a dummy bucket for tests.
2020-06-12 23:04:01 -04:00
err = obj . MakeBucketWithLocation ( context . Background ( ) , "unused-bucket" , BucketOptions { } )
2016-06-27 13:01:09 -04:00
if err != nil {
// Failed to create newbucket, abort.
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
2016-07-21 16:15:54 -04:00
var (
nilBytes [ ] byte
data = [ ] byte ( "hello" )
2016-11-22 21:18:22 -05:00
fiveMBBytes = bytes . Repeat ( [ ] byte ( "a" ) , 5 * humanize . MiByte )
2016-07-21 16:15:54 -04:00
)
2016-11-21 16:51:05 -05:00
invalidMD5 := getMD5Hash ( [ ] byte ( "meh" ) )
2016-07-21 16:15:54 -04:00
invalidMD5Header := md5Header ( [ ] byte ( "meh" ) )
testCases := [ ] struct {
bucketName string
objName string
inputData [ ] byte
inputMeta map [ string ] string
2016-10-02 18:51:49 -04:00
inputSHA256 string
2016-07-21 16:15:54 -04:00
intputDataSize int64
2016-06-27 13:01:09 -04:00
// expected error output.
expectedMd5 string
expectedError error
} {
// Test case 1-4.
// Cases with invalid bucket name.
2018-04-23 23:27:33 -04:00
{ ".test" , "obj" , [ ] byte ( "" ) , nil , "" , 0 , "" , BucketNotFound { Bucket : ".test" } } ,
{ "------" , "obj" , [ ] byte ( "" ) , nil , "" , 0 , "" , BucketNotFound { Bucket : "------" } } ,
2016-10-02 18:51:49 -04:00
{ "$this-is-not-valid-too" , "obj" , [ ] byte ( "" ) , nil , "" , 0 , "" ,
2018-04-23 23:27:33 -04:00
BucketNotFound { Bucket : "$this-is-not-valid-too" } } ,
{ "a" , "obj" , [ ] byte ( "" ) , nil , "" , 0 , "" , BucketNotFound { Bucket : "a" } } ,
2016-07-21 16:15:54 -04:00
2016-06-27 13:01:09 -04:00
// Test case - 5.
// Case with invalid object names.
2016-10-02 18:51:49 -04:00
{ bucket , "" , [ ] byte ( "" ) , nil , "" , 0 , "" , ObjectNameInvalid { Bucket : bucket , Object : "" } } ,
2016-07-21 16:15:54 -04:00
2016-06-27 13:01:09 -04:00
// Test case - 6.
// Valid object and bucket names but non-existent bucket.
2016-10-02 18:51:49 -04:00
{ "abc" , "def" , [ ] byte ( "" ) , nil , "" , 0 , "" , BucketNotFound { Bucket : "abc" } } ,
2016-07-21 16:15:54 -04:00
2016-06-27 13:01:09 -04:00
// Test case - 7.
// Input to replicate Md5 mismatch.
2017-10-22 01:30:34 -04:00
{ bucket , object , [ ] byte ( "" ) , map [ string ] string { "etag" : "d41d8cd98f00b204e9800998ecf8427f" } , "" , 0 , "" ,
2018-11-14 13:23:44 -05:00
hash . BadDigest { ExpectedMD5 : "d41d8cd98f00b204e9800998ecf8427f" , CalculatedMD5 : "d41d8cd98f00b204e9800998ecf8427e" } } ,
2016-07-21 16:15:54 -04:00
2016-06-27 13:01:09 -04:00
// Test case - 8.
2016-10-02 18:51:49 -04:00
// With incorrect sha256.
2017-10-22 01:30:34 -04:00
{ bucket , object , [ ] byte ( "abcd" ) , map [ string ] string { "etag" : "e2fc714c4727ee9395f324cd2e7f331f" } ,
"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031580" , int64 ( len ( "abcd" ) ) ,
2018-11-14 13:23:44 -05:00
"" , hash . SHA256Mismatch { ExpectedSHA256 : "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031580" ,
CalculatedSHA256 : "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589" } } ,
2016-10-02 18:51:49 -04:00
// Test case - 9.
2016-06-27 13:01:09 -04:00
// Input with size more than the size of actual data inside the reader.
2017-10-22 01:30:34 -04:00
{ bucket , object , [ ] byte ( "abcd" ) , map [ string ] string { "etag" : "e2fc714c4727ee9395f324cd2e7f331e" } , "" , int64 ( len ( "abcd" ) + 1 ) , "" ,
2018-11-14 13:23:44 -05:00
hash . BadDigest { ExpectedMD5 : "e2fc714c4727ee9395f324cd2e7f331e" , CalculatedMD5 : "e2fc714c4727ee9395f324cd2e7f331f" } } ,
2016-07-21 16:15:54 -04:00
2016-10-02 18:51:49 -04:00
// Test case - 10.
2016-06-27 13:01:09 -04:00
// Input with size less than the size of actual data inside the reader.
2017-10-22 01:30:34 -04:00
{ bucket , object , [ ] byte ( "abcd" ) , map [ string ] string { "etag" : "900150983cd24fb0d6963f7d28e17f73" } , "" , int64 ( len ( "abcd" ) - 1 ) , "" ,
2018-11-14 13:23:44 -05:00
hash . BadDigest { ExpectedMD5 : "900150983cd24fb0d6963f7d28e17f73" , CalculatedMD5 : "900150983cd24fb0d6963f7d28e17f72" } } ,
2016-07-21 16:15:54 -04:00
2016-10-02 18:51:49 -04:00
// Test case - 11-14.
2016-06-27 13:01:09 -04:00
// Validating for success cases.
2017-05-14 15:05:51 -04:00
{ bucket , object , [ ] byte ( "abcd" ) , map [ string ] string { "etag" : "e2fc714c4727ee9395f324cd2e7f331f" } , "" , int64 ( len ( "abcd" ) ) , "" , nil } ,
{ bucket , object , [ ] byte ( "efgh" ) , map [ string ] string { "etag" : "1f7690ebdd9b4caf8fab49ca1757bf27" } , "" , int64 ( len ( "efgh" ) ) , "" , nil } ,
{ bucket , object , [ ] byte ( "ijkl" ) , map [ string ] string { "etag" : "09a0877d04abf8759f99adec02baf579" } , "" , int64 ( len ( "ijkl" ) ) , "" , nil } ,
{ bucket , object , [ ] byte ( "mnop" ) , map [ string ] string { "etag" : "e132e96a5ddad6da8b07bba6f6131fef" } , "" , int64 ( len ( "mnop" ) ) , "" , nil } ,
2016-06-27 13:01:09 -04:00
2016-10-02 18:51:49 -04:00
// Test case 15-17.
2016-07-21 16:15:54 -04:00
// With no metadata
2016-11-21 16:51:05 -05:00
{ bucket , object , data , nil , "" , int64 ( len ( data ) ) , getMD5Hash ( data ) , nil } ,
{ bucket , object , nilBytes , nil , "" , int64 ( len ( nilBytes ) ) , getMD5Hash ( nilBytes ) , nil } ,
{ bucket , object , fiveMBBytes , nil , "" , int64 ( len ( fiveMBBytes ) ) , getMD5Hash ( fiveMBBytes ) , nil } ,
2016-06-27 13:01:09 -04:00
2016-10-02 18:51:49 -04:00
// Test case 18-20.
2016-07-21 16:15:54 -04:00
// With arbitrary metadata
2016-11-21 16:51:05 -05:00
{ bucket , object , data , map [ string ] string { "answer" : "42" } , "" , int64 ( len ( data ) ) , getMD5Hash ( data ) , nil } ,
{ bucket , object , nilBytes , map [ string ] string { "answer" : "42" } , "" , int64 ( len ( nilBytes ) ) , getMD5Hash ( nilBytes ) , nil } ,
{ bucket , object , fiveMBBytes , map [ string ] string { "answer" : "42" } , "" , int64 ( len ( fiveMBBytes ) ) , getMD5Hash ( fiveMBBytes ) , nil } ,
2016-07-21 16:15:54 -04:00
2016-10-02 18:51:49 -04:00
// Test case 21-23.
// With valid md5sum and sha256.
2016-11-21 16:51:05 -05:00
{ bucket , object , data , md5Header ( data ) , getSHA256Hash ( data ) , int64 ( len ( data ) ) , getMD5Hash ( data ) , nil } ,
{ bucket , object , nilBytes , md5Header ( nilBytes ) , getSHA256Hash ( nilBytes ) , int64 ( len ( nilBytes ) ) , getMD5Hash ( nilBytes ) , nil } ,
{ bucket , object , fiveMBBytes , md5Header ( fiveMBBytes ) , getSHA256Hash ( fiveMBBytes ) , int64 ( len ( fiveMBBytes ) ) , getMD5Hash ( fiveMBBytes ) , nil } ,
2016-07-21 16:15:54 -04:00
2016-10-02 18:51:49 -04:00
// Test case 24-26.
2016-07-21 16:15:54 -04:00
// data with invalid md5sum in header
2017-10-22 01:30:34 -04:00
{ bucket , object , data , invalidMD5Header , "" , int64 ( len ( data ) ) , getMD5Hash ( data ) ,
2018-11-14 13:23:44 -05:00
hash . BadDigest { ExpectedMD5 : invalidMD5 , CalculatedMD5 : getMD5Hash ( data ) } } ,
2017-10-22 01:30:34 -04:00
{ bucket , object , nilBytes , invalidMD5Header , "" , int64 ( len ( nilBytes ) ) , getMD5Hash ( nilBytes ) ,
2018-11-14 13:23:44 -05:00
hash . BadDigest { ExpectedMD5 : invalidMD5 , CalculatedMD5 : getMD5Hash ( nilBytes ) } } ,
2017-10-22 01:30:34 -04:00
{ bucket , object , fiveMBBytes , invalidMD5Header , "" , int64 ( len ( fiveMBBytes ) ) , getMD5Hash ( fiveMBBytes ) ,
2018-11-14 13:23:44 -05:00
hash . BadDigest { ExpectedMD5 : invalidMD5 , CalculatedMD5 : getMD5Hash ( fiveMBBytes ) } } ,
2016-07-21 16:15:54 -04:00
2016-10-02 18:51:49 -04:00
// Test case 27-29.
2016-07-21 16:15:54 -04:00
// data with size different from the actual number of bytes available in the reader
2016-11-21 16:51:05 -05:00
{ bucket , object , data , nil , "" , int64 ( len ( data ) - 1 ) , getMD5Hash ( data [ : len ( data ) - 1 ] ) , nil } ,
2020-09-08 17:22:04 -04:00
{ bucket , object , nilBytes , nil , "" , int64 ( len ( nilBytes ) + 1 ) , getMD5Hash ( nilBytes ) , IncompleteBody { Bucket : bucket , Object : object } } ,
2017-02-24 12:20:40 -05:00
{ bucket , object , fiveMBBytes , nil , "" , 0 , getMD5Hash ( fiveMBBytes ) , nil } ,
2016-09-16 16:06:49 -04:00
2016-10-02 18:51:49 -04:00
// Test case 30
2016-09-16 16:06:49 -04:00
// valid data with X-Amz-Meta- meta
2016-11-21 16:51:05 -05:00
{ bucket , object , data , map [ string ] string { "X-Amz-Meta-AppID" : "a42" } , "" , int64 ( len ( data ) ) , getMD5Hash ( data ) , nil } ,
2018-10-17 19:37:02 -04:00
// Test case 31
// Put an empty object with a trailing slash
{ bucket , "emptydir/" , [ ] byte { } , nil , "" , 0 , getMD5Hash ( [ ] byte { } ) , nil } ,
// Test case 32
// Put an object inside the empty directory
{ bucket , "emptydir/" + object , data , nil , "" , int64 ( len ( data ) ) , getMD5Hash ( data ) , nil } ,
// Test case 33
// Put the empty object with a trailing slash again (refer to Test case 31), this needs to succeed
{ bucket , "emptydir/" , [ ] byte { } , nil , "" , 0 , getMD5Hash ( [ ] byte { } ) , nil } ,
2016-07-21 16:15:54 -04:00
}
for i , testCase := range testCases {
2020-05-14 17:01:31 -04:00
in := mustGetPutObjReader ( t , bytes . NewReader ( testCase . inputData ) , testCase . intputDataSize , testCase . inputMeta [ "etag" ] , testCase . inputSHA256 )
objInfo , actualErr := obj . PutObject ( context . Background ( ) , testCase . bucketName , testCase . objName , in , ObjectOptions { UserDefined : testCase . inputMeta } )
2016-07-21 16:15:54 -04:00
if actualErr != nil && testCase . expectedError == nil {
t . Errorf ( "Test %d: %s: Expected to pass, but failed with: error %s." , i + 1 , instanceType , actualErr . Error ( ) )
2020-05-14 17:01:31 -04:00
continue
2016-06-27 13:01:09 -04:00
}
2016-07-21 16:15:54 -04:00
if actualErr == nil && testCase . expectedError != nil {
t . Errorf ( "Test %d: %s: Expected to fail with error \"%s\", but passed instead." , i + 1 , instanceType , testCase . expectedError . Error ( ) )
2020-05-14 17:01:31 -04:00
continue
2016-06-27 13:01:09 -04:00
}
// Failed as expected, but does it fail for the expected reason.
2016-08-25 12:39:01 -04:00
if actualErr != nil && actualErr != testCase . expectedError {
2018-10-17 19:37:02 -04:00
t . Errorf ( "Test %d: %s: Expected to fail with error \"%v\", but instead failed with error \"%v\" instead." , i + 1 , instanceType , testCase . expectedError , actualErr )
2020-05-14 17:01:31 -04:00
continue
2016-06-27 13:01:09 -04:00
}
// Test passes as expected, but the output values are verified for correctness here.
2016-07-21 16:15:54 -04:00
if actualErr == nil {
2016-06-27 13:01:09 -04:00
// Asserting whether the md5 output is correct.
2017-05-14 15:05:51 -04:00
if expectedMD5 , ok := testCase . inputMeta [ "etag" ] ; ok && expectedMD5 != objInfo . ETag {
t . Errorf ( "Test %d: %s: Calculated Md5 different from the actual one %s." , i + 1 , instanceType , objInfo . ETag )
2020-05-14 17:01:31 -04:00
continue
2016-06-27 13:01:09 -04:00
}
}
}
}
2020-06-12 23:04:01 -04:00
// Wrapper for calling PutObject tests for both Erasure multiple disks case
2016-06-28 02:54:56 -04:00
// when quorum is not available.
2016-06-27 13:01:09 -04:00
func TestObjectAPIPutObjectDiskNotFound ( t * testing . T ) {
2017-03-07 17:48:56 -05:00
ExecObjectLayerDiskAlteredTest ( t , testObjectAPIPutObjectDiskNotFound )
2016-06-27 13:01:09 -04:00
}
// Tests validate correctness of PutObject.
2017-03-07 17:48:56 -05:00
func testObjectAPIPutObjectDiskNotFound ( obj ObjectLayer , instanceType string , disks [ ] string , t * testing . T ) {
2016-06-27 13:01:09 -04:00
// Generating cases for which the PutObject fails.
bucket := "minio-bucket"
object := "minio-object"
// Create bucket.
2020-06-12 23:04:01 -04:00
err := obj . MakeBucketWithLocation ( context . Background ( ) , bucket , BucketOptions { } )
2016-06-27 13:01:09 -04:00
if err != nil {
// Failed to create newbucket, abort.
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
// Creating a dummy bucket for tests.
2020-06-12 23:04:01 -04:00
err = obj . MakeBucketWithLocation ( context . Background ( ) , "unused-bucket" , BucketOptions { } )
2016-06-27 13:01:09 -04:00
if err != nil {
// Failed to create newbucket, abort.
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
2021-01-16 15:08:02 -05:00
// Take 4 disks down, one more we loose quorum on 16 disk node.
for _ , disk := range disks [ : 4 ] {
2017-08-12 22:25:43 -04:00
os . RemoveAll ( disk )
2016-06-27 13:01:09 -04:00
}
testCases := [ ] struct {
2016-07-21 16:15:54 -04:00
bucketName string
objName string
inputData [ ] byte
inputMeta map [ string ] string
intputDataSize int64
2016-06-27 13:01:09 -04:00
// flag indicating whether the test should pass.
shouldPass bool
// expected error output.
expectedMd5 string
expectedError error
} {
// Validating for success cases.
2017-05-14 15:05:51 -04:00
{ bucket , object , [ ] byte ( "abcd" ) , map [ string ] string { "etag" : "e2fc714c4727ee9395f324cd2e7f331f" } , int64 ( len ( "abcd" ) ) , true , "" , nil } ,
{ bucket , object , [ ] byte ( "efgh" ) , map [ string ] string { "etag" : "1f7690ebdd9b4caf8fab49ca1757bf27" } , int64 ( len ( "efgh" ) ) , true , "" , nil } ,
{ bucket , object , [ ] byte ( "ijkl" ) , map [ string ] string { "etag" : "09a0877d04abf8759f99adec02baf579" } , int64 ( len ( "ijkl" ) ) , true , "" , nil } ,
{ bucket , object , [ ] byte ( "mnop" ) , map [ string ] string { "etag" : "e132e96a5ddad6da8b07bba6f6131fef" } , int64 ( len ( "mnop" ) ) , true , "" , nil } ,
2016-06-27 13:01:09 -04:00
}
2016-10-02 18:51:49 -04:00
sha256sum := ""
2016-06-27 13:01:09 -04:00
for i , testCase := range testCases {
2019-02-09 00:31:06 -05:00
objInfo , actualErr := obj . PutObject ( context . Background ( ) , testCase . bucketName , testCase . objName , mustGetPutObjReader ( t , bytes . NewReader ( testCase . inputData ) , testCase . intputDataSize , testCase . inputMeta [ "etag" ] , sha256sum ) , ObjectOptions { UserDefined : testCase . inputMeta } )
2016-06-27 13:01:09 -04:00
if actualErr != nil && testCase . shouldPass {
t . Errorf ( "Test %d: %s: Expected to pass, but failed with: <ERROR> %s." , i + 1 , instanceType , actualErr . Error ( ) )
}
if actualErr == nil && ! testCase . shouldPass {
t . Errorf ( "Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead." , i + 1 , instanceType , testCase . expectedError . Error ( ) )
}
// Failed as expected, but does it fail for the expected reason.
if actualErr != nil && ! testCase . shouldPass {
if testCase . expectedError . Error ( ) != actualErr . Error ( ) {
t . Errorf ( "Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead." , i + 1 ,
instanceType , testCase . expectedError . Error ( ) , actualErr . Error ( ) )
}
}
// Test passes as expected, but the output values are verified for correctness here.
if actualErr == nil && testCase . shouldPass {
// Asserting whether the md5 output is correct.
2017-05-14 15:05:51 -04:00
if testCase . inputMeta [ "etag" ] != objInfo . ETag {
t . Errorf ( "Test %d: %s: Calculated Md5 different from the actual one %s." , i + 1 , instanceType , objInfo . ETag )
2016-06-27 13:01:09 -04:00
}
}
}
// This causes quorum failure verify.
2017-08-12 22:25:43 -04:00
os . RemoveAll ( disks [ len ( disks ) - 1 ] )
2016-06-27 13:01:09 -04:00
// Validate the last test.
testCase := struct {
2016-07-21 16:15:54 -04:00
bucketName string
objName string
inputData [ ] byte
inputMeta map [ string ] string
intputDataSize int64
2016-06-27 13:01:09 -04:00
// flag indicating whether the test should pass.
shouldPass bool
// expected error output.
expectedMd5 string
expectedError error
} {
bucket ,
object ,
2016-07-21 16:15:54 -04:00
[ ] byte ( "mnop" ) ,
2017-05-14 15:05:51 -04:00
map [ string ] string { "etag" : "e132e96a5ddad6da8b07bba6f6131fef" } ,
2016-06-27 13:01:09 -04:00
int64 ( len ( "mnop" ) ) ,
false ,
"" ,
2020-11-23 12:12:17 -05:00
errErasureWriteQuorum ,
2016-06-27 13:01:09 -04:00
}
2016-10-02 18:51:49 -04:00
2019-02-09 00:31:06 -05:00
_ , actualErr := obj . PutObject ( context . Background ( ) , testCase . bucketName , testCase . objName , mustGetPutObjReader ( t , bytes . NewReader ( testCase . inputData ) , testCase . intputDataSize , testCase . inputMeta [ "etag" ] , sha256sum ) , ObjectOptions { UserDefined : testCase . inputMeta } )
2016-06-27 13:01:09 -04:00
if actualErr != nil && testCase . shouldPass {
t . Errorf ( "Test %d: %s: Expected to pass, but failed with: <ERROR> %s." , len ( testCases ) + 1 , instanceType , actualErr . Error ( ) )
}
// Failed as expected, but does it fail for the expected reason.
if actualErr != nil && ! testCase . shouldPass {
2020-11-23 12:12:17 -05:00
if ! errors . Is ( actualErr , testCase . expectedError ) {
2016-06-27 13:01:09 -04:00
t . Errorf ( "Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead." , len ( testCases ) + 1 , instanceType , testCase . expectedError . Error ( ) , actualErr . Error ( ) )
}
}
}
2016-06-29 05:28:46 -04:00
2020-06-12 23:04:01 -04:00
// Wrapper for calling PutObject tests for both Erasure multiple disks and single node setup.
2016-06-29 05:28:46 -04:00
func TestObjectAPIPutObjectStaleFiles ( t * testing . T ) {
ExecObjectLayerStaleFilesTest ( t , testObjectAPIPutObjectStaleFiles )
}
// Tests validate correctness of PutObject.
func testObjectAPIPutObjectStaleFiles ( obj ObjectLayer , instanceType string , disks [ ] string , t * testing . T ) {
// Generating cases for which the PutObject fails.
bucket := "minio-bucket"
object := "minio-object"
// Create bucket.
2020-06-12 23:04:01 -04:00
err := obj . MakeBucketWithLocation ( context . Background ( ) , bucket , BucketOptions { } )
2016-06-29 05:28:46 -04:00
if err != nil {
// Failed to create newbucket, abort.
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
data := [ ] byte ( "hello, world" )
// Create object.
2019-02-09 00:31:06 -05:00
_ , err = obj . PutObject ( context . Background ( ) , bucket , object , mustGetPutObjReader ( t , bytes . NewReader ( data ) , int64 ( len ( data ) ) , "" , "" ) , ObjectOptions { } )
2016-06-29 05:28:46 -04:00
if err != nil {
// Failed to create object, abort.
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
for _ , disk := range disks {
2016-11-20 17:25:43 -05:00
tmpMetaDir := path . Join ( disk , minioMetaTmpBucket )
2021-02-26 12:52:27 -05:00
files , err := ioutil . ReadDir ( tmpMetaDir )
if err != nil {
t . Fatal ( err )
}
var found bool
for _ , fi := range files {
if fi . Name ( ) == ".trash" {
continue
}
found = true
}
if found {
t . Fatalf ( "%s: expected: empty, got: non-empty %#v" , minioMetaTmpBucket , files )
2016-06-29 05:28:46 -04:00
}
}
}
2020-06-12 23:04:01 -04:00
// Wrapper for calling Multipart PutObject tests for both Erasure multiple disks and single node setup.
2016-06-29 05:28:46 -04:00
func TestObjectAPIMultipartPutObjectStaleFiles ( t * testing . T ) {
ExecObjectLayerStaleFilesTest ( t , testObjectAPIMultipartPutObjectStaleFiles )
}
// Tests validate correctness of PutObject.
func testObjectAPIMultipartPutObjectStaleFiles ( obj ObjectLayer , instanceType string , disks [ ] string , t * testing . T ) {
// Generating cases for which the PutObject fails.
bucket := "minio-bucket"
object := "minio-object"
// Create bucket.
2020-06-12 23:04:01 -04:00
err := obj . MakeBucketWithLocation ( context . Background ( ) , bucket , BucketOptions { } )
2016-06-29 05:28:46 -04:00
if err != nil {
// Failed to create newbucket, abort.
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
2018-09-10 12:42:43 -04:00
opts := ObjectOptions { }
2016-06-29 05:28:46 -04:00
// Initiate Multipart Upload on the above created bucket.
2019-02-09 00:31:06 -05:00
uploadID , err := obj . NewMultipartUpload ( context . Background ( ) , bucket , object , opts )
2016-06-29 05:28:46 -04:00
if err != nil {
// Failed to create NewMultipartUpload, abort.
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
// Upload part1.
2016-11-22 21:18:22 -05:00
fiveMBBytes := bytes . Repeat ( [ ] byte ( "a" ) , 5 * humanize . MiByte )
2016-06-29 05:28:46 -04:00
md5Writer := md5 . New ( )
md5Writer . Write ( fiveMBBytes )
etag1 := hex . EncodeToString ( md5Writer . Sum ( nil ) )
2016-10-02 18:51:49 -04:00
sha256sum := ""
2018-11-14 20:36:41 -05:00
_ , err = obj . PutObjectPart ( context . Background ( ) , bucket , object , uploadID , 1 , mustGetPutObjReader ( t , bytes . NewReader ( fiveMBBytes ) , int64 ( len ( fiveMBBytes ) ) , etag1 , sha256sum ) , opts )
2016-06-29 05:28:46 -04:00
if err != nil {
// Failed to upload object part, abort.
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
// Upload part2.
data := [ ] byte ( "hello, world" )
md5Writer = md5 . New ( )
md5Writer . Write ( data )
etag2 := hex . EncodeToString ( md5Writer . Sum ( nil ) )
2018-11-14 20:36:41 -05:00
_ , err = obj . PutObjectPart ( context . Background ( ) , bucket , object , uploadID , 2 , mustGetPutObjReader ( t , bytes . NewReader ( data ) , int64 ( len ( data ) ) , etag2 , sha256sum ) , opts )
2016-06-29 05:28:46 -04:00
if err != nil {
// Failed to upload object part, abort.
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
// Complete multipart.
2017-11-14 03:25:10 -05:00
parts := [ ] CompletePart {
2016-06-29 05:28:46 -04:00
{ ETag : etag1 , PartNumber : 1 } ,
{ ETag : etag2 , PartNumber : 2 } ,
}
2018-11-14 20:36:41 -05:00
_ , err = obj . CompleteMultipartUpload ( context . Background ( ) , bucket , object , uploadID , parts , ObjectOptions { } )
2016-06-29 05:28:46 -04:00
if err != nil {
// Failed to complete multipart upload, abort.
t . Fatalf ( "%s : %s" , instanceType , err . Error ( ) )
}
for _ , disk := range disks {
2016-11-20 17:25:43 -05:00
tmpMetaDir := path . Join ( disk , minioMetaTmpBucket )
2016-06-29 05:28:46 -04:00
files , err := ioutil . ReadDir ( tmpMetaDir )
if err != nil {
// Its OK to have non-existen tmpMetaDir.
2020-11-23 11:36:49 -05:00
if osIsNotExist ( err ) {
2016-06-29 05:28:46 -04:00
continue
}
// Print the error
t . Errorf ( "%s" , err )
}
2021-02-26 12:52:27 -05:00
var found bool
for _ , fi := range files {
if fi . Name ( ) == ".trash" {
continue
}
found = true
break
}
if found {
t . Fatalf ( "%s: expected: empty, got: non-empty. content: %#v" , tmpMetaDir , files )
2016-06-29 05:28:46 -04:00
}
}
}
2016-07-09 03:45:49 -04:00
// Benchmarks for ObjectLayer.PutObject().
2016-08-15 05:44:48 -04:00
// The intent is to benchmark PutObject for various sizes ranging from few bytes to 100MB.
2020-06-12 23:04:01 -04:00
// Also each of these Benchmarks are run both Erasure and FS backends.
2016-07-09 03:45:49 -04:00
// BenchmarkPutObjectVerySmallFS - Benchmark FS.PutObject() for object size of 10 bytes.
func BenchmarkPutObjectVerySmallFS ( b * testing . B ) {
2016-07-21 14:17:28 -04:00
benchmarkPutObject ( b , "FS" , 10 )
2016-07-09 03:45:49 -04:00
}
2020-06-12 23:04:01 -04:00
// BenchmarkPutObjectVerySmallErasure - Benchmark Erasure.PutObject() for object size of 10 bytes.
func BenchmarkPutObjectVerySmallErasure ( b * testing . B ) {
benchmarkPutObject ( b , "Erasure" , 10 )
2016-07-09 03:45:49 -04:00
}
// BenchmarkPutObject10KbFS - Benchmark FS.PutObject() for object size of 10KB.
func BenchmarkPutObject10KbFS ( b * testing . B ) {
2016-11-22 21:18:22 -05:00
benchmarkPutObject ( b , "FS" , 10 * humanize . KiByte )
2016-07-09 03:45:49 -04:00
}
2020-06-12 23:04:01 -04:00
// BenchmarkPutObject10KbErasure - Benchmark Erasure.PutObject() for object size of 10KB.
func BenchmarkPutObject10KbErasure ( b * testing . B ) {
benchmarkPutObject ( b , "Erasure" , 10 * humanize . KiByte )
2016-07-09 03:45:49 -04:00
}
// BenchmarkPutObject100KbFS - Benchmark FS.PutObject() for object size of 100KB.
func BenchmarkPutObject100KbFS ( b * testing . B ) {
2016-11-22 21:18:22 -05:00
benchmarkPutObject ( b , "FS" , 100 * humanize . KiByte )
2016-07-09 03:45:49 -04:00
}
2020-06-12 23:04:01 -04:00
// BenchmarkPutObject100KbErasure - Benchmark Erasure.PutObject() for object size of 100KB.
func BenchmarkPutObject100KbErasure ( b * testing . B ) {
benchmarkPutObject ( b , "Erasure" , 100 * humanize . KiByte )
2016-07-09 03:45:49 -04:00
}
// BenchmarkPutObject1MbFS - Benchmark FS.PutObject() for object size of 1MB.
func BenchmarkPutObject1MbFS ( b * testing . B ) {
2016-11-22 21:18:22 -05:00
benchmarkPutObject ( b , "FS" , 1 * humanize . MiByte )
2016-07-09 03:45:49 -04:00
}
2020-06-12 23:04:01 -04:00
// BenchmarkPutObject1MbErasure - Benchmark Erasure.PutObject() for object size of 1MB.
func BenchmarkPutObject1MbErasure ( b * testing . B ) {
benchmarkPutObject ( b , "Erasure" , 1 * humanize . MiByte )
2016-07-09 03:45:49 -04:00
}
// BenchmarkPutObject5MbFS - Benchmark FS.PutObject() for object size of 5MB.
func BenchmarkPutObject5MbFS ( b * testing . B ) {
2016-11-22 21:18:22 -05:00
benchmarkPutObject ( b , "FS" , 5 * humanize . MiByte )
2016-07-09 03:45:49 -04:00
}
2020-06-12 23:04:01 -04:00
// BenchmarkPutObject5MbErasure - Benchmark Erasure.PutObject() for object size of 5MB.
func BenchmarkPutObject5MbErasure ( b * testing . B ) {
benchmarkPutObject ( b , "Erasure" , 5 * humanize . MiByte )
2016-07-09 03:45:49 -04:00
}
// BenchmarkPutObject10MbFS - Benchmark FS.PutObject() for object size of 10MB.
func BenchmarkPutObject10MbFS ( b * testing . B ) {
2016-11-22 21:18:22 -05:00
benchmarkPutObject ( b , "FS" , 10 * humanize . MiByte )
2016-07-09 03:45:49 -04:00
}
2020-06-12 23:04:01 -04:00
// BenchmarkPutObject10MbErasure - Benchmark Erasure.PutObject() for object size of 10MB.
func BenchmarkPutObject10MbErasure ( b * testing . B ) {
benchmarkPutObject ( b , "Erasure" , 10 * humanize . MiByte )
2016-07-09 03:45:49 -04:00
}
// BenchmarkPutObject25MbFS - Benchmark FS.PutObject() for object size of 25MB.
func BenchmarkPutObject25MbFS ( b * testing . B ) {
2016-11-22 21:18:22 -05:00
benchmarkPutObject ( b , "FS" , 25 * humanize . MiByte )
2016-07-09 03:45:49 -04:00
}
2020-06-12 23:04:01 -04:00
// BenchmarkPutObject25MbErasure - Benchmark Erasure.PutObject() for object size of 25MB.
func BenchmarkPutObject25MbErasure ( b * testing . B ) {
benchmarkPutObject ( b , "Erasure" , 25 * humanize . MiByte )
2016-07-09 03:45:49 -04:00
}
// BenchmarkPutObject50MbFS - Benchmark FS.PutObject() for object size of 50MB.
func BenchmarkPutObject50MbFS ( b * testing . B ) {
2016-11-22 21:18:22 -05:00
benchmarkPutObject ( b , "FS" , 50 * humanize . MiByte )
2016-07-09 03:45:49 -04:00
}
2020-06-12 23:04:01 -04:00
// BenchmarkPutObject50MbErasure - Benchmark Erasure.PutObject() for object size of 50MB.
func BenchmarkPutObject50MbErasure ( b * testing . B ) {
benchmarkPutObject ( b , "Erasure" , 50 * humanize . MiByte )
2016-07-09 03:45:49 -04:00
}
2016-07-21 14:17:28 -04:00
// parallel benchmarks for ObjectLayer.PutObject() .
2016-07-10 14:08:45 -04:00
2016-07-21 14:17:28 -04:00
// BenchmarkParallelPutObjectVerySmallFS - BenchmarkParallel FS.PutObject() for object size of 10 bytes.
2016-07-10 14:08:45 -04:00
func BenchmarkParallelPutObjectVerySmallFS ( b * testing . B ) {
2016-07-21 14:17:28 -04:00
benchmarkPutObjectParallel ( b , "FS" , 10 )
2016-07-10 14:08:45 -04:00
}
2020-06-12 23:04:01 -04:00
// BenchmarkParallelPutObjectVerySmallErasure - BenchmarkParallel Erasure.PutObject() for object size of 10 bytes.
func BenchmarkParallelPutObjectVerySmallErasure ( b * testing . B ) {
benchmarkPutObjectParallel ( b , "Erasure" , 10 )
2016-07-10 14:08:45 -04:00
}
// BenchmarkParallelPutObject10KbFS - BenchmarkParallel FS.PutObject() for object size of 10KB.
func BenchmarkParallelPutObject10KbFS ( b * testing . B ) {
2016-11-22 21:18:22 -05:00
benchmarkPutObjectParallel ( b , "FS" , 10 * humanize . KiByte )
2016-07-10 14:08:45 -04:00
}
2020-06-12 23:04:01 -04:00
// BenchmarkParallelPutObject10KbErasure - BenchmarkParallel Erasure.PutObject() for object size of 10KB.
func BenchmarkParallelPutObject10KbErasure ( b * testing . B ) {
benchmarkPutObjectParallel ( b , "Erasure" , 10 * humanize . KiByte )
2016-07-10 14:08:45 -04:00
}
// BenchmarkParallelPutObject100KbFS - BenchmarkParallel FS.PutObject() for object size of 100KB.
func BenchmarkParallelPutObject100KbFS ( b * testing . B ) {
2016-11-22 21:18:22 -05:00
benchmarkPutObjectParallel ( b , "FS" , 100 * humanize . KiByte )
2016-07-10 14:08:45 -04:00
}
2020-06-12 23:04:01 -04:00
// BenchmarkParallelPutObject100KbErasure - BenchmarkParallel Erasure.PutObject() for object size of 100KB.
func BenchmarkParallelPutObject100KbErasure ( b * testing . B ) {
benchmarkPutObjectParallel ( b , "Erasure" , 100 * humanize . KiByte )
2016-07-10 14:08:45 -04:00
}
// BenchmarkParallelPutObject1MbFS - BenchmarkParallel FS.PutObject() for object size of 1MB.
func BenchmarkParallelPutObject1MbFS ( b * testing . B ) {
2016-11-22 21:18:22 -05:00
benchmarkPutObjectParallel ( b , "FS" , 1 * humanize . MiByte )
2016-07-10 14:08:45 -04:00
}
2020-06-12 23:04:01 -04:00
// BenchmarkParallelPutObject1MbErasure - BenchmarkParallel Erasure.PutObject() for object size of 1MB.
func BenchmarkParallelPutObject1MbErasure ( b * testing . B ) {
benchmarkPutObjectParallel ( b , "Erasure" , 1 * humanize . MiByte )
2016-07-10 14:08:45 -04:00
}
// BenchmarkParallelPutObject5MbFS - BenchmarkParallel FS.PutObject() for object size of 5MB.
func BenchmarkParallelPutObject5MbFS ( b * testing . B ) {
2016-11-22 21:18:22 -05:00
benchmarkPutObjectParallel ( b , "FS" , 5 * humanize . MiByte )
2016-07-10 14:08:45 -04:00
}
2020-06-12 23:04:01 -04:00
// BenchmarkParallelPutObject5MbErasure - BenchmarkParallel Erasure.PutObject() for object size of 5MB.
func BenchmarkParallelPutObject5MbErasure ( b * testing . B ) {
benchmarkPutObjectParallel ( b , "Erasure" , 5 * humanize . MiByte )
2016-07-10 14:08:45 -04:00
}
// BenchmarkParallelPutObject10MbFS - BenchmarkParallel FS.PutObject() for object size of 10MB.
func BenchmarkParallelPutObject10MbFS ( b * testing . B ) {
2016-11-22 21:18:22 -05:00
benchmarkPutObjectParallel ( b , "FS" , 10 * humanize . MiByte )
2016-07-10 14:08:45 -04:00
}
2020-06-12 23:04:01 -04:00
// BenchmarkParallelPutObject10MbErasure - BenchmarkParallel Erasure.PutObject() for object size of 10MB.
func BenchmarkParallelPutObject10MbErasure ( b * testing . B ) {
benchmarkPutObjectParallel ( b , "Erasure" , 10 * humanize . MiByte )
2016-07-10 14:08:45 -04:00
}
// BenchmarkParallelPutObject25MbFS - BenchmarkParallel FS.PutObject() for object size of 25MB.
func BenchmarkParallelPutObject25MbFS ( b * testing . B ) {
2016-11-22 21:18:22 -05:00
benchmarkPutObjectParallel ( b , "FS" , 25 * humanize . MiByte )
2016-07-10 14:08:45 -04:00
}
2020-06-12 23:04:01 -04:00
// BenchmarkParallelPutObject25MbErasure - BenchmarkParallel Erasure.PutObject() for object size of 25MB.
func BenchmarkParallelPutObject25MbErasure ( b * testing . B ) {
benchmarkPutObjectParallel ( b , "Erasure" , 25 * humanize . MiByte )
2016-07-10 14:08:45 -04:00
}