mirror of https://github.com/minio/minio.git
512 lines
16 KiB
Go
512 lines
16 KiB
Go
/*
|
|
*
|
|
* Mint, (C) 2017 Minio, Inc.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*
|
|
*/
|
|
|
|
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"encoding/xml"
|
|
"errors"
|
|
"fmt"
|
|
"math/rand"
|
|
"net/http"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/aws/aws-sdk-go/aws"
|
|
"github.com/aws/aws-sdk-go/aws/credentials"
|
|
"github.com/aws/aws-sdk-go/aws/session"
|
|
"github.com/aws/aws-sdk-go/service/s3"
|
|
log "github.com/sirupsen/logrus"
|
|
)
|
|
|
|
const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569"
|
|
const (
|
|
letterIdxBits = 6 // 6 bits to represent a letter index
|
|
letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
|
|
letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits
|
|
)
|
|
const (
|
|
PASS = "PASS" // Indicate that a test passed
|
|
FAIL = "FAIL" // Indicate that a test failed
|
|
NA = "NA" // Indicate that a test is not applicable
|
|
)
|
|
|
|
type ErrorResponse struct {
|
|
XMLName xml.Name `xml:"Error" json:"-"`
|
|
Code string
|
|
Message string
|
|
BucketName string
|
|
Key string
|
|
RequestID string `xml:"RequestId"`
|
|
HostID string `xml:"HostId"`
|
|
|
|
// Region where the bucket is located. This header is returned
|
|
// only in HEAD bucket and ListObjects response.
|
|
Region string
|
|
|
|
// Headers of the returned S3 XML error
|
|
Headers http.Header `xml:"-" json:"-"`
|
|
}
|
|
|
|
type mintJSONFormatter struct {
|
|
}
|
|
|
|
func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) {
|
|
data := make(log.Fields, len(entry.Data))
|
|
for k, v := range entry.Data {
|
|
switch v := v.(type) {
|
|
case error:
|
|
// Otherwise errors are ignored by `encoding/json`
|
|
// https://github.com/sirupsen/logrus/issues/137
|
|
data[k] = v.Error()
|
|
default:
|
|
data[k] = v
|
|
}
|
|
}
|
|
|
|
serialized, err := json.Marshal(data)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err)
|
|
}
|
|
return append(serialized, '\n'), nil
|
|
}
|
|
|
|
// log successful test runs
|
|
func successLogger(function string, args map[string]interface{}, startTime time.Time) *log.Entry {
|
|
// calculate the test case duration
|
|
duration := time.Since(startTime)
|
|
// log with the fields as per mint
|
|
fields := log.Fields{"name": "aws-sdk-go", "function": function, "args": args, "duration": duration.Nanoseconds() / 1000000, "status": PASS}
|
|
return log.WithFields(fields)
|
|
}
|
|
|
|
// log failed test runs
|
|
func failureLog(function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry {
|
|
// calculate the test case duration
|
|
duration := time.Since(startTime)
|
|
var fields log.Fields
|
|
// log with the fields as per mint
|
|
if err != nil {
|
|
fields = log.Fields{"name": "aws-sdk-go", "function": function, "args": args,
|
|
"duration": duration.Nanoseconds() / 1000000, "status": FAIL, "alert": alert, "message": message, "error": err}
|
|
} else {
|
|
fields = log.Fields{"name": "aws-sdk-go", "function": function, "args": args,
|
|
"duration": duration.Nanoseconds() / 1000000, "status": FAIL, "alert": alert, "message": message}
|
|
}
|
|
return log.WithFields(fields)
|
|
}
|
|
|
|
func randString(n int, src rand.Source, prefix string) string {
|
|
b := make([]byte, n)
|
|
// A rand.Int63() generates 63 random bits, enough for letterIdxMax letters!
|
|
for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
|
|
if remain == 0 {
|
|
cache, remain = src.Int63(), letterIdxMax
|
|
}
|
|
if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
|
|
b[i] = letterBytes[idx]
|
|
i--
|
|
}
|
|
cache >>= letterIdxBits
|
|
remain--
|
|
}
|
|
return prefix + string(b[0:30-len(prefix)])
|
|
}
|
|
|
|
func cleanup(s3Client *s3.S3, bucket string, object string, function string,
|
|
args map[string]interface{}, startTime time.Time, deleteBucket bool) {
|
|
|
|
// Deleting the object, just in case it was created. Will not check for errors.
|
|
s3Client.DeleteObject(&s3.DeleteObjectInput{
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String(object),
|
|
})
|
|
|
|
if deleteBucket {
|
|
_, err := s3Client.DeleteBucket(&s3.DeleteBucketInput{
|
|
Bucket: aws.String(bucket),
|
|
})
|
|
if err != nil {
|
|
failureLog(function, args, startTime, "", "AWS SDK Go DeleteBucket Failed", err).Fatal()
|
|
return
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
func testPresignedPutInvalidHash(s3Client *s3.S3) {
|
|
startTime := time.Now()
|
|
function := "PresignedPut"
|
|
bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-")
|
|
object := "presignedTest"
|
|
expiry := 1 * time.Minute
|
|
args := map[string]interface{}{
|
|
"bucketName": bucket,
|
|
"objectName": object,
|
|
"expiry": expiry,
|
|
}
|
|
|
|
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
|
|
Bucket: aws.String(bucket),
|
|
})
|
|
if err != nil {
|
|
failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal()
|
|
return
|
|
}
|
|
defer cleanup(s3Client, bucket, object, function, args, startTime, true)
|
|
|
|
req, _ := s3Client.PutObjectRequest(&s3.PutObjectInput{
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String(object),
|
|
ContentType: aws.String("application/octet-stream"),
|
|
})
|
|
|
|
req.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "invalid-sha256")
|
|
url, err := req.Presign(expiry)
|
|
if err != nil {
|
|
failureLog(function, args, startTime, "", "AWS SDK Go presigned Put request creation failed", err).Fatal()
|
|
return
|
|
}
|
|
|
|
rreq, err := http.NewRequest("PUT", url, bytes.NewReader([]byte("")))
|
|
rreq.Header.Add("X-Amz-Content-Sha256", "invalid-sha256")
|
|
rreq.Header.Add("Content-Type", "application/octet-stream")
|
|
|
|
resp, err := http.DefaultClient.Do(rreq)
|
|
if err != nil {
|
|
failureLog(function, args, startTime, "", "AWS SDK Go presigned put request failed", err).Fatal()
|
|
return
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
dec := xml.NewDecoder(resp.Body)
|
|
errResp := ErrorResponse{}
|
|
err = dec.Decode(&errResp)
|
|
if err != nil {
|
|
failureLog(function, args, startTime, "", "AWS SDK Go unmarshalling xml failed", err).Fatal()
|
|
return
|
|
}
|
|
|
|
if errResp.Code != "XAmzContentSHA256Mismatch" {
|
|
failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go presigned PUT expected to fail with XAmzContentSHA256Mismatch but got %v", errResp.Code), errors.New("AWS S3 error code mismatch")).Fatal()
|
|
return
|
|
}
|
|
|
|
successLogger(function, args, startTime).Info()
|
|
}
|
|
|
|
func testListObjects(s3Client *s3.S3) {
|
|
startTime := time.Now()
|
|
function := "testListObjects"
|
|
bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-")
|
|
object1 := "testObject1"
|
|
object2 := "testObject2"
|
|
expiry := 1 * time.Minute
|
|
args := map[string]interface{}{
|
|
"bucketName": bucket,
|
|
"objectName1": object1,
|
|
"objectName2": object2,
|
|
"expiry": expiry,
|
|
}
|
|
|
|
getKeys := func(objects []*s3.Object) []string {
|
|
var rv []string
|
|
for _, obj := range objects {
|
|
rv = append(rv, *obj.Key)
|
|
}
|
|
return rv
|
|
}
|
|
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
|
|
Bucket: aws.String(bucket),
|
|
})
|
|
if err != nil {
|
|
failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal()
|
|
return
|
|
}
|
|
defer cleanup(s3Client, bucket, object1, function, args, startTime, true)
|
|
defer cleanup(s3Client, bucket, object2, function, args, startTime, false)
|
|
|
|
listInput := &s3.ListObjectsV2Input{
|
|
Bucket: aws.String(bucket),
|
|
MaxKeys: aws.Int64(1000),
|
|
Prefix: aws.String(""),
|
|
}
|
|
result, err := s3Client.ListObjectsV2(listInput)
|
|
if err != nil {
|
|
failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go listobjects expected to success but got %v", err), err).Fatal()
|
|
return
|
|
}
|
|
if *result.KeyCount != 0 {
|
|
failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go listobjects with prefix '' expected 0 key but got %v, %v", result.KeyCount, getKeys(result.Contents)), errors.New("AWS S3 key count mismatch")).Fatal()
|
|
return
|
|
}
|
|
putInput1 := &s3.PutObjectInput{
|
|
Body: aws.ReadSeekCloser(strings.NewReader("filetoupload")),
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String(object1),
|
|
}
|
|
_, err = s3Client.PutObject(putInput1)
|
|
if err != nil {
|
|
failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to success but got %v", err), err).Fatal()
|
|
return
|
|
}
|
|
putInput2 := &s3.PutObjectInput{
|
|
Body: aws.ReadSeekCloser(strings.NewReader("filetoupload")),
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String(object2),
|
|
}
|
|
_, err = s3Client.PutObject(putInput2)
|
|
if err != nil {
|
|
failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to success but got %v", err), err).Fatal()
|
|
return
|
|
}
|
|
result, err = s3Client.ListObjectsV2(listInput)
|
|
if err != nil {
|
|
failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go listobjects expected to success but got %v", err), err).Fatal()
|
|
return
|
|
}
|
|
if *result.KeyCount != 2 {
|
|
failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go listobjects with prefix '' expected 2 key but got %v, %v", *result.KeyCount, getKeys(result.Contents)), errors.New("AWS S3 key count mismatch")).Fatal()
|
|
return
|
|
}
|
|
|
|
successLogger(function, args, startTime).Info()
|
|
}
|
|
|
|
func testSelectObject(s3Client *s3.S3) {
|
|
startTime := time.Now()
|
|
function := "testSelectObject"
|
|
bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-")
|
|
object1 := "object1.csv"
|
|
object2 := "object2.csv"
|
|
args := map[string]interface{}{
|
|
"bucketName": bucket,
|
|
"objectName1": object1,
|
|
"objectName2": object2,
|
|
}
|
|
|
|
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
|
|
Bucket: aws.String(bucket),
|
|
})
|
|
if err != nil {
|
|
failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal()
|
|
return
|
|
}
|
|
|
|
// Test comma field separator
|
|
inputCsv1 := `year,gender,ethnicity,firstname,count,rank
|
|
2011,FEMALE,ASIAN AND PACIFIC ISLANDER,SOPHIA,119,1
|
|
2011,FEMALE,ASIAN AND PACIFIC ISLANDER,CHLOE,106,2
|
|
2011,FEMALE,ASIAN AND PACIFIC ISLANDER,EMILY,93,3
|
|
2011,FEMALE,ASIAN AND PACIFIC ISLANDER,OLIVIA,89,4
|
|
2011,FEMALE,ASIAN AND PACIFIC ISLANDER,EMMA,75,5
|
|
2011,FEMALE,ASIAN AND PACIFIC ISLANDER,ISABELLA,67,6
|
|
2011,FEMALE,ASIAN AND PACIFIC ISLANDER,TIFFANY,54,7
|
|
2011,FEMALE,ASIAN AND PACIFIC ISLANDER,ASHLEY,52,8
|
|
2011,FEMALE,ASIAN AND PACIFIC ISLANDER,FIONA,48,9
|
|
2011,FEMALE,ASIAN AND PACIFIC ISLANDER,ANGELA,47,10
|
|
`
|
|
|
|
outputCSV1 := `2011
|
|
2011
|
|
2011
|
|
2011
|
|
2011
|
|
2011
|
|
2011
|
|
2011
|
|
2011
|
|
2011
|
|
`
|
|
|
|
putInput1 := &s3.PutObjectInput{
|
|
Body: aws.ReadSeekCloser(strings.NewReader(inputCsv1)),
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String(object1),
|
|
}
|
|
_, err = s3Client.PutObject(putInput1)
|
|
|
|
defer cleanup(s3Client, bucket, object1, function, args, startTime, true)
|
|
|
|
params := &s3.SelectObjectContentInput{
|
|
Bucket: &bucket,
|
|
Key: &object1,
|
|
ExpressionType: aws.String(s3.ExpressionTypeSql),
|
|
Expression: aws.String("SELECT s._1 FROM S3Object s"),
|
|
RequestProgress: &s3.RequestProgress{},
|
|
InputSerialization: &s3.InputSerialization{
|
|
CompressionType: aws.String("NONE"),
|
|
CSV: &s3.CSVInput{
|
|
FileHeaderInfo: aws.String(s3.FileHeaderInfoIgnore),
|
|
FieldDelimiter: aws.String(","),
|
|
RecordDelimiter: aws.String("\n"),
|
|
},
|
|
},
|
|
OutputSerialization: &s3.OutputSerialization{
|
|
CSV: &s3.CSVOutput{},
|
|
},
|
|
}
|
|
|
|
resp, err := s3Client.SelectObjectContent(params)
|
|
if err != nil {
|
|
failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go Select object failed %v", err), err).Fatal()
|
|
return
|
|
}
|
|
defer resp.EventStream.Close()
|
|
|
|
payload := ""
|
|
for event := range resp.EventStream.Events() {
|
|
switch v := event.(type) {
|
|
case *s3.RecordsEvent:
|
|
// s3.RecordsEvent.Records is a byte slice of select records
|
|
payload = string(v.Payload)
|
|
}
|
|
}
|
|
|
|
if err := resp.EventStream.Err(); err != nil {
|
|
failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go Select object failed %v", err), err).Fatal()
|
|
return
|
|
}
|
|
|
|
if payload != outputCSV1 {
|
|
failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go Select object output mismatch %v", payload), errors.New("AWS S3 select object mismatch")).Fatal()
|
|
return
|
|
}
|
|
|
|
// Test unicode field separator
|
|
inputCsv2 := `"year"╦"gender"╦"ethnicity"╦"firstname"╦"count"╦"rank"
|
|
"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"SOPHIA"╦"119"╦"1"
|
|
"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"CHLOE"╦"106"╦"2"
|
|
"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"EMILY"╦"93"╦"3"
|
|
"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"OLIVIA"╦"89"╦"4"
|
|
"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"EMMA"╦"75"╦"5"
|
|
"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"ISABELLA"╦"67"╦"6"
|
|
"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"TIFFANY"╦"54"╦"7"
|
|
"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"ASHLEY"╦"52"╦"8"
|
|
"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"FIONA"╦"48"╦"9"
|
|
"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"ANGELA"╦"47"╦"10"
|
|
`
|
|
|
|
outputCSV2 := `2011
|
|
2011
|
|
2011
|
|
2011
|
|
2011
|
|
2011
|
|
2011
|
|
2011
|
|
2011
|
|
2011
|
|
`
|
|
|
|
putInput2 := &s3.PutObjectInput{
|
|
Body: aws.ReadSeekCloser(strings.NewReader(inputCsv2)),
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String(object2),
|
|
}
|
|
_, err = s3Client.PutObject(putInput2)
|
|
|
|
defer cleanup(s3Client, bucket, object2, function, args, startTime, false)
|
|
|
|
params2 := &s3.SelectObjectContentInput{
|
|
Bucket: &bucket,
|
|
Key: &object2,
|
|
ExpressionType: aws.String(s3.ExpressionTypeSql),
|
|
Expression: aws.String("SELECT s._1 FROM S3Object s"),
|
|
RequestProgress: &s3.RequestProgress{},
|
|
InputSerialization: &s3.InputSerialization{
|
|
CompressionType: aws.String("NONE"),
|
|
CSV: &s3.CSVInput{
|
|
FileHeaderInfo: aws.String(s3.FileHeaderInfoIgnore),
|
|
FieldDelimiter: aws.String("╦"),
|
|
RecordDelimiter: aws.String("\n"),
|
|
},
|
|
},
|
|
OutputSerialization: &s3.OutputSerialization{
|
|
CSV: &s3.CSVOutput{},
|
|
},
|
|
}
|
|
|
|
resp, err = s3Client.SelectObjectContent(params2)
|
|
if err != nil {
|
|
failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go Select object failed for unicode separator %v", err), err).Fatal()
|
|
return
|
|
}
|
|
defer resp.EventStream.Close()
|
|
|
|
for event := range resp.EventStream.Events() {
|
|
switch v := event.(type) {
|
|
case *s3.RecordsEvent:
|
|
// s3.RecordsEvent.Records is a byte slice of select records
|
|
payload = string(v.Payload)
|
|
}
|
|
}
|
|
|
|
if err := resp.EventStream.Err(); err != nil {
|
|
failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go Select object failed for unicode separator %v", err), err).Fatal()
|
|
return
|
|
}
|
|
|
|
if payload != outputCSV2 {
|
|
failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go Select object output mismatch %v", payload), errors.New("AWS S3 select object mismatch")).Fatal()
|
|
return
|
|
}
|
|
|
|
successLogger(function, args, startTime).Info()
|
|
}
|
|
|
|
func main() {
|
|
endpoint := os.Getenv("SERVER_ENDPOINT")
|
|
accessKey := os.Getenv("ACCESS_KEY")
|
|
secretKey := os.Getenv("SECRET_KEY")
|
|
secure := os.Getenv("ENABLE_HTTPS")
|
|
sdkEndpoint := "http://" + endpoint
|
|
if secure == "1" {
|
|
sdkEndpoint = "https://" + endpoint
|
|
}
|
|
|
|
creds := credentials.NewStaticCredentials(accessKey, secretKey, "")
|
|
newSession := session.New()
|
|
s3Config := &aws.Config{
|
|
Credentials: creds,
|
|
Endpoint: aws.String(sdkEndpoint),
|
|
Region: aws.String("us-east-1"),
|
|
S3ForcePathStyle: aws.Bool(true),
|
|
}
|
|
|
|
// Create an S3 service object in the default region.
|
|
s3Client := s3.New(newSession, s3Config)
|
|
|
|
// Output to stdout instead of the default stderr
|
|
log.SetOutput(os.Stdout)
|
|
// create custom formatter
|
|
mintFormatter := mintJSONFormatter{}
|
|
// set custom formatter
|
|
log.SetFormatter(&mintFormatter)
|
|
// log Info or above -- success cases are Info level, failures are Fatal level
|
|
log.SetLevel(log.InfoLevel)
|
|
// execute tests
|
|
testPresignedPutInvalidHash(s3Client)
|
|
testListObjects(s3Client)
|
|
testSelectObject(s3Client)
|
|
}
|