tests: Add ListBucketHandler tests. (#2701)

part-3 final fix for #2412
This commit is contained in:
Harshavardhana 2016-09-14 23:53:42 -07:00 committed by GitHub
parent 6f73d597e0
commit 03430d0db8
9 changed files with 170 additions and 53 deletions

View File

@ -300,3 +300,74 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType string, t Tes
t.Errorf("Test %s: Expected the response status to be `http.StatusForbidden`, but instead found `%d`", instanceType, rec.Code) t.Errorf("Test %s: Expected the response status to be `http.StatusForbidden`, but instead found `%d`", instanceType, rec.Code)
} }
} }
// Wrapper for calling TestListBucketsHandler tests for both XL multiple disks and single node setup.
func TestListBucketsHandler(t *testing.T) {
ExecObjectLayerTest(t, testListBuckets)
}
// testListBucketsHandler - Tests validate listing of buckets.
func testListBucketsHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
// get random bucket name.
bucketName := getRandomBucketName()
// Register the API end points with XL/FS object layer.
apiRouter := initTestAPIEndPoints(obj, []string{"ListBuckets"})
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
rootPath, err := newTestConfig("us-east-1")
if err != nil {
t.Fatalf("Init Test config failed")
}
// remove the root folder after the test ends.
defer removeAll(rootPath)
credentials := serverConfig.GetCredential()
// bucketnames[0].
// objectNames[0].
// uploadIds [0].
// Create bucket before initiating NewMultipartUpload.
err = obj.MakeBucket(bucketName)
if err != nil {
// Failed to create newbucket, abort.
t.Fatalf("%s : %s", instanceType, err.Error())
}
testCases := []struct {
bucketName string
accessKey string
secretKey string
expectedRespStatus int
}{
// Validate a good case request succeeds.
{
bucketName: bucketName,
accessKey: credentials.AccessKeyID,
secretKey: credentials.SecretAccessKey,
expectedRespStatus: http.StatusOK,
},
// Validate a bad case request fails with http.StatusForbidden.
{
bucketName: bucketName,
accessKey: "",
secretKey: "",
expectedRespStatus: http.StatusForbidden,
},
}
for i, testCase := range testCases {
// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
rec := httptest.NewRecorder()
req, lerr := newTestSignedRequest("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey)
if lerr != nil {
t.Fatalf("Test %d: %s: Failed to create HTTP request for ListBucketsHandler: <ERROR> %v", i+1, instanceType, lerr)
}
// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
// Call the ServeHTTP to execute the handler.
apiRouter.ServeHTTP(rec, req)
if rec.Code != testCase.expectedRespStatus {
t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
}
}
}

View File

@ -222,7 +222,7 @@ func TestErasureReadUtils(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
objLayer, err := getXLObjectLayer(disks) objLayer, err := getXLObjectLayer(disks, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@ -99,7 +99,7 @@ func TestInitEventNotifier(t *testing.T) {
if err != nil { if err != nil {
t.Fatal("Unable to create directories for XL backend. ", err) t.Fatal("Unable to create directories for XL backend. ", err)
} }
xl, err := getXLObjectLayer(disks) xl, err := getXLObjectLayer(disks, nil)
if err != nil { if err != nil {
t.Fatal("Unable to initialize XL backend.", err) t.Fatal("Unable to initialize XL backend.", err)
} }

View File

@ -268,7 +268,7 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
// Create an instance of xl backend. // Create an instance of xl backend.
obj, err := getXLObjectLayer(fsDirs) obj, err := getXLObjectLayer(fsDirs, nil)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
@ -300,7 +300,7 @@ func TestFormatXLHealFreshDisksErrorExpected(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
// Create an instance of xl backend. // Create an instance of xl backend.
obj, err := getXLObjectLayer(fsDirs) obj, err := getXLObjectLayer(fsDirs, nil)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
@ -584,7 +584,7 @@ func TestInitFormatXLErrors(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
// Create an instance of xl backend. // Create an instance of xl backend.
obj, err := getXLObjectLayer(fsDirs) obj, err := getXLObjectLayer(fsDirs, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -680,7 +680,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
} }
// Create an instance of xl backend. // Create an instance of xl backend.
obj, err := getXLObjectLayer(fsDirs) obj, err := getXLObjectLayer(fsDirs, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -705,7 +705,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
obj, err = getXLObjectLayer(fsDirs) obj, err = getXLObjectLayer(fsDirs, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -730,7 +730,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
obj, err = getXLObjectLayer(fsDirs) obj, err = getXLObjectLayer(fsDirs, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -753,7 +753,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
obj, err = getXLObjectLayer(fsDirs) obj, err = getXLObjectLayer(fsDirs, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -779,7 +779,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
} }
// Everything is fine, should return nil // Everything is fine, should return nil
obj, err := getXLObjectLayer(fsDirs) obj, err := getXLObjectLayer(fsDirs, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -796,7 +796,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
} }
// Disks 0..15 are nil // Disks 0..15 are nil
obj, err = getXLObjectLayer(fsDirs) obj, err = getXLObjectLayer(fsDirs, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -815,7 +815,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
} }
// One disk returns Faulty Disk // One disk returns Faulty Disk
obj, err = getXLObjectLayer(fsDirs) obj, err = getXLObjectLayer(fsDirs, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -836,7 +836,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
} }
// One disk is not found, heal corrupted disks should return nil // One disk is not found, heal corrupted disks should return nil
obj, err = getXLObjectLayer(fsDirs) obj, err = getXLObjectLayer(fsDirs, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -853,7 +853,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
} }
// Remove format.json of all disks // Remove format.json of all disks
obj, err = getXLObjectLayer(fsDirs) obj, err = getXLObjectLayer(fsDirs, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -874,7 +874,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
} }
// Corrupted format json in one disk // Corrupted format json in one disk
obj, err = getXLObjectLayer(fsDirs) obj, err = getXLObjectLayer(fsDirs, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -899,7 +899,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
} }
// Everything is fine, should return nil // Everything is fine, should return nil
obj, err := getXLObjectLayer(fsDirs) obj, err := getXLObjectLayer(fsDirs, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -915,7 +915,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
} }
// Disks 0..15 are nil // Disks 0..15 are nil
obj, err = getXLObjectLayer(fsDirs) obj, err = getXLObjectLayer(fsDirs, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -934,7 +934,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
} }
// One disk returns Faulty Disk // One disk returns Faulty Disk
obj, err = getXLObjectLayer(fsDirs) obj, err = getXLObjectLayer(fsDirs, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -955,7 +955,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
} }
// One disk is not found, heal corrupted disks should return nil // One disk is not found, heal corrupted disks should return nil
obj, err = getXLObjectLayer(fsDirs) obj, err = getXLObjectLayer(fsDirs, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -972,7 +972,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
} }
// Remove format.json of all disks // Remove format.json of all disks
obj, err = getXLObjectLayer(fsDirs) obj, err = getXLObjectLayer(fsDirs, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -993,7 +993,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
} }
// Remove format.json of all disks // Remove format.json of all disks
obj, err = getXLObjectLayer(fsDirs) obj, err = getXLObjectLayer(fsDirs, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@ -20,6 +20,7 @@ import (
"bytes" "bytes"
"encoding/base64" "encoding/base64"
"fmt" "fmt"
"io/ioutil"
"mime/multipart" "mime/multipart"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
@ -80,6 +81,8 @@ func testPostPolicyHandler(obj ObjectLayer, instanceType string, t TestErrHandle
// remove the root folder after the test ends. // remove the root folder after the test ends.
defer removeAll(rootPath) defer removeAll(rootPath)
credentials := serverConfig.GetCredential()
// bucketnames[0]. // bucketnames[0].
// objectNames[0]. // objectNames[0].
// uploadIds [0]. // uploadIds [0].
@ -96,31 +99,50 @@ func testPostPolicyHandler(obj ObjectLayer, instanceType string, t TestErrHandle
objectName string objectName string
data []byte data []byte
expectedRespStatus int expectedRespStatus int
shouldPass bool accessKey string
secretKey string
malformedBody bool
}{ }{
// Success case. // Success case.
{ {
objectName: "test", objectName: "test",
data: []byte("Hello, World"), data: []byte("Hello, World"),
expectedRespStatus: http.StatusNoContent, expectedRespStatus: http.StatusNoContent,
shouldPass: true, accessKey: credentials.AccessKeyID,
secretKey: credentials.SecretAccessKey,
malformedBody: false,
}, },
// Bad case. // Bad case invalid request.
{ {
objectName: "test", objectName: "test",
data: []byte("Hello, World"), data: []byte("Hello, World"),
expectedRespStatus: http.StatusBadRequest, expectedRespStatus: http.StatusBadRequest,
shouldPass: false, accessKey: "",
secretKey: "",
malformedBody: false,
},
// Bad case malformed input.
{
objectName: "test",
data: []byte("Hello, World"),
expectedRespStatus: http.StatusBadRequest,
accessKey: credentials.AccessKeyID,
secretKey: credentials.SecretAccessKey,
malformedBody: true,
}, },
} }
for i, testCase := range testCases { for i, testCase := range testCases {
// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
rec := httptest.NewRecorder() rec := httptest.NewRecorder()
req, perr := newPostRequest("", bucketName, testCase.objectName, testCase.data, testCase.shouldPass) req, perr := newPostRequest("", bucketName, testCase.objectName, testCase.data, testCase.accessKey, testCase.secretKey)
if perr != nil { if perr != nil {
t.Fatalf("Test %d: %s: Failed to create HTTP request for PostPolicyHandler: <ERROR> %v", i+1, instanceType, perr) t.Fatalf("Test %d: %s: Failed to create HTTP request for PostPolicyHandler: <ERROR> %v", i+1, instanceType, perr)
} }
if testCase.malformedBody {
// Change the request body.
req.Body = ioutil.NopCloser(bytes.NewReader([]byte("Hello,")))
}
// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler. // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
// Call the ServeHTTP to execute the handler. // Call the ServeHTTP to execute the handler.
apiRouter.ServeHTTP(rec, req) apiRouter.ServeHTTP(rec, req)
@ -128,7 +150,6 @@ func testPostPolicyHandler(obj ObjectLayer, instanceType string, t TestErrHandle
t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
} }
} }
} }
// postPresignSignatureV4 - presigned signature for PostPolicy requests. // postPresignSignatureV4 - presigned signature for PostPolicy requests.
@ -140,33 +161,29 @@ func postPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, l
return signature return signature
} }
func newPostRequest(endPoint, bucketName, objectName string, objData []byte, shouldPass bool) (*http.Request, error) { func newPostRequest(endPoint, bucketName, objectName string, objData []byte, accessKey, secretKey string) (*http.Request, error) {
// Keep time. // Keep time.
t := time.Now().UTC() t := time.Now().UTC()
// Expire the request five minutes from now. // Expire the request five minutes from now.
expirationTime := t.Add(time.Minute * 5) expirationTime := t.Add(time.Minute * 5)
// Get the user credential. // Get the user credential.
credentials := serverConfig.GetCredential() credStr := getCredential(accessKey, serverConfig.GetRegion(), t)
credStr := getCredential(credentials.AccessKeyID, serverConfig.GetRegion(), t)
// Create a new post policy. // Create a new post policy.
policy := newPostPolicyBytes(credStr, bucketName, objectName, expirationTime) policy := newPostPolicyBytes(credStr, bucketName, objectName, expirationTime)
// Only need the encoding. // Only need the encoding.
encodedPolicy := base64.StdEncoding.EncodeToString(policy) encodedPolicy := base64.StdEncoding.EncodeToString(policy)
formData := make(map[string]string) // Presign with V4 signature based on the policy.
if shouldPass { signature := postPresignSignatureV4(encodedPolicy, t, secretKey, serverConfig.GetRegion())
// Presign with V4 signature based on the policy.
signature := postPresignSignatureV4(encodedPolicy, t, credentials.SecretAccessKey, serverConfig.GetRegion())
formData = map[string]string{ formData := map[string]string{
"bucket": bucketName, "bucket": bucketName,
"key": objectName, "key": objectName,
"x-amz-credential": credStr, "x-amz-credential": credStr,
"policy": encodedPolicy, "policy": encodedPolicy,
"x-amz-signature": signature, "x-amz-signature": signature,
"x-amz-date": t.Format(iso8601DateFormat), "x-amz-date": t.Format(iso8601DateFormat),
"x-amz-algorithm": "AWS4-HMAC-SHA256", "x-amz-algorithm": "AWS4-HMAC-SHA256",
}
} }
// Create the multipart form. // Create the multipart form.

View File

@ -211,6 +211,7 @@ func waitForFormattingDisks(disks, ignoredDisks []string) ([]StorageAPI, error)
for index, disk := range disks { for index, disk := range disks {
// Check if disk is ignored. // Check if disk is ignored.
if disksSet.Contains(disk) { if disksSet.Contains(disk) {
// Set this situation as disk not found.
storageDisks[index] = nil storageDisks[index] = nil
continue continue
} }
@ -222,6 +223,6 @@ func waitForFormattingDisks(disks, ignoredDisks []string) ([]StorageAPI, error)
} }
storageDisks[index] = storage storageDisks[index] = storage
} }
// Start wait loop retrying formatting disks.
return retryFormattingDisks(disks, storageDisks) return retryFormattingDisks(disks, storageDisks)
} }

View File

@ -68,7 +68,7 @@ func prepareXL() (ObjectLayer, []string, error) {
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
obj, err := getXLObjectLayer(fsDirs) obj, err := getXLObjectLayer(fsDirs, nil)
if err != nil { if err != nil {
removeRoots(fsDirs) removeRoots(fsDirs)
return nil, nil, err return nil, nil, err
@ -498,6 +498,8 @@ func signRequest(req *http.Request, accessKey, secretKey string) error {
} }
sort.Strings(headers) sort.Strings(headers)
region := serverConfig.GetRegion()
// Get canonical headers. // Get canonical headers.
var buf bytes.Buffer var buf bytes.Buffer
for _, k := range headers { for _, k := range headers {
@ -549,7 +551,7 @@ func signRequest(req *http.Request, accessKey, secretKey string) error {
// Get scope. // Get scope.
scope := strings.Join([]string{ scope := strings.Join([]string{
currTime.Format(yyyymmdd), currTime.Format(yyyymmdd),
"us-east-1", region,
"s3", "s3",
"aws4_request", "aws4_request",
}, "/") }, "/")
@ -559,8 +561,8 @@ func signRequest(req *http.Request, accessKey, secretKey string) error {
stringToSign = stringToSign + hex.EncodeToString(sum256([]byte(canonicalRequest))) stringToSign = stringToSign + hex.EncodeToString(sum256([]byte(canonicalRequest)))
date := sumHMAC([]byte("AWS4"+secretKey), []byte(currTime.Format(yyyymmdd))) date := sumHMAC([]byte("AWS4"+secretKey), []byte(currTime.Format(yyyymmdd)))
region := sumHMAC(date, []byte("us-east-1")) regionHMAC := sumHMAC(date, []byte(region))
service := sumHMAC(region, []byte("s3")) service := sumHMAC(regionHMAC, []byte("s3"))
signingKey := sumHMAC(service, []byte("aws4_request")) signingKey := sumHMAC(service, []byte("aws4_request"))
signature := hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign))) signature := hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
@ -731,7 +733,7 @@ func makeTestBackend(disks []string, instanceType string) (ObjectLayer, error) {
return objLayer, err return objLayer, err
case "XL": case "XL":
objectLayer, err := getXLObjectLayer(disks) objectLayer, err := getXLObjectLayer(disks, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -1107,12 +1109,12 @@ func getRandomDisks(N int) ([]string, error) {
} }
// getXLObjectLayer - Instantiates XL object layer and returns it. // getXLObjectLayer - Instantiates XL object layer and returns it.
func getXLObjectLayer(erasureDisks []string) (ObjectLayer, error) { func getXLObjectLayer(erasureDisks []string, ignoredDisks []string) (ObjectLayer, error) {
err := formatDisks(erasureDisks, nil) err := formatDisks(erasureDisks, ignoredDisks)
if err != nil { if err != nil {
return nil, err return nil, err
} }
objLayer, err := newXLObjects(erasureDisks, nil) objLayer, err := newXLObjects(erasureDisks, ignoredDisks)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -1260,7 +1262,7 @@ func ExecObjectLayerStaleFilesTest(t *testing.T, objTest objTestStaleFilesType)
if err != nil { if err != nil {
t.Fatalf("Initialization of disks for XL setup: %s", err) t.Fatalf("Initialization of disks for XL setup: %s", err)
} }
objLayer, err := getXLObjectLayer(erasureDisks) objLayer, err := getXLObjectLayer(erasureDisks, nil)
if err != nil { if err != nil {
t.Fatalf("Initialization of object layer failed for XL setup: %s", err) t.Fatalf("Initialization of object layer failed for XL setup: %s", err)
} }
@ -1288,6 +1290,9 @@ func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Hand
// Iterate the list of API functions requested for and register them in mux HTTP handler. // Iterate the list of API functions requested for and register them in mux HTTP handler.
for _, apiFunction := range apiFunctions { for _, apiFunction := range apiFunctions {
switch apiFunction { switch apiFunction {
// Register ListBuckets handler.
case "ListBuckets":
apiRouter.Methods("GET").HandlerFunc(api.ListBucketsHandler)
// Register GetObject handler. // Register GetObject handler.
case "GetObject`": case "GetObject`":
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.GetObjectHandler) bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.GetObjectHandler)

View File

@ -200,6 +200,10 @@ func (d byDiskTotal) Less(i, j int) bool {
func (xl xlObjects) StorageInfo() StorageInfo { func (xl xlObjects) StorageInfo() StorageInfo {
var disksInfo []disk.Info var disksInfo []disk.Info
for _, storageDisk := range xl.storageDisks { for _, storageDisk := range xl.storageDisks {
if storageDisk == nil {
// Storage disk is empty, perhaps ignored disk or not available.
continue
}
info, err := storageDisk.DiskInfo() info, err := storageDisk.DiskInfo()
if err != nil { if err != nil {
errorIf(err, "Unable to fetch disk info for %#v", storageDisk) errorIf(err, "Unable to fetch disk info for %#v", storageDisk)

View File

@ -115,6 +115,25 @@ func TestStorageInfo(t *testing.T) {
if disks16Info.Total <= 0 { if disks16Info.Total <= 0 {
t.Fatalf("Diskinfo total values should be greater 0") t.Fatalf("Diskinfo total values should be greater 0")
} }
objLayer, err = newXLObjects(fsDirs, fsDirs[:4])
if err != nil {
t.Fatalf("Unable to initialize 'XL' object layer with ignored disks %s.", fsDirs[:4])
}
// Get storage info first attempt.
disks16Info = objLayer.StorageInfo()
// This test assumes homogenity between all disks,
// i.e if we loose one disk the effective storage
// usage values is assumed to decrease. If we have
// heterogenous environment this is not true all the time.
if disks16Info.Free <= 0 {
t.Fatalf("Diskinfo total free values should be greater 0")
}
if disks16Info.Total <= 0 {
t.Fatalf("Diskinfo total values should be greater 0")
}
} }
// TestNewXL - tests initialization of all input disks // TestNewXL - tests initialization of all input disks