mirror of
https://github.com/minio/minio.git
synced 2025-11-21 18:26:04 -05:00
migrate all bucket metadata into a single file (#9586)
this is a major overhaul by migrating off all bucket metadata related configs into a single object '.metadata.bin' this allows us for faster bootups across 1000's of buckets and as well as keeps the code simple enough for future work and additions. Additionally also fixes #9396, #9394
This commit is contained in:
Harshavardhana
GitHub
@@ -35,13 +35,8 @@ import (
|
||||
|
||||
jwtgo "github.com/dgrijalva/jwt-go"
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
|
||||
xjwt "github.com/minio/minio/cmd/jwt"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/bucket/policy"
|
||||
"github.com/minio/minio/pkg/bucket/policy/condition"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
)
|
||||
|
||||
// Implement a dummy flush writer.
|
||||
@@ -533,30 +528,6 @@ func testListObjectsWebHandler(obj ObjectLayer, instanceType string, t TestErrHa
|
||||
if err == nil {
|
||||
t.Fatalf("Expected error `%s`", err)
|
||||
}
|
||||
|
||||
bucketPolicy := &policy.Policy{
|
||||
Version: policy.DefaultVersion,
|
||||
Statements: []policy.Statement{policy.NewStatement(
|
||||
policy.Allow,
|
||||
policy.NewPrincipal("*"),
|
||||
policy.NewActionSet(policy.ListBucketAction),
|
||||
policy.NewResourceSet(policy.NewResource(bucketName, "")),
|
||||
condition.NewFunctions(),
|
||||
)},
|
||||
}
|
||||
|
||||
if err = obj.SetBucketPolicy(context.Background(), bucketName, bucketPolicy); err != nil {
|
||||
t.Fatalf("unexpected error. %v", err)
|
||||
}
|
||||
globalPolicySys.Set(bucketName, bucketPolicy)
|
||||
defer globalPolicySys.Remove(bucketName)
|
||||
|
||||
// Unauthenticated ListObjects with READ bucket policy should succeed.
|
||||
reply, err = test("")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
verifyReply(reply)
|
||||
}
|
||||
|
||||
// Wrapper for calling RemoveObject Web Handler
|
||||
@@ -687,69 +658,6 @@ func testGenerateAuthWebHandler(obj ObjectLayer, instanceType string, t TestErrH
|
||||
}
|
||||
}
|
||||
|
||||
// Wrapper for calling Set Auth Handler
|
||||
func TestWebHandlerSetAuth(t *testing.T) {
|
||||
ExecObjectLayerTest(t, testSetAuthWebHandler)
|
||||
}
|
||||
|
||||
// testSetAuthWebHandler - Test SetAuth web handler
|
||||
func testSetAuthWebHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
|
||||
// Register the API end points with XL/FS object layer.
|
||||
apiRouter := initTestWebRPCEndPoint(obj)
|
||||
credentials, err := auth.GetNewCredentials()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
globalIAMSys.SetUser(credentials.AccessKey, madmin.UserInfo{
|
||||
SecretKey: credentials.SecretKey,
|
||||
Status: madmin.AccountEnabled,
|
||||
})
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
authorization, err := getWebRPCToken(apiRouter, credentials.AccessKey, credentials.SecretKey)
|
||||
if err != nil {
|
||||
t.Fatal("Cannot authenticate")
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
currentAccessKey string
|
||||
currentSecretKey string
|
||||
newAccessKey string
|
||||
newSecretKey string
|
||||
success bool
|
||||
}{
|
||||
{"", "", "", "", false},
|
||||
{"1", "1", "1", "1", false},
|
||||
{credentials.AccessKey, credentials.SecretKey, "azerty", "bar", false},
|
||||
{credentials.AccessKey, credentials.SecretKey, "azerty", "foooooooooooooo", true},
|
||||
}
|
||||
|
||||
// Iterating over the test cases, calling the function under test and asserting the response.
|
||||
for i, testCase := range testCases {
|
||||
setAuthRequest := SetAuthArgs{CurrentAccessKey: testCase.currentAccessKey, CurrentSecretKey: testCase.currentSecretKey, NewAccessKey: testCase.newAccessKey, NewSecretKey: testCase.newSecretKey}
|
||||
setAuthReply := &SetAuthReply{}
|
||||
req, err := newTestWebRPCRequest("Web.SetAuth", authorization, setAuthRequest)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: Failed to create HTTP request: <ERROR> %v", i+1, err)
|
||||
}
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("Test %d: Expected the response status to be 200, but instead found `%d`", i+1, rec.Code)
|
||||
}
|
||||
err = getTestWebRPCResponse(rec, &setAuthReply)
|
||||
if testCase.success && err != nil {
|
||||
t.Fatalf("Test %d: Supposed to succeed but failed, %v", i+1, err)
|
||||
}
|
||||
if !testCase.success && err == nil {
|
||||
t.Fatalf("Test %d: Supposed to fail but succeeded, %v", i+1, err)
|
||||
}
|
||||
if testCase.success && setAuthReply.Token == "" {
|
||||
t.Fatalf("Test %d: Failed to set auth keys", i+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebCreateURLToken(t *testing.T) {
|
||||
ExecObjectLayerTest(t, testCreateURLToken)
|
||||
}
|
||||
@@ -892,28 +800,6 @@ func testUploadWebHandler(obj ObjectLayer, instanceType string, t TestErrHandler
|
||||
t.Fatalf("Expected the response status to be 403, but instead found `%d`", code)
|
||||
}
|
||||
|
||||
bucketPolicy := &policy.Policy{
|
||||
Version: policy.DefaultVersion,
|
||||
Statements: []policy.Statement{policy.NewStatement(
|
||||
policy.Allow,
|
||||
policy.NewPrincipal("*"),
|
||||
policy.NewActionSet(policy.PutObjectAction),
|
||||
policy.NewResourceSet(policy.NewResource(bucketName, "*")),
|
||||
condition.NewFunctions(),
|
||||
)},
|
||||
}
|
||||
|
||||
if err := obj.SetBucketPolicy(context.Background(), bucketName, bucketPolicy); err != nil {
|
||||
t.Fatalf("unexpected error. %v", err)
|
||||
}
|
||||
globalPolicySys.Set(bucketName, bucketPolicy)
|
||||
defer globalPolicySys.Remove(bucketName)
|
||||
|
||||
// Unauthenticated upload with WRITE policy should succeed.
|
||||
code = test("", true)
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("Expected the response status to be 200, but instead found `%d`", code)
|
||||
}
|
||||
}
|
||||
|
||||
// Wrapper for calling Download Handler
|
||||
@@ -1010,33 +896,6 @@ func testDownloadWebHandler(obj ObjectLayer, instanceType string, t TestErrHandl
|
||||
if code != http.StatusForbidden {
|
||||
t.Fatalf("Expected the response status to be 403, but instead found `%d`", code)
|
||||
}
|
||||
|
||||
bucketPolicy := &policy.Policy{
|
||||
Version: policy.DefaultVersion,
|
||||
Statements: []policy.Statement{policy.NewStatement(
|
||||
policy.Allow,
|
||||
policy.NewPrincipal("*"),
|
||||
policy.NewActionSet(policy.GetObjectAction),
|
||||
policy.NewResourceSet(policy.NewResource(bucketName, "*")),
|
||||
condition.NewFunctions(),
|
||||
)},
|
||||
}
|
||||
|
||||
if err := obj.SetBucketPolicy(context.Background(), bucketName, bucketPolicy); err != nil {
|
||||
t.Fatalf("unexpected error. %v", err)
|
||||
}
|
||||
globalPolicySys.Set(bucketName, bucketPolicy)
|
||||
defer globalPolicySys.Remove(bucketName)
|
||||
|
||||
// Unauthenticated download with READ policy should succeed.
|
||||
code, bodyContent = test("")
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("Expected the response status to be 200, but instead found `%d`", code)
|
||||
}
|
||||
|
||||
if !bytes.Equal(bodyContent, content) {
|
||||
t.Fatalf("The downloaded file is corrupted")
|
||||
}
|
||||
}
|
||||
|
||||
// Test web.DownloadZip
|
||||
@@ -1229,251 +1088,6 @@ func testWebPresignedGetHandler(obj ObjectLayer, instanceType string, t TestErrH
|
||||
}
|
||||
}
|
||||
|
||||
// Wrapper for calling GetBucketPolicy Handler
|
||||
func TestWebHandlerGetBucketPolicyHandler(t *testing.T) {
|
||||
ExecObjectLayerTest(t, testWebGetBucketPolicyHandler)
|
||||
}
|
||||
|
||||
// testWebGetBucketPolicyHandler - Test GetBucketPolicy web handler
|
||||
func testWebGetBucketPolicyHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
|
||||
// Register the API end points with XL/FS object layer.
|
||||
apiRouter := initTestWebRPCEndPoint(obj)
|
||||
credentials := globalActiveCred
|
||||
|
||||
authorization, err := getWebRPCToken(apiRouter, credentials.AccessKey, credentials.SecretKey)
|
||||
if err != nil {
|
||||
t.Fatal("Cannot authenticate")
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
bucketName := getRandomBucketName()
|
||||
if err = obj.MakeBucketWithLocation(context.Background(), bucketName, "", false); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
bucketPolicy := &policy.Policy{
|
||||
Version: policy.DefaultVersion,
|
||||
Statements: []policy.Statement{
|
||||
policy.NewStatement(
|
||||
policy.Allow,
|
||||
policy.NewPrincipal("*"),
|
||||
policy.NewActionSet(policy.GetBucketLocationAction, policy.ListBucketAction),
|
||||
policy.NewResourceSet(policy.NewResource(bucketName, "")),
|
||||
condition.NewFunctions(),
|
||||
),
|
||||
policy.NewStatement(
|
||||
policy.Allow,
|
||||
policy.NewPrincipal("*"),
|
||||
policy.NewActionSet(policy.GetObjectAction),
|
||||
policy.NewResourceSet(policy.NewResource(bucketName, "*")),
|
||||
condition.NewFunctions(),
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
if err = savePolicyConfig(context.Background(), obj, bucketName, bucketPolicy); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
bucketName string
|
||||
prefix string
|
||||
expectedResult miniogopolicy.BucketPolicy
|
||||
}{
|
||||
{bucketName, "", miniogopolicy.BucketPolicyReadOnly},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
args := &GetBucketPolicyArgs{BucketName: testCase.bucketName, Prefix: testCase.prefix}
|
||||
reply := &GetBucketPolicyRep{}
|
||||
req, err := newTestWebRPCRequest("Web.GetBucketPolicy", authorization, args)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: Failed to create HTTP request: <ERROR> %v", i+1, err)
|
||||
}
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("Test %d: Expected the response status to be 200, but instead found `%d`", i+1, rec.Code)
|
||||
}
|
||||
if err = getTestWebRPCResponse(rec, &reply); err != nil {
|
||||
t.Fatalf("Test %d: Should succeed but it didn't, %v", i+1, err)
|
||||
}
|
||||
if testCase.expectedResult != reply.Policy {
|
||||
t.Fatalf("Test %d: expected: %v, got: %v", i+1, testCase.expectedResult, reply.Policy)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Wrapper for calling ListAllBucketPolicies Handler
|
||||
func TestWebHandlerListAllBucketPoliciesHandler(t *testing.T) {
|
||||
ExecObjectLayerTest(t, testWebListAllBucketPoliciesHandler)
|
||||
}
|
||||
|
||||
// testWebListAllBucketPoliciesHandler - Test ListAllBucketPolicies web handler
|
||||
func testWebListAllBucketPoliciesHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
|
||||
// Register the API end points with XL/FS object layer.
|
||||
apiRouter := initTestWebRPCEndPoint(obj)
|
||||
credentials := globalActiveCred
|
||||
|
||||
authorization, err := getWebRPCToken(apiRouter, credentials.AccessKey, credentials.SecretKey)
|
||||
if err != nil {
|
||||
t.Fatal("Cannot authenticate")
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
bucketName := getRandomBucketName()
|
||||
if err = obj.MakeBucketWithLocation(context.Background(), bucketName, "", false); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
func1, err := condition.NewStringEqualsFunc(condition.S3Prefix, "hello")
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to create string equals condition function. %v", err)
|
||||
}
|
||||
|
||||
bucketPolicy := &policy.Policy{
|
||||
Version: policy.DefaultVersion,
|
||||
Statements: []policy.Statement{
|
||||
policy.NewStatement(
|
||||
policy.Allow,
|
||||
policy.NewPrincipal("*"),
|
||||
policy.NewActionSet(policy.GetBucketLocationAction),
|
||||
policy.NewResourceSet(policy.NewResource(bucketName, "")),
|
||||
condition.NewFunctions(),
|
||||
),
|
||||
policy.NewStatement(
|
||||
policy.Allow,
|
||||
policy.NewPrincipal("*"),
|
||||
policy.NewActionSet(policy.ListBucketAction),
|
||||
policy.NewResourceSet(policy.NewResource(bucketName, "")),
|
||||
condition.NewFunctions(func1),
|
||||
),
|
||||
policy.NewStatement(
|
||||
policy.Allow,
|
||||
policy.NewPrincipal("*"),
|
||||
policy.NewActionSet(policy.ListBucketMultipartUploadsAction),
|
||||
policy.NewResourceSet(policy.NewResource(bucketName, "")),
|
||||
condition.NewFunctions(),
|
||||
),
|
||||
policy.NewStatement(
|
||||
policy.Allow,
|
||||
policy.NewPrincipal("*"),
|
||||
policy.NewActionSet(
|
||||
policy.AbortMultipartUploadAction,
|
||||
policy.DeleteObjectAction,
|
||||
policy.GetObjectAction,
|
||||
policy.ListMultipartUploadPartsAction,
|
||||
policy.PutObjectAction,
|
||||
),
|
||||
policy.NewResourceSet(policy.NewResource(bucketName, "hello*")),
|
||||
condition.NewFunctions(),
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
if err = savePolicyConfig(context.Background(), obj, bucketName, bucketPolicy); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
testCaseResult1 := []BucketAccessPolicy{{
|
||||
Bucket: bucketName,
|
||||
Prefix: "hello",
|
||||
Policy: miniogopolicy.BucketPolicyReadWrite,
|
||||
}}
|
||||
testCases := []struct {
|
||||
bucketName string
|
||||
expectedResult []BucketAccessPolicy
|
||||
}{
|
||||
{bucketName, testCaseResult1},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
args := &ListAllBucketPoliciesArgs{BucketName: testCase.bucketName}
|
||||
reply := &ListAllBucketPoliciesRep{}
|
||||
req, err := newTestWebRPCRequest("Web.ListAllBucketPolicies", authorization, args)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: Failed to create HTTP request: <ERROR> %v", i+1, err)
|
||||
}
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("Test %d: Expected the response status to be 200, but instead found `%d`", i+1, rec.Code)
|
||||
}
|
||||
if err = getTestWebRPCResponse(rec, &reply); err != nil {
|
||||
t.Fatalf("Test %d: Should succeed but it didn't, %v", i+1, err)
|
||||
}
|
||||
if !reflect.DeepEqual(testCase.expectedResult, reply.Policies) {
|
||||
t.Fatalf("Test %d: expected: %v, got: %v", i+1, testCase.expectedResult, reply.Policies)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Wrapper for calling SetBucketPolicy Handler
|
||||
func TestWebHandlerSetBucketPolicyHandler(t *testing.T) {
|
||||
ExecObjectLayerTest(t, testWebSetBucketPolicyHandler)
|
||||
}
|
||||
|
||||
// testWebSetBucketPolicyHandler - Test SetBucketPolicy web handler
|
||||
func testWebSetBucketPolicyHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
|
||||
// Register the API end points with XL/FS object layer.
|
||||
apiRouter := initTestWebRPCEndPoint(obj)
|
||||
credentials := globalActiveCred
|
||||
|
||||
authorization, err := getWebRPCToken(apiRouter, credentials.AccessKey, credentials.SecretKey)
|
||||
if err != nil {
|
||||
t.Fatal("Cannot authenticate")
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
// Create a bucket
|
||||
bucketName := getRandomBucketName()
|
||||
if err = obj.MakeBucketWithLocation(context.Background(), bucketName, "", false); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
bucketName string
|
||||
prefix string
|
||||
policy string
|
||||
pass bool
|
||||
}{
|
||||
// Invalid bucket name
|
||||
{"", "", "readonly", false},
|
||||
// Invalid policy
|
||||
{bucketName, "", "foo", false},
|
||||
// Valid parameters
|
||||
{bucketName, "", "readwrite", true},
|
||||
// None is valid and policy should be removed.
|
||||
{bucketName, "", "none", true},
|
||||
// Setting none again meants should return an error.
|
||||
{bucketName, "", "none", false},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
args := &SetBucketPolicyWebArgs{BucketName: testCase.bucketName, Prefix: testCase.prefix, Policy: testCase.policy}
|
||||
reply := &WebGenericRep{}
|
||||
// Call SetBucketPolicy RPC
|
||||
req, err := newTestWebRPCRequest("Web.SetBucketPolicy", authorization, args)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: Failed to create HTTP request: <ERROR> %v", i+1, err)
|
||||
}
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
// Check if we have 200 OK
|
||||
if testCase.pass && rec.Code != http.StatusOK {
|
||||
t.Fatalf("Test %d: Expected the response status to be 200, but instead found `%d`", i+1, rec.Code)
|
||||
}
|
||||
// Parse RPC response
|
||||
err = getTestWebRPCResponse(rec, &reply)
|
||||
if testCase.pass && err != nil {
|
||||
t.Fatalf("Test %d: Should succeed but it didn't, %#v", i+1, err)
|
||||
}
|
||||
if !testCase.pass && err == nil {
|
||||
t.Fatalf("Test %d: Should fail it didn't", i+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestWebCheckAuthorization - Test Authorization for all web handlers
|
||||
func TestWebCheckAuthorization(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
Reference in New Issue
Block a user