MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 12:52:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

migrate all bucket metadata into a single file (#9586)

Browse Source 
this is a major overhaul by migrating off all
bucket metadata related configs into a single
object '.metadata.bin' this allows us for faster
bootups across 1000's of buckets and as well
as keeps the code simple enough for future
work and additions.

Additionally also fixes #9396, #9394
This commit is contained in:
Harshavardhana
2020-05-19 13:53:54 -07:00
committed by GitHub
parent 730775bb4e
commit bd032d13ff
48 changed files with 1648 additions and 2845 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
cmd/bucket-policy.go
View File

@@ -18,14 +18,12 @@ package cmd
import (
"bytes"
"context"
"encoding/json"
"errors"
"net/http"
"net/url"
"path"
"strconv"
"strings"
"sync"
"time"
jsoniter "github.com/json-iterator/go"
@@ -38,53 +36,58 @@ import (
// PolicySys - policy subsystem.
type PolicySys struct {
sync.RWMutex
bucketPolicyMap map[string]*policy.Policy
}
// Set - sets policy to given bucket name. If policy is empty, existing policy is removed.
func (sys *PolicySys) Set(bucketName string, policy *policy.Policy) {
// Get returns stored bucket policy
func (sys *PolicySys) Get(bucket string) (*policy.Policy, error) {
if globalIsGateway {
// Set policy is a non-op under gateway mode.
return
objAPI := newObjectLayerFn()
if objAPI == nil {
return nil, errServerNotInitialized
}
return objAPI.GetBucketPolicy(GlobalContext, bucket)
}
sys.Lock()
defer sys.Unlock()
if policy.IsEmpty() {
delete(sys.bucketPolicyMap, bucketName)
} else {
sys.bucketPolicyMap[bucketName] = policy
configData, err := globalBucketMetadataSys.GetConfig(bucket, bucketPolicyConfig)
if err != nil {
if !errors.Is(err, errConfigNotFound) {
return nil, err
}
return nil, BucketPolicyNotFound{Bucket: bucket}
}
}
// Remove - removes policy for given bucket name.
func (sys *PolicySys) Remove(bucketName string) {
sys.Lock()
defer sys.Unlock()
delete(sys.bucketPolicyMap, bucketName)
return policy.ParseConfig(bytes.NewReader(configData), bucket)
}
// IsAllowed - checks given policy args is allowed to continue the Rest API.
func (sys *PolicySys) IsAllowed(args policy.Args) bool {
if globalIsGateway {
// When gateway is enabled, no cached value
// is used to validate bucket policies.
objAPI := newObjectLayerFn()
if objAPI != nil {
config, err := objAPI.GetBucketPolicy(GlobalContext, args.BucketName)
if err == nil {
return config.IsAllowed(args)
}
if objAPI == nil {
return false
}
p, err := objAPI.GetBucketPolicy(GlobalContext, args.BucketName)
if err == nil {
return p.IsAllowed(args)
}
return args.IsOwner
}
// If policy is available for given bucket, check the policy.
p, found := sys.bucketPolicyMap[args.BucketName]
if found {
return p.IsAllowed(args)
}
configData, err := globalBucketMetadataSys.GetConfig(args.BucketName, bucketPolicyConfig)
if err != nil {
if !errors.Is(err, errConfigNotFound) {
logger.LogIf(GlobalContext, err)
}
} else {
sys.RLock()
defer sys.RUnlock()
// If policy is available for given bucket, check the policy.
if p, found := sys.bucketPolicyMap[args.BucketName]; found {
p, err = policy.ParseConfig(bytes.NewReader(configData), args.BucketName)
if err != nil {
logger.LogIf(GlobalContext, err)
} else {
return p.IsAllowed(args)
}
}
@@ -97,28 +100,19 @@ func (sys *PolicySys) IsAllowed(args policy.Args) bool {
// Loads policies for all buckets into PolicySys.
func (sys *PolicySys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
for _, bucket := range buckets {
config, err := getPolicyConfig(objAPI, bucket.Name)
configData, err := globalBucketMetadataSys.GetConfig(bucket.Name, bucketPolicyConfig)
if err != nil {
if _, ok := err.(BucketPolicyNotFound); ok {
if errors.Is(err, errConfigNotFound) {
continue
}
return err
}
// This part is specifically written to handle migration
// when the Version string is empty, this was allowed
// in all previous minio releases but we need to migrate
// those policies by properly setting the Version string
// from now on.
if config.Version == "" {
logger.Info("Found in-consistent bucket policies, Migrating them for Bucket: (%s)", bucket.Name)
config.Version = policy.DefaultVersion
if err = savePolicyConfig(GlobalContext, objAPI, bucket.Name, config); err != nil {
logger.LogIf(GlobalContext, err)
return err
}
config, err := policy.ParseConfig(bytes.NewReader(configData), bucket.Name)
if err != nil {
logger.LogIf(GlobalContext, err)
continue
}
sys.Set(bucket.Name, config)
sys.bucketPolicyMap[bucket.Name] = config
}
return nil
}
@@ -226,50 +220,6 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
return args
}
// getPolicyConfig - get policy config for given bucket name.
func getPolicyConfig(objAPI ObjectLayer, bucketName string) (*policy.Policy, error) {
// Construct path to policy.json for the given bucket.
configFile := path.Join(bucketConfigPrefix, bucketName, bucketPolicyConfig)
configData, err := readConfig(GlobalContext, objAPI, configFile)
if err != nil {
if err == errConfigNotFound {
err = BucketPolicyNotFound{Bucket: bucketName}
}
return nil, err
}
return policy.ParseConfig(bytes.NewReader(configData), bucketName)
}
func savePolicyConfig(ctx context.Context, objAPI ObjectLayer, bucketName string, bucketPolicy *policy.Policy) error {
data, err := json.Marshal(bucketPolicy)
if err != nil {
return err
}
// Construct path to policy.json for the given bucket.
configFile := path.Join(bucketConfigPrefix, bucketName, bucketPolicyConfig)
return saveConfig(ctx, objAPI, configFile, data)
}
func removePolicyConfig(ctx context.Context, objAPI ObjectLayer, bucketName string) error {
// Construct path to policy.json for the given bucket.
configFile := path.Join(bucketConfigPrefix, bucketName, bucketPolicyConfig)
if err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile); err != nil {
if _, ok := err.(ObjectNotFound); ok {
return BucketPolicyNotFound{Bucket: bucketName}
}
return err
}
return nil
}
// PolicyToBucketAccessPolicy - converts policy.Policy to minio-go/policy.BucketAccessPolicy.
func PolicyToBucketAccessPolicy(bucketPolicy *policy.Policy) (*miniogopolicy.BucketAccessPolicy, error) {
// Return empty BucketAccessPolicy for empty bucket policy.