MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-09 05:34:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

migrate all bucket metadata into a single file (#9586)

Browse Source 
this is a major overhaul by migrating off all
bucket metadata related configs into a single
object '.metadata.bin' this allows us for faster
bootups across 1000's of buckets and as well
as keeps the code simple enough for future
work and additions.

Additionally also fixes #9396, #9394
This commit is contained in:
Harshavardhana
2020-05-19 13:53:54 -07:00
committed by GitHub
parent 730775bb4e
commit bd032d13ff
48 changed files with 1648 additions and 2845 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
cmd/bucket-lifecycle.go
View File

@@ -18,10 +18,7 @@ package cmd
import (
"bytes"
"context"
"encoding/xml"
"path"
"sync"
"errors"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/bucket/lifecycle"
@@ -35,83 +32,32 @@ const (
// LifecycleSys - Bucket lifecycle subsystem.
type LifecycleSys struct {
sync.RWMutex
bucketLifecycleMap map[string]*lifecycle.Lifecycle
}
// Set - sets lifecycle config to given bucket name.
func (sys *LifecycleSys) Set(bucketName string, lifecycle *lifecycle.Lifecycle) {
if globalIsGateway {
// no-op
return
}
sys.Lock()
defer sys.Unlock()
sys.bucketLifecycleMap[bucketName] = lifecycle
}
// Get - gets lifecycle config associated to a given bucket name.
func (sys *LifecycleSys) Get(bucketName string) (lc *lifecycle.Lifecycle, ok bool) {
func (sys *LifecycleSys) Get(bucketName string) (lc *lifecycle.Lifecycle, err error) {
if globalIsGateway {
// When gateway is enabled, no cached value
// is used to validate life cycle policies.
objAPI := newObjectLayerWithoutSafeModeFn()
if objAPI == nil {
return
return nil, errServerNotInitialized
}
l, err := objAPI.GetBucketLifecycle(GlobalContext, bucketName)
return nil, BucketLifecycleNotFound{Bucket: bucketName}
}
lc, ok := sys.bucketLifecycleMap[bucketName]
if !ok {
configData, err := globalBucketMetadataSys.GetConfig(bucketName, bucketLifecycleConfig)
if err != nil {
return
if errors.Is(err, errConfigNotFound) {
return nil, BucketLifecycleNotFound{Bucket: bucketName}
}
return nil, err
}
return l, true
return lifecycle.ParseLifecycleConfig(bytes.NewReader(configData))
}
sys.Lock()
defer sys.Unlock()
lc, ok = sys.bucketLifecycleMap[bucketName]
return
}
func saveLifecycleConfig(ctx context.Context, objAPI ObjectLayer, bucketName string, bucketLifecycle *lifecycle.Lifecycle) error {
data, err := xml.Marshal(bucketLifecycle)
if err != nil {
return err
}
// Construct path to lifecycle.xml for the given bucket.
configFile := path.Join(bucketConfigPrefix, bucketName, bucketLifecycleConfig)
return saveConfig(ctx, objAPI, configFile, data)
}
// getLifecycleConfig - get lifecycle config for given bucket name.
func getLifecycleConfig(objAPI ObjectLayer, bucketName string) (*lifecycle.Lifecycle, error) {
// Construct path to lifecycle.xml for the given bucket.
configFile := path.Join(bucketConfigPrefix, bucketName, bucketLifecycleConfig)
configData, err := readConfig(GlobalContext, objAPI, configFile)
if err != nil {
if err == errConfigNotFound {
err = BucketLifecycleNotFound{Bucket: bucketName}
}
return nil, err
}
return lifecycle.ParseLifecycleConfig(bytes.NewReader(configData))
}
func removeLifecycleConfig(ctx context.Context, objAPI ObjectLayer, bucketName string) error {
// Construct path to lifecycle.xml for the given bucket.
configFile := path.Join(bucketConfigPrefix, bucketName, bucketLifecycleConfig)
if err := deleteConfig(ctx, objAPI, configFile); err != nil {
if err == errConfigNotFound {
return BucketLifecycleNotFound{Bucket: bucketName}
}
return err
}
return nil
return lc, nil
}
// NewLifecycleSys - creates new lifecycle system.
@@ -140,31 +86,21 @@ func (sys *LifecycleSys) Init(buckets []BucketInfo, objAPI ObjectLayer) error {
// Loads lifecycle policies for all buckets into LifecycleSys.
func (sys *LifecycleSys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
for _, bucket := range buckets {
config, err := getLifecycleConfig(objAPI, bucket.Name)
configData, err := globalBucketMetadataSys.GetConfig(bucket.Name, bucketLifecycleConfig)
if err != nil {
if _, ok := err.(BucketLifecycleNotFound); ok {
if errors.Is(err, errConfigNotFound) {
continue
}
// Quorum errors should be returned.
return err
}
// Do not load the lifecycle configuration if it is not valid
err = config.Validate()
config, err := lifecycle.ParseLifecycleConfig(bytes.NewReader(configData))
if err != nil {
logger.LogIf(context.Background(), err)
logger.LogIf(GlobalContext, err)
continue
}
sys.Set(bucket.Name, config)
sys.bucketLifecycleMap[bucket.Name] = config
}
return nil
}
// Remove - removes lifecycle config for given bucket name.
func (sys *LifecycleSys) Remove(bucketName string) {
sys.Lock()
defer sys.Unlock()
delete(sys.bucketLifecycleMap, bucketName)
}