simplify further bucket configuration properly (#9650)

This PR is a continuation from #9586, now the
entire parsing logic is fully merged into
bucket metadata sub-system, simplify the
quota API further by reducing the remove
quota handler implementation.
This commit is contained in:
Harshavardhana 2020-05-20 10:18:15 -07:00 committed by GitHub
parent 0cc2ed04f5
commit 6656fa3066
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 372 additions and 594 deletions

View File

@ -68,21 +68,9 @@ func prepareAdminXLTestBed(ctx context.Context) (*adminXLTestBed, error) {
globalEndpoints = mustGetZoneEndpoints(xlDirs...) globalEndpoints = mustGetZoneEndpoints(xlDirs...)
globalConfigSys = NewConfigSys() newAllSubsystems()
globalIAMSys = NewIAMSys() initAllSubsystems(objLayer)
globalIAMSys.Init(ctx, objLayer)
buckets, err := objLayer.ListBuckets(ctx)
if err != nil {
return nil, err
}
globalPolicySys = NewPolicySys()
globalPolicySys.Init(buckets, objLayer)
globalNotificationSys = NewNotificationSys(globalEndpoints)
globalNotificationSys.Init(buckets, objLayer)
// Setup admin mgmt REST API handlers. // Setup admin mgmt REST API handlers.
adminRouter := mux.NewRouter() adminRouter := mux.NewRouter()

View File

@ -17,7 +17,7 @@
package cmd package cmd
import ( import (
"errors" "encoding/json"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
@ -101,46 +101,18 @@ func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *
return return
} }
configData, err := globalBucketMetadataSys.GetConfig(bucket, bucketQuotaConfigFile) config, err := globalBucketMetadataSys.GetQuotaConfig(bucket)
if err != nil { if err != nil {
if errors.Is(err, errConfigNotFound) { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, BucketQuotaConfigNotFound{Bucket: bucket}), r.URL) return
} else { }
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
} configData, err := json.Marshal(config)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return return
} }
// Write success response. // Write success response.
writeSuccessResponseJSON(w, configData) writeSuccessResponseJSON(w, configData)
} }
// RemoveBucketQuotaConfigHandler - removes Bucket quota configuration.
// ----------
// Removes quota configuration on the specified bucket.
func (a adminAPIHandlers) RemoveBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "RemoveBucketQuotaConfig")
defer logger.AuditLog(w, r, "RemoveBucketQuotaConfig", mustGetClaimsFromToken(r))
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketQuotaAdminAction)
if objectAPI == nil {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
return
}
vars := mux.Vars(r)
bucket := vars["bucket"]
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if err := globalBucketMetadataSys.Update(bucket, bucketQuotaConfigFile, nil); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
// Write success response.
writeSuccessNoContent(w)
}

View File

@ -177,9 +177,6 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps, enab
// PutBucketQuotaConfig // PutBucketQuotaConfig
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc( adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc(
httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}") httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
// RemoveBucketQuotaConfig
adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-bucket-quota").HandlerFunc(
httpTraceHdrs(adminAPI.RemoveBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
} }
// -- Top APIs -- // -- Top APIs --

View File

@ -18,7 +18,6 @@ package cmd
import ( import (
"encoding/xml" "encoding/xml"
"errors"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
@ -128,11 +127,14 @@ func (api objectAPIHandlers) GetBucketEncryptionHandler(w http.ResponseWriter, r
return return
} }
configData, err := globalBucketMetadataSys.GetConfig(bucket, bucketSSEConfig) config, err := globalBucketMetadataSys.GetSSEConfig(bucket)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
configData, err := xml.Marshal(config)
if err != nil { if err != nil {
if errors.Is(err, errConfigNotFound) {
err = BucketSSEConfigNotFound{Bucket: bucket}
}
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return return
} }

View File

@ -17,7 +17,6 @@
package cmd package cmd
import ( import (
"bytes"
"errors" "errors"
"io" "io"
@ -25,56 +24,15 @@ import (
) )
// BucketSSEConfigSys - in-memory cache of bucket encryption config // BucketSSEConfigSys - in-memory cache of bucket encryption config
type BucketSSEConfigSys struct { type BucketSSEConfigSys struct{}
bucketSSEConfigMap map[string]*bucketsse.BucketSSEConfig
}
// NewBucketSSEConfigSys - Creates an empty in-memory bucket encryption configuration cache // NewBucketSSEConfigSys - Creates an empty in-memory bucket encryption configuration cache
func NewBucketSSEConfigSys() *BucketSSEConfigSys { func NewBucketSSEConfigSys() *BucketSSEConfigSys {
return &BucketSSEConfigSys{ return &BucketSSEConfigSys{}
bucketSSEConfigMap: make(map[string]*bucketsse.BucketSSEConfig),
}
}
// load - Loads the bucket encryption configuration for the given list of buckets
func (sys *BucketSSEConfigSys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
for _, bucket := range buckets {
configData, err := globalBucketMetadataSys.GetConfig(bucket.Name, bucketSSEConfig)
if err != nil {
if errors.Is(err, errConfigNotFound) {
continue
}
return err
}
config, err := bucketsse.ParseBucketSSEConfig(bytes.NewReader(configData))
if err != nil {
return err
}
sys.bucketSSEConfigMap[bucket.Name] = config
}
return nil
}
// Init - Initializes in-memory bucket encryption config cache for the given list of buckets
func (sys *BucketSSEConfigSys) Init(buckets []BucketInfo, objAPI ObjectLayer) error {
if objAPI == nil {
return errServerNotInitialized
}
// We don't cache bucket encryption config in gateway mode, nothing to do.
if globalIsGateway {
return nil
}
// Load bucket encryption config cache once during boot.
return sys.load(buckets, objAPI)
} }
// Get - gets bucket encryption config for the given bucket. // Get - gets bucket encryption config for the given bucket.
func (sys *BucketSSEConfigSys) Get(bucket string) (config *bucketsse.BucketSSEConfig, err error) { func (sys *BucketSSEConfigSys) Get(bucket string) (*bucketsse.BucketSSEConfig, error) {
if globalIsGateway { if globalIsGateway {
objAPI := newObjectLayerWithoutSafeModeFn() objAPI := newObjectLayerWithoutSafeModeFn()
if objAPI == nil { if objAPI == nil {
@ -84,18 +42,7 @@ func (sys *BucketSSEConfigSys) Get(bucket string) (config *bucketsse.BucketSSECo
return nil, BucketSSEConfigNotFound{Bucket: bucket} return nil, BucketSSEConfigNotFound{Bucket: bucket}
} }
config, ok := sys.bucketSSEConfigMap[bucket] return globalBucketMetadataSys.GetSSEConfig(bucket)
if !ok {
configData, err := globalBucketMetadataSys.GetConfig(bucket, bucketSSEConfig)
if err != nil {
if errors.Is(err, errConfigNotFound) {
return nil, BucketSSEConfigNotFound{Bucket: bucket}
}
return nil, err
}
return bucketsse.ParseBucketSSEConfig(bytes.NewReader(configData))
}
return config, nil
} }
// validateBucketSSEConfig parses bucket encryption configuration and validates if it is supported by MinIO. // validateBucketSSEConfig parses bucket encryption configuration and validates if it is supported by MinIO.

View File

@ -19,7 +19,6 @@ package cmd
import ( import (
"encoding/base64" "encoding/base64"
"encoding/xml" "encoding/xml"
"errors"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
@ -1069,11 +1068,14 @@ func (api objectAPIHandlers) GetBucketObjectLockConfigHandler(w http.ResponseWri
return return
} }
configData, err := globalBucketMetadataSys.GetConfig(bucket, objectLockConfig) config, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
configData, err := xml.Marshal(config)
if err != nil { if err != nil {
if errors.Is(err, errConfigNotFound) {
err = BucketObjectLockConfigNotFound{Bucket: bucket}
}
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return return
} }
@ -1148,11 +1150,14 @@ func (api objectAPIHandlers) GetBucketTaggingHandler(w http.ResponseWriter, r *h
return return
} }
configData, err := globalBucketMetadataSys.GetConfig(bucket, bucketTaggingConfigFile) config, err := globalBucketMetadataSys.GetTaggingConfig(bucket)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
configData, err := xml.Marshal(config)
if err != nil { if err != nil {
if errors.Is(err, errConfigNotFound) {
err = BucketTaggingNotFound{Bucket: bucket}
}
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return return
} }

View File

@ -18,7 +18,6 @@ package cmd
import ( import (
"encoding/xml" "encoding/xml"
"errors"
"io" "io"
"net/http" "net/http"
@ -120,11 +119,14 @@ func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r
return return
} }
configData, err := globalBucketMetadataSys.GetConfig(bucket, bucketLifecycleConfig) config, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
configData, err := xml.Marshal(config)
if err != nil { if err != nil {
if errors.Is(err, errConfigNotFound) {
err = BucketLifecycleNotFound{Bucket: bucket}
}
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return return
} }

View File

@ -17,10 +17,6 @@
package cmd package cmd
import ( import (
"bytes"
"errors"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/bucket/lifecycle" "github.com/minio/minio/pkg/bucket/lifecycle"
) )
@ -31,9 +27,7 @@ const (
) )
// LifecycleSys - Bucket lifecycle subsystem. // LifecycleSys - Bucket lifecycle subsystem.
type LifecycleSys struct { type LifecycleSys struct{}
bucketLifecycleMap map[string]*lifecycle.Lifecycle
}
// Get - gets lifecycle config associated to a given bucket name. // Get - gets lifecycle config associated to a given bucket name.
func (sys *LifecycleSys) Get(bucketName string) (lc *lifecycle.Lifecycle, err error) { func (sys *LifecycleSys) Get(bucketName string) (lc *lifecycle.Lifecycle, err error) {
@ -46,61 +40,10 @@ func (sys *LifecycleSys) Get(bucketName string) (lc *lifecycle.Lifecycle, err er
return nil, BucketLifecycleNotFound{Bucket: bucketName} return nil, BucketLifecycleNotFound{Bucket: bucketName}
} }
lc, ok := sys.bucketLifecycleMap[bucketName] return globalBucketMetadataSys.GetLifecycleConfig(bucketName)
if !ok {
configData, err := globalBucketMetadataSys.GetConfig(bucketName, bucketLifecycleConfig)
if err != nil {
if errors.Is(err, errConfigNotFound) {
return nil, BucketLifecycleNotFound{Bucket: bucketName}
}
return nil, err
}
return lifecycle.ParseLifecycleConfig(bytes.NewReader(configData))
}
return lc, nil
} }
// NewLifecycleSys - creates new lifecycle system. // NewLifecycleSys - creates new lifecycle system.
func NewLifecycleSys() *LifecycleSys { func NewLifecycleSys() *LifecycleSys {
return &LifecycleSys{ return &LifecycleSys{}
bucketLifecycleMap: make(map[string]*lifecycle.Lifecycle),
}
}
// Init - initializes lifecycle system from lifecycle.xml of all buckets.
func (sys *LifecycleSys) Init(buckets []BucketInfo, objAPI ObjectLayer) error {
if objAPI == nil {
return errServerNotInitialized
}
// In gateway mode, we always fetch the bucket lifecycle configuration from the gateway backend.
// So, this is a no-op for gateway servers.
if globalIsGateway {
return nil
}
// Load LifecycleSys once during boot.
return sys.load(buckets, objAPI)
}
// Loads lifecycle policies for all buckets into LifecycleSys.
func (sys *LifecycleSys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
for _, bucket := range buckets {
configData, err := globalBucketMetadataSys.GetConfig(bucket.Name, bucketLifecycleConfig)
if err != nil {
if errors.Is(err, errConfigNotFound) {
continue
}
return err
}
config, err := lifecycle.ParseLifecycleConfig(bytes.NewReader(configData))
if err != nil {
logger.LogIf(GlobalContext, err)
continue
}
sys.bucketLifecycleMap[bucket.Name] = config
}
return nil
} }

View File

@ -19,19 +19,23 @@ package cmd
import ( import (
"bytes" "bytes"
"context" "context"
"errors"
"fmt" "fmt"
"sync" "sync"
"github.com/minio/minio-go/v6/pkg/tags"
bucketsse "github.com/minio/minio/pkg/bucket/encryption"
"github.com/minio/minio/pkg/bucket/lifecycle"
objectlock "github.com/minio/minio/pkg/bucket/object/lock"
"github.com/minio/minio/pkg/bucket/policy" "github.com/minio/minio/pkg/bucket/policy"
"github.com/minio/minio/pkg/event"
"github.com/minio/minio/pkg/madmin"
) )
// BucketMetadataSys captures all bucket metadata for a given cluster. // BucketMetadataSys captures all bucket metadata for a given cluster.
type BucketMetadataSys struct { type BucketMetadataSys struct {
sync.RWMutex sync.RWMutex
metadataMap map[string]BucketMetadata metadataMap map[string]BucketMetadata
// Do fallback to old config when unable to find a bucket config
fallback bool
} }
// Remove bucket metadata from memory. // Remove bucket metadata from memory.
@ -84,13 +88,9 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
return errInvalidArgument return errInvalidArgument
} }
defer globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket) meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
if err != nil {
sys.Lock() return err
meta, ok := sys.metadataMap[bucket]
sys.Unlock()
if !ok {
return BucketNotFound{Bucket: bucket}
} }
switch configFile { switch configFile {
@ -116,9 +116,8 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
return err return err
} }
sys.Lock() sys.Set(bucket, meta)
sys.metadataMap[bucket] = meta globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
sys.Unlock()
return nil return nil
} }
@ -143,62 +142,134 @@ func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
return meta, nil return meta, nil
} }
// GetTaggingConfig returns configured tagging config
// The returned object may not be modified.
func (sys *BucketMetadataSys) GetTaggingConfig(bucket string) (*tags.Tags, error) {
meta, err := sys.GetConfig(bucket)
if err != nil {
if errors.Is(err, errConfigNotFound) {
return nil, BucketTaggingNotFound{Bucket: bucket}
}
return nil, err
}
if meta.taggingConfig == nil {
return nil, BucketTaggingNotFound{Bucket: bucket}
}
return meta.taggingConfig, nil
}
// GetObjectLockConfig returns configured object lock config
// The returned object may not be modified.
func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Config, error) {
meta, err := sys.GetConfig(bucket)
if err != nil {
if errors.Is(err, errConfigNotFound) {
return nil, BucketObjectLockConfigNotFound{Bucket: bucket}
}
return nil, err
}
if meta.objectLockConfig == nil {
return nil, BucketObjectLockConfigNotFound{Bucket: bucket}
}
return meta.objectLockConfig, nil
}
// GetLifecycleConfig returns configured lifecycle config
// The returned object may not be modified.
func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Lifecycle, error) {
meta, err := sys.GetConfig(bucket)
if err != nil {
if errors.Is(err, errConfigNotFound) {
return nil, BucketLifecycleNotFound{Bucket: bucket}
}
return nil, err
}
if meta.lifecycleConfig == nil {
return nil, BucketLifecycleNotFound{Bucket: bucket}
}
return meta.lifecycleConfig, nil
}
// GetNotificationConfig returns configured notification config
// The returned object may not be modified.
func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Config, error) {
meta, err := sys.GetConfig(bucket)
if err != nil {
return nil, err
}
return meta.notificationConfig, nil
}
// GetSSEConfig returns configured SSE config
// The returned object may not be modified.
func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEConfig, error) {
meta, err := sys.GetConfig(bucket)
if err != nil {
if errors.Is(err, errConfigNotFound) {
return nil, BucketSSEConfigNotFound{Bucket: bucket}
}
return nil, err
}
if meta.sseConfig == nil {
return nil, BucketSSEConfigNotFound{Bucket: bucket}
}
return meta.sseConfig, nil
}
// GetPolicyConfig returns configured bucket policy
// The returned object may not be modified.
func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, error) {
meta, err := sys.GetConfig(bucket)
if err != nil {
if errors.Is(err, errConfigNotFound) {
return nil, BucketPolicyNotFound{Bucket: bucket}
}
return nil, err
}
if meta.policyConfig == nil {
return nil, BucketPolicyNotFound{Bucket: bucket}
}
return meta.policyConfig, nil
}
// GetQuotaConfig returns configured bucket quota
// The returned object may not be modified.
func (sys *BucketMetadataSys) GetQuotaConfig(bucket string) (*madmin.BucketQuota, error) {
meta, err := sys.GetConfig(bucket)
if err != nil {
return nil, err
}
return meta.quotaConfig, nil
}
// GetConfig returns a specific configuration from the bucket metadata. // GetConfig returns a specific configuration from the bucket metadata.
// The returned data should be copied before being modified. // The returned object may not be modified.
func (sys *BucketMetadataSys) GetConfig(bucket string, configFile string) ([]byte, error) { func (sys *BucketMetadataSys) GetConfig(bucket string) (BucketMetadata, error) {
objAPI := newObjectLayerWithoutSafeModeFn()
if objAPI == nil {
return newBucketMetadata(bucket), errServerNotInitialized
}
if globalIsGateway { if globalIsGateway {
return nil, NotImplemented{} return newBucketMetadata(bucket), NotImplemented{}
} }
if bucket == minioMetaBucket { if bucket == minioMetaBucket {
return nil, errInvalidArgument return newBucketMetadata(bucket), errInvalidArgument
} }
sys.RLock() sys.Lock()
defer sys.RUnlock() defer sys.Unlock()
meta, ok := sys.metadataMap[bucket] meta, ok := sys.metadataMap[bucket]
if !ok { if ok {
if !sys.fallback { return meta, nil
return nil, errConfigNotFound
}
objAPI := newObjectLayerWithoutSafeModeFn()
if objAPI == nil {
return nil, errServerNotInitialized
}
var err error
meta, err = loadBucketMetadata(GlobalContext, objAPI, bucket)
if err != nil {
return nil, err
}
sys.metadataMap[bucket] = meta
} }
meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
var configData []byte if err != nil {
switch configFile { return meta, err
case bucketPolicyConfig:
configData = meta.PolicyConfigJSON
case bucketNotificationConfig:
configData = meta.NotificationXML
case bucketLifecycleConfig:
configData = meta.LifecycleConfigXML
case bucketSSEConfig:
configData = meta.EncryptionConfigXML
case bucketTaggingConfigFile:
configData = meta.TaggingConfigXML
case objectLockConfig:
configData = meta.ObjectLockConfigurationXML
case bucketQuotaConfigFile:
configData = meta.QuotaConfigJSON
default:
return nil, fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
} }
sys.metadataMap[bucket] = meta
if len(configData) == 0 { return meta, nil
return nil, errConfigNotFound
}
return configData, nil
} }
// Init - initializes bucket metadata system for all buckets. // Init - initializes bucket metadata system for all buckets.
@ -218,7 +289,6 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, ob
} }
// Loads bucket metadata for all buckets into BucketMetadataSys. // Loads bucket metadata for all buckets into BucketMetadataSys.
// When if done successfully fallback will be disabled.
func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error { func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
sys.Lock() sys.Lock()
defer sys.Unlock() defer sys.Unlock()
@ -230,7 +300,6 @@ func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, ob
} }
sys.metadataMap[bucket.Name] = meta sys.metadataMap[bucket.Name] = meta
} }
sys.fallback = false
return nil return nil
} }
@ -238,8 +307,5 @@ func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, ob
func NewBucketMetadataSys() *BucketMetadataSys { func NewBucketMetadataSys() *BucketMetadataSys {
return &BucketMetadataSys{ return &BucketMetadataSys{
metadataMap: make(map[string]BucketMetadata), metadataMap: make(map[string]BucketMetadata),
// Do fallback until all buckets have been loaded.
fallback: true,
} }
} }

View File

@ -17,14 +17,23 @@
package cmd package cmd
import ( import (
"bytes"
"context" "context"
"encoding/binary" "encoding/binary"
"encoding/xml"
"errors" "errors"
"fmt" "fmt"
"path" "path"
"time" "time"
"github.com/minio/minio-go/v6/pkg/tags"
"github.com/minio/minio/cmd/logger" "github.com/minio/minio/cmd/logger"
bucketsse "github.com/minio/minio/pkg/bucket/encryption"
"github.com/minio/minio/pkg/bucket/lifecycle"
objectlock "github.com/minio/minio/pkg/bucket/object/lock"
"github.com/minio/minio/pkg/bucket/policy"
"github.com/minio/minio/pkg/event"
"github.com/minio/minio/pkg/madmin"
) )
const ( const (
@ -58,6 +67,15 @@ type BucketMetadata struct {
EncryptionConfigXML []byte EncryptionConfigXML []byte
TaggingConfigXML []byte TaggingConfigXML []byte
QuotaConfigJSON []byte QuotaConfigJSON []byte
// Unexported fields. Must be updated atomically.
policyConfig *policy.Policy
notificationConfig *event.Config
lifecycleConfig *lifecycle.Lifecycle
objectLockConfig *objectlock.Config
sseConfig *bucketsse.BucketSSEConfig
taggingConfig *tags.Tags
quotaConfig *madmin.BucketQuota
} }
// newBucketMetadata creates BucketMetadata with the supplied name and Created to Now. // newBucketMetadata creates BucketMetadata with the supplied name and Created to Now.
@ -112,6 +130,76 @@ func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket strin
return b, b.convertLegacyConfigs(ctx, objectAPI) return b, b.convertLegacyConfigs(ctx, objectAPI)
} }
// parseAllConfigs will parse all configs and populate the private fields.
// The first error encountered is returned.
func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLayer) (err error) {
if len(b.PolicyConfigJSON) != 0 {
b.policyConfig, err = policy.ParseConfig(bytes.NewReader(b.PolicyConfigJSON), b.Name)
if err != nil {
return err
}
} else {
b.policyConfig = nil
}
if len(b.NotificationXML) != 0 {
if err = xml.Unmarshal(b.NotificationXML, b.notificationConfig); err != nil {
return err
}
} else {
b.notificationConfig = &event.Config{
XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
}
b.notificationConfig.SetRegion(globalServerRegion)
}
if len(b.LifecycleConfigXML) != 0 {
b.lifecycleConfig, err = lifecycle.ParseLifecycleConfig(bytes.NewReader(b.LifecycleConfigXML))
if err != nil {
return err
}
} else {
b.lifecycleConfig = nil
}
if len(b.EncryptionConfigXML) != 0 {
b.sseConfig, err = bucketsse.ParseBucketSSEConfig(bytes.NewReader(b.EncryptionConfigXML))
if err != nil {
return err
}
} else {
b.sseConfig = nil
}
if len(b.TaggingConfigXML) != 0 {
b.taggingConfig, err = tags.ParseBucketXML(bytes.NewReader(b.TaggingConfigXML))
if err != nil {
return err
}
} else {
b.taggingConfig = nil
}
if len(b.ObjectLockConfigurationXML) != 0 {
b.objectLockConfig, err = objectlock.ParseObjectLockConfig(bytes.NewReader(b.ObjectLockConfigurationXML))
if err != nil {
return err
}
} else {
b.objectLockConfig = nil
}
if len(b.QuotaConfigJSON) != 0 {
qcfg, err := parseBucketQuota(b.Name, b.QuotaConfigJSON)
if err != nil {
return err
}
b.quotaConfig = qcfg
} else {
b.quotaConfig = &madmin.BucketQuota{}
}
return nil
}
func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer) error { func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer) error {
legacyConfigs := []string{ legacyConfigs := []string{
legacyBucketObjectLockEnabledConfigFile, legacyBucketObjectLockEnabledConfigFile,
@ -150,7 +238,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
if len(configs) == 0 { if len(configs) == 0 {
// nothing to update, return right away. // nothing to update, return right away.
return nil return b.parseAllConfigs(ctx, objectAPI)
} }
for legacyFile, configData := range configs { for legacyFile, configData := range configs {
@ -194,6 +282,10 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
// Save config to supplied ObjectLayer api. // Save config to supplied ObjectLayer api.
func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error { func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
if err := b.parseAllConfigs(ctx, api); err != nil {
return err
}
data := make([]byte, 4, b.Msgsize()+4) data := make([]byte, 4, b.Msgsize()+4)
// Initialize the header. // Initialize the header.

View File

@ -69,56 +69,43 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
return return
} }
var config = event.Config{ config, err := globalBucketMetadataSys.GetNotificationConfig(bucketName)
XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
}
config.SetRegion(globalServerRegion)
configData, err := globalBucketMetadataSys.GetConfig(bucketName, bucketNotificationConfig)
if err != nil { if err != nil {
if err != errConfigNotFound { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) return
return }
} if err = config.Validate(globalServerRegion, globalNotificationSys.targetList); err != nil {
} else { arnErr, ok := err.(*event.ErrARNNotFound)
if err = xml.Unmarshal(configData, &config); err != nil { if ok {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) for i, queue := range config.QueueList {
return // Remove ARN not found queues, because we previously allowed
} // adding unexpected entries into the config.
//
if err = config.Validate(globalServerRegion, globalNotificationSys.targetList); err != nil { // With newer config disallowing changing / turning off
arnErr, ok := err.(*event.ErrARNNotFound) // notification targets without removing ARN in notification
if ok { // configuration we won't see this problem anymore.
for i, queue := range config.QueueList { if reflect.DeepEqual(queue.ARN, arnErr.ARN) && i < len(config.QueueList) {
// Remove ARN not found queues, because we previously allowed config.QueueList = append(config.QueueList[:i],
// adding unexpected entries into the config. config.QueueList[i+1:]...)
//
// With newer config disallowing changing / turning off
// notification targets without removing ARN in notification
// configuration we won't see this problem anymore.
if reflect.DeepEqual(queue.ARN, arnErr.ARN) && i < len(config.QueueList) {
config.QueueList = append(config.QueueList[:i],
config.QueueList[i+1:]...)
}
// This is a one time activity we shall do this
// here and allow stale ARN to be removed. We shall
// never reach a stage where we will have stale
// notification configs.
} }
} else { // This is a one time activity we shall do this
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) // here and allow stale ARN to be removed. We shall
return // never reach a stage where we will have stale
// notification configs.
} }
} else {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
} }
} }
notificationBytes, err := xml.Marshal(config) configData, err := xml.Marshal(config)
if err != nil { if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return return
} }
writeSuccessResponseXML(w, notificationBytes) writeSuccessResponseXML(w, configData)
} }
// PutBucketNotificationHandler - This HTTP handler stores given notification configuration as per // PutBucketNotificationHandler - This HTTP handler stores given notification configuration as per

View File

@ -17,9 +17,7 @@
package cmd package cmd
import ( import (
"bytes"
"context" "context"
"errors"
"math" "math"
"net/http" "net/http"
@ -30,9 +28,7 @@ import (
) )
// BucketObjectLockSys - map of bucket and retention configuration. // BucketObjectLockSys - map of bucket and retention configuration.
type BucketObjectLockSys struct { type BucketObjectLockSys struct{}
retentionMap map[string]objectlock.Retention
}
// Get - Get retention configuration. // Get - Get retention configuration.
func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention, err error) { func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention, err error) {
@ -45,20 +41,13 @@ func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention,
return r, nil return r, nil
} }
r, ok := sys.retentionMap[bucketName] config, err := globalBucketMetadataSys.GetObjectLockConfig(bucketName)
if ok {
return r, nil
}
configData, err := globalBucketMetadataSys.GetConfig(bucketName, objectLockConfig)
if err != nil { if err != nil {
if errors.Is(err, errConfigNotFound) { if _, ok := err.(BucketObjectLockConfigNotFound); ok {
return r, nil return r, nil
} }
return r, err return r, err
}
config, err := objectlock.ParseObjectLockConfig(bytes.NewReader(configData))
if err != nil {
return r, err
} }
return config.ToRetention(), nil return config.ToRetention(), nil
} }
@ -418,46 +407,5 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
// NewBucketObjectLockSys returns initialized BucketObjectLockSys // NewBucketObjectLockSys returns initialized BucketObjectLockSys
func NewBucketObjectLockSys() *BucketObjectLockSys { func NewBucketObjectLockSys() *BucketObjectLockSys {
return &BucketObjectLockSys{ return &BucketObjectLockSys{}
retentionMap: make(map[string]objectlock.Retention),
}
}
// Init - initializes bucket object lock config system for all buckets
func (sys *BucketObjectLockSys) Init(buckets []BucketInfo, objAPI ObjectLayer) error {
if objAPI == nil {
return errServerNotInitialized
}
// In gateway mode, we always fetch the bucket object lock configuration from the gateway backend.
// So, this is a no-op for gateway servers.
if globalIsGateway {
return nil
}
// Load BucketObjectLockSys once during boot.
return sys.load(buckets, objAPI)
}
func (sys *BucketObjectLockSys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
for _, bucket := range buckets {
ctx := logger.SetReqInfo(GlobalContext, &logger.ReqInfo{BucketName: bucket.Name})
configData, err := globalBucketMetadataSys.GetConfig(bucket.Name, objectLockConfig)
if err != nil {
if errors.Is(err, errConfigNotFound) {
continue
}
return err
}
retention := objectlock.Retention{}
config, err := objectlock.ParseObjectLockConfig(bytes.NewReader(configData))
if err != nil {
logger.LogIf(ctx, err)
sys.retentionMap[bucket.Name] = retention
continue
}
sys.retentionMap[bucket.Name] = config.ToRetention()
}
return nil
} }

View File

@ -18,7 +18,6 @@ package cmd
import ( import (
"encoding/json" "encoding/json"
"errors"
"io" "io"
"net/http" "net/http"
@ -165,11 +164,14 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
} }
// Read bucket access policy. // Read bucket access policy.
configData, err := globalBucketMetadataSys.GetConfig(bucket, bucketPolicyConfig) config, err := globalBucketMetadataSys.GetPolicyConfig(bucket)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
configData, err := json.Marshal(config)
if err != nil { if err != nil {
if errors.Is(err, errConfigNotFound) {
err = BucketPolicyNotFound{Bucket: bucket}
}
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return return
} }

View File

@ -17,9 +17,7 @@
package cmd package cmd
import ( import (
"bytes"
"encoding/json" "encoding/json"
"errors"
"net/http" "net/http"
"net/url" "net/url"
"strconv" "strconv"
@ -35,9 +33,7 @@ import (
) )
// PolicySys - policy subsystem. // PolicySys - policy subsystem.
type PolicySys struct { type PolicySys struct{}
bucketPolicyMap map[string]*policy.Policy
}
// Get returns stored bucket policy // Get returns stored bucket policy
func (sys *PolicySys) Get(bucket string) (*policy.Policy, error) { func (sys *PolicySys) Get(bucket string) (*policy.Policy, error) {
@ -48,48 +44,19 @@ func (sys *PolicySys) Get(bucket string) (*policy.Policy, error) {
} }
return objAPI.GetBucketPolicy(GlobalContext, bucket) return objAPI.GetBucketPolicy(GlobalContext, bucket)
} }
configData, err := globalBucketMetadataSys.GetConfig(bucket, bucketPolicyConfig) return globalBucketMetadataSys.GetPolicyConfig(bucket)
if err != nil {
if !errors.Is(err, errConfigNotFound) {
return nil, err
}
return nil, BucketPolicyNotFound{Bucket: bucket}
}
return policy.ParseConfig(bytes.NewReader(configData), bucket)
} }
// IsAllowed - checks given policy args is allowed to continue the Rest API. // IsAllowed - checks given policy args is allowed to continue the Rest API.
func (sys *PolicySys) IsAllowed(args policy.Args) bool { func (sys *PolicySys) IsAllowed(args policy.Args) bool {
if globalIsGateway { p, err := sys.Get(args.BucketName)
objAPI := newObjectLayerFn() if err == nil {
if objAPI == nil {
return false
}
p, err := objAPI.GetBucketPolicy(GlobalContext, args.BucketName)
if err == nil {
return p.IsAllowed(args)
}
return args.IsOwner
}
// If policy is available for given bucket, check the policy.
p, found := sys.bucketPolicyMap[args.BucketName]
if found {
return p.IsAllowed(args) return p.IsAllowed(args)
} }
configData, err := globalBucketMetadataSys.GetConfig(args.BucketName, bucketPolicyConfig) // Log unhandled errors.
if err != nil { if _, ok := err.(BucketPolicyNotFound); !ok {
if !errors.Is(err, errConfigNotFound) { logger.LogIf(GlobalContext, err)
logger.LogIf(GlobalContext, err)
}
} else {
p, err = policy.ParseConfig(bytes.NewReader(configData), args.BucketName)
if err != nil {
logger.LogIf(GlobalContext, err)
} else {
return p.IsAllowed(args)
}
} }
// As policy is not available for given bucket name, returns IsOwner i.e. // As policy is not available for given bucket name, returns IsOwner i.e.
@ -97,47 +64,9 @@ func (sys *PolicySys) IsAllowed(args policy.Args) bool {
return args.IsOwner return args.IsOwner
} }
// Loads policies for all buckets into PolicySys.
func (sys *PolicySys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
for _, bucket := range buckets {
configData, err := globalBucketMetadataSys.GetConfig(bucket.Name, bucketPolicyConfig)
if err != nil {
if errors.Is(err, errConfigNotFound) {
continue
}
return err
}
config, err := policy.ParseConfig(bytes.NewReader(configData), bucket.Name)
if err != nil {
logger.LogIf(GlobalContext, err)
continue
}
sys.bucketPolicyMap[bucket.Name] = config
}
return nil
}
// Init - initializes policy system from policy.json of all buckets.
func (sys *PolicySys) Init(buckets []BucketInfo, objAPI ObjectLayer) error {
if objAPI == nil {
return errServerNotInitialized
}
// In gateway mode, we don't need to load the policies
// from the backend.
if globalIsGateway {
return nil
}
// Load PolicySys once during boot.
return sys.load(buckets, objAPI)
}
// NewPolicySys - creates new policy system. // NewPolicySys - creates new policy system.
func NewPolicySys() *PolicySys { func NewPolicySys() *PolicySys {
return &PolicySys{ return &PolicySys{}
bucketPolicyMap: make(map[string]*policy.Policy),
}
} }
func getConditionValues(r *http.Request, lc string, username string, claims map[string]interface{}) map[string][]string { func getConditionValues(r *http.Request, lc string, username string, claims map[string]interface{}) map[string][]string {

View File

@ -19,7 +19,6 @@ package cmd
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"sync" "sync"
"time" "time"
@ -32,95 +31,34 @@ import (
) )
// BucketQuotaSys - map of bucket and quota configuration. // BucketQuotaSys - map of bucket and quota configuration.
type BucketQuotaSys struct { type BucketQuotaSys struct{}
quotaMap map[string]madmin.BucketQuota
}
// Get - Get quota configuration. // Get - Get quota configuration.
func (sys *BucketQuotaSys) Get(bucketName string) (q madmin.BucketQuota, err error) { func (sys *BucketQuotaSys) Get(bucketName string) (*madmin.BucketQuota, error) {
if globalIsGateway { if globalIsGateway {
objAPI := newObjectLayerFn() objAPI := newObjectLayerFn()
if objAPI == nil { if objAPI == nil {
return q, errServerNotInitialized return nil, errServerNotInitialized
} }
return q, BucketQuotaConfigNotFound{Bucket: bucketName} return &madmin.BucketQuota{}, nil
} }
q, ok := sys.quotaMap[bucketName] return globalBucketMetadataSys.GetQuotaConfig(bucketName)
if !ok {
configData, err := globalBucketMetadataSys.GetConfig(bucketName, bucketQuotaConfigFile)
if err != nil {
if errors.Is(err, errConfigNotFound) {
return q, BucketQuotaConfigNotFound{Bucket: bucketName}
}
return q, err
}
return parseBucketQuota(bucketName, configData)
}
return q, nil
}
// Buckets - list of buckets with quota configuration
func (sys *BucketQuotaSys) Buckets() []string {
var buckets []string
for k := range sys.quotaMap {
buckets = append(buckets, k)
}
return buckets
}
// Init initialize bucket quota sys configuration with all buckets.
func (sys *BucketQuotaSys) Init(buckets []BucketInfo, objAPI ObjectLayer) error {
if objAPI == nil {
return errServerNotInitialized
}
// In gateway mode, we do not support bucket quota.
// So, this is a no-op for gateway servers.
if globalIsGateway {
return nil
}
return sys.load(buckets, objAPI)
}
func (sys *BucketQuotaSys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
for _, bucket := range buckets {
ctx := logger.SetReqInfo(GlobalContext, &logger.ReqInfo{BucketName: bucket.Name})
configData, err := globalBucketMetadataSys.GetConfig(bucket.Name, bucketQuotaConfigFile)
if err != nil {
if errors.Is(err, errConfigNotFound) {
continue
}
return err
}
quotaCfg, err := parseBucketQuota(bucket.Name, configData)
if err != nil {
if _, ok := err.(BucketQuotaConfigNotFound); !ok {
logger.LogIf(ctx, err)
}
continue
}
sys.quotaMap[bucket.Name] = quotaCfg
}
return nil
} }
// NewBucketQuotaSys returns initialized BucketQuotaSys // NewBucketQuotaSys returns initialized BucketQuotaSys
func NewBucketQuotaSys() *BucketQuotaSys { func NewBucketQuotaSys() *BucketQuotaSys {
return &BucketQuotaSys{quotaMap: map[string]madmin.BucketQuota{}} return &BucketQuotaSys{}
} }
// parseBucketQuota parses BucketQuota from json // parseBucketQuota parses BucketQuota from json
func parseBucketQuota(bucket string, data []byte) (quotaCfg madmin.BucketQuota, err error) { func parseBucketQuota(bucket string, data []byte) (quotaCfg *madmin.BucketQuota, err error) {
if len(data) == 0 { quotaCfg = &madmin.BucketQuota{}
return quotaCfg, BucketQuotaConfigNotFound{Bucket: bucket} if err = json.Unmarshal(data, quotaCfg); err != nil {
}
if err = json.Unmarshal(data, &quotaCfg); err != nil {
return quotaCfg, err return quotaCfg, err
} }
if !quotaCfg.Type.IsValid() { if !quotaCfg.IsValid() {
return quotaCfg, fmt.Errorf("Invalid quota type %s", quotaCfg.Type) return quotaCfg, fmt.Errorf("Invalid quota config %#v", quotaCfg)
} }
return return
} }
@ -131,7 +69,12 @@ type bucketStorageCache struct {
mu sync.Mutex mu sync.Mutex
} }
func (b *bucketStorageCache) check(ctx context.Context, q madmin.BucketQuota, bucket string, size int64) error { func (b *bucketStorageCache) check(ctx context.Context, q *madmin.BucketQuota, bucket string, size int64) error {
if q.Quota == 0 {
// No quota set return quickly.
return nil
}
b.mu.Lock() b.mu.Lock()
defer b.mu.Unlock() defer b.mu.Unlock()
if time.Since(b.lastUpdate) > 10*time.Second { if time.Since(b.lastUpdate) > 10*time.Second {
@ -143,7 +86,7 @@ func (b *bucketStorageCache) check(ctx context.Context, q madmin.BucketQuota, bu
b.bucketsSizes = dui.BucketsSizes b.bucketsSizes = dui.BucketsSizes
} }
currUsage := b.bucketsSizes[bucket] currUsage := b.bucketsSizes[bucket]
if (currUsage+uint64(size)) > q.Quota && q.Quota > 0 { if (currUsage + uint64(size)) > q.Quota {
return BucketQuotaExceeded{Bucket: bucket} return BucketQuotaExceeded{Bucket: bucket}
} }
return nil return nil
@ -155,15 +98,13 @@ func enforceBucketQuota(ctx context.Context, bucket string, size int64) error {
q, err := globalBucketQuotaSys.Get(bucket) q, err := globalBucketQuotaSys.Get(bucket)
if err != nil { if err != nil {
if _, ok := err.(BucketQuotaConfigNotFound); !ok {
return err
}
return nil return nil
} }
if q.Type == madmin.FIFOQuota { if q.Type == madmin.FIFOQuota {
return nil return nil
} }
return globalBucketStorageCache.check(ctx, q, bucket, size) return globalBucketStorageCache.check(ctx, q, bucket, size)
} }
@ -198,7 +139,12 @@ func enforceFIFOQuota(ctx context.Context, objectAPI ObjectLayer) error {
if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOff { if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOff {
return nil return nil
} }
for _, bucket := range globalBucketQuotaSys.Buckets() { buckets, err := objectAPI.ListBuckets(ctx)
if err != nil {
return err
}
for _, binfo := range buckets {
bucket := binfo.Name
// Check if the current bucket has quota restrictions, if not skip it // Check if the current bucket has quota restrictions, if not skip it
cfg, err := globalBucketQuotaSys.Get(bucket) cfg, err := globalBucketQuotaSys.Get(bucket)
if err != nil { if err != nil {

View File

@ -410,7 +410,6 @@ func (fs *FSObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
if err == nil { if err == nil {
created = meta.Created created = meta.Created
} }
globalBucketMetadataSys.Set(fi.Name(), meta)
bucketInfos = append(bucketInfos, BucketInfo{ bucketInfos = append(bucketInfos, BucketInfo{
Name: fi.Name(), Name: fi.Name(),

View File

@ -20,7 +20,6 @@ import (
"bytes" "bytes"
"context" "context"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"io" "io"
"net/url" "net/url"
@ -613,15 +612,11 @@ func (sys *NotificationSys) AddRemoteTarget(bucketName string, target event.Targ
func (sys *NotificationSys) load(buckets []BucketInfo, objAPI ObjectLayer) error { func (sys *NotificationSys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
for _, bucket := range buckets { for _, bucket := range buckets {
ctx := logger.SetReqInfo(GlobalContext, &logger.ReqInfo{BucketName: bucket.Name}) ctx := logger.SetReqInfo(GlobalContext, &logger.ReqInfo{BucketName: bucket.Name})
configData, err := globalBucketMetadataSys.GetConfig(bucket.Name, bucketNotificationConfig) config, err := globalBucketMetadataSys.GetNotificationConfig(bucket.Name)
if err != nil { if err != nil {
if errors.Is(err, errConfigNotFound) {
continue
}
return err return err
} }
config, err := event.ParseConfig(bytes.NewReader(configData), globalServerRegion, globalNotificationSys.targetList) if err = config.Validate(globalServerRegion, globalNotificationSys.targetList); err != nil {
if err != nil {
if _, ok := err.(*event.ErrARNNotFound); !ok { if _, ok := err.(*event.ErrARNNotFound); !ok {
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
} }

View File

@ -601,10 +601,8 @@ func (s *peerRESTServer) LoadBucketMetadataHandler(w http.ResponseWriter, r *htt
meta, err := loadBucketMetadata(r.Context(), objAPI, bucketName) meta, err := loadBucketMetadata(r.Context(), objAPI, bucketName)
if err != nil { if err != nil {
if !errors.Is(err, errConfigNotFound) { s.writeErrorResponse(w, err)
s.writeErrorResponse(w, err) return
return
}
} }
globalBucketMetadataSys.Set(bucketName, meta) globalBucketMetadataSys.Set(bucketName, meta)

View File

@ -27,8 +27,7 @@ import (
) )
func TestPolicySysIsAllowed(t *testing.T) { func TestPolicySysIsAllowed(t *testing.T) {
policySys := NewPolicySys() p := &policy.Policy{
policySys.bucketPolicyMap["mybucket"] = &policy.Policy{
Version: policy.DefaultVersion, Version: policy.DefaultVersion,
Statements: []policy.Statement{ Statements: []policy.Statement{
policy.NewStatement( policy.NewStatement(
@ -121,22 +120,21 @@ func TestPolicySysIsAllowed(t *testing.T) {
} }
testCases := []struct { testCases := []struct {
policySys *PolicySys
args policy.Args args policy.Args
expectedResult bool expectedResult bool
}{ }{
{policySys, anonGetBucketLocationArgs, true}, {anonGetBucketLocationArgs, true},
{policySys, anonPutObjectActionArgs, true}, {anonPutObjectActionArgs, true},
{policySys, anonGetObjectActionArgs, false}, {anonGetObjectActionArgs, false},
{policySys, getBucketLocationArgs, true}, {getBucketLocationArgs, true},
{policySys, putObjectActionArgs, true}, {putObjectActionArgs, true},
{policySys, getObjectActionArgs, true}, {getObjectActionArgs, true},
{policySys, yourbucketAnonGetObjectActionArgs, false}, {yourbucketAnonGetObjectActionArgs, false},
{policySys, yourbucketGetObjectActionArgs, true}, {yourbucketGetObjectActionArgs, true},
} }
for i, testCase := range testCases { for i, testCase := range testCases {
result := testCase.policySys.IsAllowed(testCase.args) result := p.IsAllowed(testCase.args)
if result != testCase.expectedResult { if result != testCase.expectedResult {
t.Fatalf("case %v: expected: %v, got: %v\n", i+1, testCase.expectedResult, result) t.Fatalf("case %v: expected: %v, got: %v\n", i+1, testCase.expectedResult, result)

View File

@ -335,31 +335,6 @@ func initAllSubsystems(newObject ObjectLayer) (err error) {
return fmt.Errorf("Unable to initialize notification system: %w", err) return fmt.Errorf("Unable to initialize notification system: %w", err)
} }
// Initialize policy system.
if err = globalPolicySys.Init(buckets, newObject); err != nil {
return fmt.Errorf("Unable to initialize policy system: %w", err)
}
// Initialize lifecycle system.
if err = globalLifecycleSys.Init(buckets, newObject); err != nil {
return fmt.Errorf("Unable to initialize lifecycle system: %w", err)
}
// Initialize bucket encryption subsystem.
if err = globalBucketSSEConfigSys.Init(buckets, newObject); err != nil {
return fmt.Errorf("Unable to initialize bucket encryption subsystem: %w", err)
}
// Initialize bucket object lock.
if err = globalBucketObjectLockSys.Init(buckets, newObject); err != nil {
return fmt.Errorf("Unable to initialize object lock system: %w", err)
}
// Initialize bucket quota system.
if err = globalBucketQuotaSys.Init(buckets, newObject); err != nil {
return fmt.Errorf("Unable to initialize bucket quota system: %w", err)
}
// Populate existing buckets to the etcd backend // Populate existing buckets to the etcd backend
if globalDNSConfig != nil { if globalDNSConfig != nil {
initFederatorBackend(buckets, newObject) initFederatorBackend(buckets, newObject)

View File

@ -323,8 +323,10 @@ func (z *xlZones) MakeBucketWithLocation(ctx context.Context, bucket, location s
} }
// If it doesn't exist we get a new, so ignore errors // If it doesn't exist we get a new, so ignore errors
meta, _ := globalBucketMetadataSys.Get(bucket) meta := newBucketMetadata(bucket)
meta.ObjectLockConfigurationXML = defaultBucketObjectLockConfig if lockEnabled {
meta.ObjectLockConfigurationXML = defaultBucketObjectLockConfig
}
if err := meta.Save(ctx, z); err != nil { if err := meta.Save(ctx, z); err != nil {
return toObjectErr(err, bucket) return toObjectErr(err, bucket)
} }
@ -351,8 +353,10 @@ func (z *xlZones) MakeBucketWithLocation(ctx context.Context, bucket, location s
} }
// If it doesn't exist we get a new, so ignore errors // If it doesn't exist we get a new, so ignore errors
meta, _ := globalBucketMetadataSys.Get(bucket) meta := newBucketMetadata(bucket)
meta.ObjectLockConfigurationXML = defaultBucketObjectLockConfig if lockEnabled {
meta.ObjectLockConfigurationXML = defaultBucketObjectLockConfig
}
if err := meta.Save(ctx, z); err != nil { if err := meta.Save(ctx, z); err != nil {
return toObjectErr(err, bucket) return toObjectErr(err, bucket)
} }
@ -1296,7 +1300,6 @@ func (z *xlZones) ListBuckets(ctx context.Context) (buckets []BucketInfo, err er
if err == nil { if err == nil {
buckets[i].Created = meta.Created buckets[i].Created = meta.Created
} }
globalBucketMetadataSys.Set(buckets[i].Name, meta)
} }
return buckets, nil return buckets, nil
} }
@ -1519,7 +1522,6 @@ func (z *xlZones) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error) {
if err == nil { if err == nil {
healBuckets[i].Created = meta.Created healBuckets[i].Created = meta.Created
} }
globalBucketMetadataSys.Set(healBuckets[i].Name, meta)
} }
return healBuckets, nil return healBuckets, nil

View File

@ -43,32 +43,17 @@ func (t QuotaType) IsValid() bool {
// BucketQuota holds bucket quota restrictions // BucketQuota holds bucket quota restrictions
type BucketQuota struct { type BucketQuota struct {
Quota uint64 `json:"quota"` Quota uint64 `json:"quota"`
Type QuotaType `json:"quotatype"` Type QuotaType `json:"quotatype,omitempty"`
} }
// RemoveBucketQuota - removes quota config on a bucket. // IsValid returns false if quota is invalid
func (adm *AdminClient) RemoveBucketQuota(ctx context.Context, bucket string) error { // empty quota when Quota == 0 is always true.
func (q BucketQuota) IsValid() bool {
queryValues := url.Values{} if q.Quota > 0 {
queryValues.Set("bucket", bucket) return q.Type.IsValid()
reqData := requestData{
relPath: adminAPIPrefix + "/remove-bucket-quota",
queryValues: queryValues,
} }
// Empty configs are valid.
// Execute DELETE on /minio/admin/v3/remove-bucket-quota to delete bucket quota. return true
resp, err := adm.executeMethod(ctx, http.MethodDelete, reqData)
defer closeResponse(resp)
if err != nil {
return err
}
if resp.StatusCode != http.StatusNoContent {
return httpRespToErrorResponse(resp)
}
return nil
} }
// GetBucketQuota - get info on a user // GetBucketQuota - get info on a user
@ -104,9 +89,9 @@ func (adm *AdminClient) GetBucketQuota(ctx context.Context, bucket string) (q Bu
return q, nil return q, nil
} }
// SetBucketQuota - sets a bucket's quota. // SetBucketQuota - sets a bucket's quota, if quota is set to '0'
// quota is disabled.
func (adm *AdminClient) SetBucketQuota(ctx context.Context, bucket string, quota uint64, quotaType QuotaType) error { func (adm *AdminClient) SetBucketQuota(ctx context.Context, bucket string, quota uint64, quotaType QuotaType) error {
data, err := json.Marshal(BucketQuota{ data, err := json.Marshal(BucketQuota{
Quota: quota, Quota: quota,
Type: quotaType, Type: quotaType,