MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 12:52:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

migrate all bucket metadata into a single file (#9586)

Browse Source 
this is a major overhaul by migrating off all
bucket metadata related configs into a single
object '.metadata.bin' this allows us for faster
bootups across 1000's of buckets and as well
as keeps the code simple enough for future
work and additions.

Additionally also fixes #9396, #9394
This commit is contained in:
Harshavardhana
2020-05-19 13:53:54 -07:00
committed by GitHub
parent 730775bb4e
commit bd032d13ff
48 changed files with 1648 additions and 2845 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/bucket/encryption/bucket-sse-config.go
View File

@@ -87,7 +87,7 @@ func ParseBucketSSEConfig(r io.Reader) (*BucketSSEConfig, error) {
// Validates server-side encryption config rules
// Only one rule is allowed on AWS S3
if len(config.Rules) != 1 {
return nil, errors.New("Only one server-side encryption rule is allowed")
return nil, errors.New("only one server-side encryption rule is allowed at a time")
}
for _, rule := range config.Rules {
@@ -98,7 +98,7 @@ func ParseBucketSSEConfig(r io.Reader) (*BucketSSEConfig, error) {
}
case AWSKms:
if rule.DefaultEncryptionAction.MasterKeyID == "" {
return nil, errors.New("MasterKeyID is missing")
return nil, errors.New("MasterKeyID is missing with aws:kms")
}
}
}

4
pkg/bucket/encryption/bucket-sse-config_test.go
View File

@@ -90,7 +90,7 @@ func TestParseBucketSSEConfig(t *testing.T) {
// 3. Invalid - more than one rule
{
inputXML: `<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`,
expectedErr: errors.New("Only one server-side encryption rule is allowed"),
expectedErr: errors.New("only one server-side encryption rule is allowed at a time"),
shouldPass: false,
},
// 4. Invalid XML - master key ID present along with AES256
@@ -102,7 +102,7 @@ func TestParseBucketSSEConfig(t *testing.T) {
// 5. Invalid XML - master key ID not provided when algorithm is set to aws:kms algorithm
{
inputXML: `<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>aws:kms</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`,
expectedErr: errors.New("MasterKeyID is missing"),
expectedErr: errors.New("MasterKeyID is missing with aws:kms"),
shouldPass: false,
},
// 6. Invalid Algorithm

18
pkg/bucket/object/lock/lock.go
View File

@@ -137,13 +137,9 @@ func UTCNowNTP() (time.Time, error) {
// Retention - bucket level retention configuration.
type Retention struct {
Mode RetMode
Validity time.Duration
}
// IsEmpty - returns whether retention is empty or not.
func (r Retention) IsEmpty() bool {
return !r.Mode.Valid() || r.Validity == 0
Mode RetMode
Validity time.Duration
LockEnabled bool
}
// Retain - check whether given date is retainable by validity time.
@@ -236,7 +232,7 @@ func (config *Config) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error
}
if parsedConfig.ObjectLockEnabled != "Enabled" {
return fmt.Errorf("only 'Enabled' value is allowd to ObjectLockEnabled element")
return fmt.Errorf("only 'Enabled' value is allowed to ObjectLockEnabled element")
}
*config = Config(parsedConfig)
@@ -244,8 +240,10 @@ func (config *Config) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error
}
// ToRetention - convert to Retention type.
func (config *Config) ToRetention() (r *Retention) {
r = &Retention{}
func (config *Config) ToRetention() Retention {
r := Retention{
LockEnabled: config.ObjectLockEnabled == "Enabled",
}
if config.Rule != nil {
r.Mode = config.Rule.DefaultRetention.Mode

20
pkg/bucket/object/lock/lock_test.go
View File

@@ -154,17 +154,17 @@ func TestParseObjectLockConfig(t *testing.T) {
expectErr bool
}{
{
value: "<ObjectLockConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\" ><ObjectLockEnabled>yes</ObjectLockEnabled></ObjectLockConfiguration>",
expectedErr: fmt.Errorf("only 'Enabled' value is allowd to ObjectLockEnabled element"),
value: `<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><ObjectLockEnabled>yes</ObjectLockEnabled></ObjectLockConfiguration>`,
expectedErr: fmt.Errorf("only 'Enabled' value is allowed to ObjectLockEnabled element"),
expectErr: true,
},
{
value: "<ObjectLockConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>COMPLIANCE</Mode><Days>0</Days></DefaultRetention></Rule></ObjectLockConfiguration>",
value: `<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>COMPLIANCE</Mode><Days>0</Days></DefaultRetention></Rule></ObjectLockConfiguration>`,
expectedErr: fmt.Errorf("Default retention period must be a positive integer value for 'Days'"),
expectErr: true,
},
{
value: "<ObjectLockConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>COMPLIANCE</Mode><Days>30</Days></DefaultRetention></Rule></ObjectLockConfiguration>",
value: `<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>COMPLIANCE</Mode><Days>30</Days></DefaultRetention></Rule></ObjectLockConfiguration>`,
expectedErr: nil,
expectErr: false,
},
@@ -190,17 +190,17 @@ func TestParseObjectRetention(t *testing.T) {
expectErr bool
}{
{
value: "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Retention xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Mode>string</Mode><RetainUntilDate>2020-01-02T15:04:05Z</RetainUntilDate></Retention>",
value: `<?xml version="1.0" encoding="UTF-8"?><Retention xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Mode>string</Mode><RetainUntilDate>2020-01-02T15:04:05Z</RetainUntilDate></Retention>`,
expectedErr: ErrUnknownWORMModeDirective,
expectErr: true,
},
{
value: "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Retention xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Mode>COMPLIANCE</Mode><RetainUntilDate>2017-01-02T15:04:05Z</RetainUntilDate></Retention>",
value: `<?xml version="1.0" encoding="UTF-8"?><Retention xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Mode>COMPLIANCE</Mode><RetainUntilDate>2017-01-02T15:04:05Z</RetainUntilDate></Retention>`,
expectedErr: ErrPastObjectLockRetainDate,
expectErr: true,
},
{
value: "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Retention xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Mode>GOVERNANCE</Mode><RetainUntilDate>2057-01-02T15:04:05Z</RetainUntilDate></Retention>",
value: `<?xml version="1.0" encoding="UTF-8"?><Retention xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Mode>GOVERNANCE</Mode><RetainUntilDate>2057-01-02T15:04:05Z</RetainUntilDate></Retention>`,
expectedErr: nil,
expectErr: false,
},
@@ -458,17 +458,17 @@ func TestParseObjectLegalHold(t *testing.T) {
expectErr bool
}{
{
value: "<?xml version=\"1.0\" encoding=\"UTF-8\"?><LegalHold xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Status>string</Status></LegalHold>",
value: `<?xml version="1.0" encoding="UTF-8"?><LegalHold xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>string</Status></LegalHold>`,
expectedErr: ErrMalformedXML,
expectErr: true,
},
{
value: "<?xml version=\"1.0\" encoding=\"UTF-8\"?><LegalHold xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Status>ON</Status></LegalHold>",
value: `<?xml version="1.0" encoding="UTF-8"?><LegalHold xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>ON</Status></LegalHold>`,
expectedErr: nil,
expectErr: false,
},
{
value: "<?xml version=\"1.0\" encoding=\"UTF-8\"?><LegalHold xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Status>On</Status></LegalHold>",
value: `<?xml version="1.0" encoding="UTF-8"?><LegalHold xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>On</Status></LegalHold>`,
expectedErr: ErrMalformedXML,
expectErr: true,
},