Add support for site replication healing (#14572)

heal bucket metadata and IAM entries for
sites participating in site replication from
the site with the most updated entry.

Co-authored-by: Harshavardhana <harsha@minio.io>
Co-authored-by: Aditya Manthramurthy <aditya@minio.io>
This commit is contained in:
Poorna 2022-04-24 15:06:31 +05:30 committed by GitHub
parent d087e28dce
commit 3a64580663
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 1735 additions and 339 deletions

View File

@ -120,7 +120,7 @@ func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *
return return
} }
config, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket) config, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket)
if err != nil { if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return return

View File

@ -1182,8 +1182,8 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
lcfg, _ := globalBucketObjectLockSys.Get(bucket.Name) lcfg, _ := globalBucketObjectLockSys.Get(bucket.Name)
quota, _ := globalBucketQuotaSys.Get(ctx, bucket.Name) quota, _ := globalBucketQuotaSys.Get(ctx, bucket.Name)
rcfg, _ := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket.Name) rcfg, _, _ := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket.Name)
tcfg, _ := globalBucketMetadataSys.GetTaggingConfig(bucket.Name) tcfg, _, _ := globalBucketMetadataSys.GetTaggingConfig(bucket.Name)
acctInfo.Buckets = append(acctInfo.Buckets, madmin.BucketAccessInfo{ acctInfo.Buckets = append(acctInfo.Buckets, madmin.BucketAccessInfo{
Name: bucket.Name, Name: bucket.Name,

View File

@ -158,7 +158,7 @@ func (api objectAPIHandlers) GetBucketEncryptionHandler(w http.ResponseWriter, r
return return
} }
config, err := globalBucketMetadataSys.GetSSEConfig(bucket) config, _, err := globalBucketMetadataSys.GetSSEConfig(bucket)
if err != nil { if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return return

View File

@ -43,7 +43,8 @@ func (sys *BucketSSEConfigSys) Get(bucket string) (*sse.BucketSSEConfig, error)
return nil, BucketSSEConfigNotFound{Bucket: bucket} return nil, BucketSSEConfigNotFound{Bucket: bucket}
} }
return globalBucketMetadataSys.GetSSEConfig(bucket) sseCfg, _, err := globalBucketMetadataSys.GetSSEConfig(bucket)
return sseCfg, err
} }
// validateBucketSSEConfig parses bucket encryption configuration and validates if it is supported by MinIO. // validateBucketSSEConfig parses bucket encryption configuration and validates if it is supported by MinIO.

View File

@ -1374,7 +1374,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
} }
// Deny object locking configuration settings on existing buckets without object lock enabled. // Deny object locking configuration settings on existing buckets without object lock enabled.
if _, err = globalBucketMetadataSys.GetObjectLockConfig(bucket); err != nil { if _, _, err = globalBucketMetadataSys.GetObjectLockConfig(bucket); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return return
} }
@ -1427,7 +1427,7 @@ func (api objectAPIHandlers) GetBucketObjectLockConfigHandler(w http.ResponseWri
return return
} }
config, err := globalBucketMetadataSys.GetObjectLockConfig(bucket) config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
if err != nil { if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return return
@ -1529,7 +1529,7 @@ func (api objectAPIHandlers) GetBucketTaggingHandler(w http.ResponseWriter, r *h
return return
} }
config, err := globalBucketMetadataSys.GetTaggingConfig(bucket) config, _, err := globalBucketMetadataSys.GetTaggingConfig(bucket)
if err != nil { if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return return
@ -1677,7 +1677,7 @@ func (api objectAPIHandlers) GetBucketReplicationConfigHandler(w http.ResponseWr
return return
} }
config, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket) config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
if err != nil { if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return return
@ -1820,7 +1820,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
return return
} }
config, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket) config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
if err != nil { if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return return
@ -1908,7 +1908,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.Response
return return
} }
if _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket); err != nil { if _, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return return
} }

View File

@ -23,6 +23,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"sync" "sync"
"time"
"github.com/minio/madmin-go" "github.com/minio/madmin-go"
"github.com/minio/minio-go/v7/pkg/tags" "github.com/minio/minio-go/v7/pkg/tags"
@ -111,21 +112,26 @@ func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configF
switch configFile { switch configFile {
case bucketPolicyConfig: case bucketPolicyConfig:
meta.PolicyConfigJSON = configData meta.PolicyConfigJSON = configData
meta.PolicyConfigUpdatedAt = UTCNow()
case bucketNotificationConfig: case bucketNotificationConfig:
meta.NotificationConfigXML = configData meta.NotificationConfigXML = configData
case bucketLifecycleConfig: case bucketLifecycleConfig:
meta.LifecycleConfigXML = configData meta.LifecycleConfigXML = configData
case bucketSSEConfig: case bucketSSEConfig:
meta.EncryptionConfigXML = configData meta.EncryptionConfigXML = configData
meta.EncryptionConfigUpdatedAt = UTCNow()
case bucketTaggingConfig: case bucketTaggingConfig:
meta.TaggingConfigXML = configData meta.TaggingConfigXML = configData
meta.TaggingConfigUpdatedAt = UTCNow()
case bucketQuotaConfigFile: case bucketQuotaConfigFile:
meta.QuotaConfigJSON = configData meta.QuotaConfigJSON = configData
meta.QuotaConfigUpdatedAt = UTCNow()
case objectLockConfig: case objectLockConfig:
if !globalIsErasure && !globalIsDistErasure { if !globalIsErasure && !globalIsDistErasure {
return NotImplemented{} return NotImplemented{}
} }
meta.ObjectLockConfigXML = configData meta.ObjectLockConfigXML = configData
meta.ObjectLockConfigUpdatedAt = UTCNow()
case bucketVersioningConfig: case bucketVersioningConfig:
if !globalIsErasure && !globalIsDistErasure { if !globalIsErasure && !globalIsDistErasure {
return NotImplemented{} return NotImplemented{}
@ -136,6 +142,7 @@ func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configF
return NotImplemented{} return NotImplemented{}
} }
meta.ReplicationConfigXML = configData meta.ReplicationConfigXML = configData
meta.ReplicationConfigUpdatedAt = UTCNow()
case bucketTargetsFile: case bucketTargetsFile:
meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(meta.Name, configData, kms.Context{ meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(meta.Name, configData, kms.Context{
bucket: meta.Name, bucket: meta.Name,
@ -196,34 +203,34 @@ func (sys *BucketMetadataSys) GetVersioningConfig(bucket string) (*versioning.Ve
// GetTaggingConfig returns configured tagging config // GetTaggingConfig returns configured tagging config
// The returned object may not be modified. // The returned object may not be modified.
func (sys *BucketMetadataSys) GetTaggingConfig(bucket string) (*tags.Tags, error) { func (sys *BucketMetadataSys) GetTaggingConfig(bucket string) (*tags.Tags, time.Time, error) {
meta, err := sys.GetConfig(GlobalContext, bucket) meta, err := sys.GetConfig(GlobalContext, bucket)
if err != nil { if err != nil {
if errors.Is(err, errConfigNotFound) { if errors.Is(err, errConfigNotFound) {
return nil, BucketTaggingNotFound{Bucket: bucket} return nil, time.Time{}, BucketTaggingNotFound{Bucket: bucket}
} }
return nil, err return nil, time.Time{}, err
} }
if meta.taggingConfig == nil { if meta.taggingConfig == nil {
return nil, BucketTaggingNotFound{Bucket: bucket} return nil, time.Time{}, BucketTaggingNotFound{Bucket: bucket}
} }
return meta.taggingConfig, nil return meta.taggingConfig, meta.TaggingConfigUpdatedAt, nil
} }
// GetObjectLockConfig returns configured object lock config // GetObjectLockConfig returns configured object lock config
// The returned object may not be modified. // The returned object may not be modified.
func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Config, error) { func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Config, time.Time, error) {
meta, err := sys.GetConfig(GlobalContext, bucket) meta, err := sys.GetConfig(GlobalContext, bucket)
if err != nil { if err != nil {
if errors.Is(err, errConfigNotFound) { if errors.Is(err, errConfigNotFound) {
return nil, BucketObjectLockConfigNotFound{Bucket: bucket} return nil, time.Time{}, BucketObjectLockConfigNotFound{Bucket: bucket}
} }
return nil, err return nil, time.Time{}, err
} }
if meta.objectLockConfig == nil { if meta.objectLockConfig == nil {
return nil, BucketObjectLockConfigNotFound{Bucket: bucket} return nil, time.Time{}, BucketObjectLockConfigNotFound{Bucket: bucket}
} }
return meta.objectLockConfig, nil return meta.objectLockConfig, meta.ObjectLockConfigUpdatedAt, nil
} }
// GetLifecycleConfig returns configured lifecycle config // GetLifecycleConfig returns configured lifecycle config
@ -283,18 +290,27 @@ func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Confi
// GetSSEConfig returns configured SSE config // GetSSEConfig returns configured SSE config
// The returned object may not be modified. // The returned object may not be modified.
func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEConfig, error) { func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEConfig, time.Time, error) {
meta, err := sys.GetConfig(GlobalContext, bucket) meta, err := sys.GetConfig(GlobalContext, bucket)
if err != nil { if err != nil {
if errors.Is(err, errConfigNotFound) { if errors.Is(err, errConfigNotFound) {
return nil, BucketSSEConfigNotFound{Bucket: bucket} return nil, time.Time{}, BucketSSEConfigNotFound{Bucket: bucket}
} }
return nil, err return nil, time.Time{}, err
} }
if meta.sseConfig == nil { if meta.sseConfig == nil {
return nil, BucketSSEConfigNotFound{Bucket: bucket} return nil, time.Time{}, BucketSSEConfigNotFound{Bucket: bucket}
} }
return meta.sseConfig, nil return meta.sseConfig, meta.EncryptionConfigUpdatedAt, nil
}
// CreatedAt returns the time of creation of bucket
func (sys *BucketMetadataSys) CreatedAt(bucket string) (time.Time, error) {
meta, err := sys.GetConfig(GlobalContext, bucket)
if err != nil {
return time.Time{}, err
}
return meta.Created.UTC(), nil
} }
// GetPolicyConfig returns configured bucket policy // GetPolicyConfig returns configured bucket policy
@ -323,29 +339,29 @@ func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, er
// GetQuotaConfig returns configured bucket quota // GetQuotaConfig returns configured bucket quota
// The returned object may not be modified. // The returned object may not be modified.
func (sys *BucketMetadataSys) GetQuotaConfig(ctx context.Context, bucket string) (*madmin.BucketQuota, error) { func (sys *BucketMetadataSys) GetQuotaConfig(ctx context.Context, bucket string) (*madmin.BucketQuota, time.Time, error) {
meta, err := sys.GetConfig(ctx, bucket) meta, err := sys.GetConfig(ctx, bucket)
if err != nil { if err != nil {
return nil, err return nil, time.Time{}, err
} }
return meta.quotaConfig, nil return meta.quotaConfig, meta.QuotaConfigUpdatedAt, nil
} }
// GetReplicationConfig returns configured bucket replication config // GetReplicationConfig returns configured bucket replication config
// The returned object may not be modified. // The returned object may not be modified.
func (sys *BucketMetadataSys) GetReplicationConfig(ctx context.Context, bucket string) (*replication.Config, error) { func (sys *BucketMetadataSys) GetReplicationConfig(ctx context.Context, bucket string) (*replication.Config, time.Time, error) {
meta, err := sys.GetConfig(ctx, bucket) meta, err := sys.GetConfig(ctx, bucket)
if err != nil { if err != nil {
if errors.Is(err, errConfigNotFound) { if errors.Is(err, errConfigNotFound) {
return nil, BucketReplicationConfigNotFound{Bucket: bucket} return nil, time.Time{}, BucketReplicationConfigNotFound{Bucket: bucket}
} }
return nil, err return nil, time.Time{}, err
} }
if meta.replicationConfig == nil { if meta.replicationConfig == nil {
return nil, BucketReplicationConfigNotFound{Bucket: bucket} return nil, time.Time{}, BucketReplicationConfigNotFound{Bucket: bucket}
} }
return meta.replicationConfig, nil return meta.replicationConfig, meta.ReplicationConfigUpdatedAt, nil
} }
// GetBucketTargetsConfig returns configured bucket targets for this bucket // GetBucketTargetsConfig returns configured bucket targets for this bucket

View File

@ -81,6 +81,12 @@ type BucketMetadata struct {
ReplicationConfigXML []byte ReplicationConfigXML []byte
BucketTargetsConfigJSON []byte BucketTargetsConfigJSON []byte
BucketTargetsConfigMetaJSON []byte BucketTargetsConfigMetaJSON []byte
PolicyConfigUpdatedAt time.Time
ObjectLockConfigUpdatedAt time.Time
EncryptionConfigUpdatedAt time.Time
TaggingConfigUpdatedAt time.Time
QuotaConfigUpdatedAt time.Time
ReplicationConfigUpdatedAt time.Time
// Unexported fields. Must be updated atomically. // Unexported fields. Must be updated atomically.
policyConfig *policy.Policy policyConfig *policy.Policy
@ -98,9 +104,10 @@ type BucketMetadata struct {
// newBucketMetadata creates BucketMetadata with the supplied name and Created to Now. // newBucketMetadata creates BucketMetadata with the supplied name and Created to Now.
func newBucketMetadata(name string) BucketMetadata { func newBucketMetadata(name string) BucketMetadata {
now := UTCNow()
return BucketMetadata{ return BucketMetadata{
Name: name, Name: name,
Created: UTCNow(), Created: now,
notificationConfig: &event.Config{ notificationConfig: &event.Config{
XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/", XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
}, },
@ -157,8 +164,13 @@ func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket strin
if err := b.convertLegacyConfigs(ctx, objectAPI); err != nil { if err := b.convertLegacyConfigs(ctx, objectAPI); err != nil {
return b, err return b, err
} }
// migrate unencrypted remote targets // migrate unencrypted remote targets
return b, b.migrateTargetConfig(ctx, objectAPI) if err = b.migrateTargetConfig(ctx, objectAPI); err != nil {
return b, err
}
b.defaultTimestamps()
return b, nil
} }
// parseAllConfigs will parse all configs and populate the private fields. // parseAllConfigs will parse all configs and populate the private fields.
@ -347,6 +359,32 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
return nil return nil
} }
// default timestamps to metadata Created timestamp if unset.
func (b *BucketMetadata) defaultTimestamps() {
if b.PolicyConfigUpdatedAt.IsZero() {
b.PolicyConfigUpdatedAt = b.Created
}
if b.EncryptionConfigUpdatedAt.IsZero() {
b.EncryptionConfigUpdatedAt = b.Created
}
if b.TaggingConfigUpdatedAt.IsZero() {
b.TaggingConfigUpdatedAt = b.Created
}
if b.ObjectLockConfigUpdatedAt.IsZero() {
b.ObjectLockConfigUpdatedAt = b.Created
}
if b.QuotaConfigUpdatedAt.IsZero() {
b.QuotaConfigUpdatedAt = b.Created
}
if b.ReplicationConfigUpdatedAt.IsZero() {
b.ReplicationConfigUpdatedAt = b.Created
}
}
// Save config to supplied ObjectLayer api. // Save config to supplied ObjectLayer api.
func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error { func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
if err := b.parseAllConfigs(ctx, api); err != nil { if err := b.parseAllConfigs(ctx, api); err != nil {

View File

@ -108,6 +108,42 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON") err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
return return
} }
case "PolicyConfigUpdatedAt":
z.PolicyConfigUpdatedAt, err = dc.ReadTime()
if err != nil {
err = msgp.WrapError(err, "PolicyConfigUpdatedAt")
return
}
case "ObjectLockConfigUpdatedAt":
z.ObjectLockConfigUpdatedAt, err = dc.ReadTime()
if err != nil {
err = msgp.WrapError(err, "ObjectLockConfigUpdatedAt")
return
}
case "EncryptionConfigUpdatedAt":
z.EncryptionConfigUpdatedAt, err = dc.ReadTime()
if err != nil {
err = msgp.WrapError(err, "EncryptionConfigUpdatedAt")
return
}
case "TaggingConfigUpdatedAt":
z.TaggingConfigUpdatedAt, err = dc.ReadTime()
if err != nil {
err = msgp.WrapError(err, "TaggingConfigUpdatedAt")
return
}
case "QuotaConfigUpdatedAt":
z.QuotaConfigUpdatedAt, err = dc.ReadTime()
if err != nil {
err = msgp.WrapError(err, "QuotaConfigUpdatedAt")
return
}
case "ReplicationConfigUpdatedAt":
z.ReplicationConfigUpdatedAt, err = dc.ReadTime()
if err != nil {
err = msgp.WrapError(err, "ReplicationConfigUpdatedAt")
return
}
default: default:
err = dc.Skip() err = dc.Skip()
if err != nil { if err != nil {
@ -121,9 +157,9 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
// EncodeMsg implements msgp.Encodable // EncodeMsg implements msgp.Encodable
func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) { func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 14 // map header, size 20
// write "Name" // write "Name"
err = en.Append(0x8e, 0xa4, 0x4e, 0x61, 0x6d, 0x65) err = en.Append(0xde, 0x0, 0x14, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
if err != nil { if err != nil {
return return
} }
@ -262,15 +298,75 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON") err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
return return
} }
// write "PolicyConfigUpdatedAt"
err = en.Append(0xb5, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
if err != nil {
return
}
err = en.WriteTime(z.PolicyConfigUpdatedAt)
if err != nil {
err = msgp.WrapError(err, "PolicyConfigUpdatedAt")
return
}
// write "ObjectLockConfigUpdatedAt"
err = en.Append(0xb9, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
if err != nil {
return
}
err = en.WriteTime(z.ObjectLockConfigUpdatedAt)
if err != nil {
err = msgp.WrapError(err, "ObjectLockConfigUpdatedAt")
return
}
// write "EncryptionConfigUpdatedAt"
err = en.Append(0xb9, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
if err != nil {
return
}
err = en.WriteTime(z.EncryptionConfigUpdatedAt)
if err != nil {
err = msgp.WrapError(err, "EncryptionConfigUpdatedAt")
return
}
// write "TaggingConfigUpdatedAt"
err = en.Append(0xb6, 0x54, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
if err != nil {
return
}
err = en.WriteTime(z.TaggingConfigUpdatedAt)
if err != nil {
err = msgp.WrapError(err, "TaggingConfigUpdatedAt")
return
}
// write "QuotaConfigUpdatedAt"
err = en.Append(0xb4, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
if err != nil {
return
}
err = en.WriteTime(z.QuotaConfigUpdatedAt)
if err != nil {
err = msgp.WrapError(err, "QuotaConfigUpdatedAt")
return
}
// write "ReplicationConfigUpdatedAt"
err = en.Append(0xba, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
if err != nil {
return
}
err = en.WriteTime(z.ReplicationConfigUpdatedAt)
if err != nil {
err = msgp.WrapError(err, "ReplicationConfigUpdatedAt")
return
}
return return
} }
// MarshalMsg implements msgp.Marshaler // MarshalMsg implements msgp.Marshaler
func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) { func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize()) o = msgp.Require(b, z.Msgsize())
// map header, size 14 // map header, size 20
// string "Name" // string "Name"
o = append(o, 0x8e, 0xa4, 0x4e, 0x61, 0x6d, 0x65) o = append(o, 0xde, 0x0, 0x14, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
o = msgp.AppendString(o, z.Name) o = msgp.AppendString(o, z.Name)
// string "Created" // string "Created"
o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64) o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
@ -311,6 +407,24 @@ func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
// string "BucketTargetsConfigMetaJSON" // string "BucketTargetsConfigMetaJSON"
o = append(o, 0xbb, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x4a, 0x53, 0x4f, 0x4e) o = append(o, 0xbb, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x4a, 0x53, 0x4f, 0x4e)
o = msgp.AppendBytes(o, z.BucketTargetsConfigMetaJSON) o = msgp.AppendBytes(o, z.BucketTargetsConfigMetaJSON)
// string "PolicyConfigUpdatedAt"
o = append(o, 0xb5, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
o = msgp.AppendTime(o, z.PolicyConfigUpdatedAt)
// string "ObjectLockConfigUpdatedAt"
o = append(o, 0xb9, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
o = msgp.AppendTime(o, z.ObjectLockConfigUpdatedAt)
// string "EncryptionConfigUpdatedAt"
o = append(o, 0xb9, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
o = msgp.AppendTime(o, z.EncryptionConfigUpdatedAt)
// string "TaggingConfigUpdatedAt"
o = append(o, 0xb6, 0x54, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
o = msgp.AppendTime(o, z.TaggingConfigUpdatedAt)
// string "QuotaConfigUpdatedAt"
o = append(o, 0xb4, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
o = msgp.AppendTime(o, z.QuotaConfigUpdatedAt)
// string "ReplicationConfigUpdatedAt"
o = append(o, 0xba, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
o = msgp.AppendTime(o, z.ReplicationConfigUpdatedAt)
return return
} }
@ -416,6 +530,42 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON") err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
return return
} }
case "PolicyConfigUpdatedAt":
z.PolicyConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
if err != nil {
err = msgp.WrapError(err, "PolicyConfigUpdatedAt")
return
}
case "ObjectLockConfigUpdatedAt":
z.ObjectLockConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
if err != nil {
err = msgp.WrapError(err, "ObjectLockConfigUpdatedAt")
return
}
case "EncryptionConfigUpdatedAt":
z.EncryptionConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
if err != nil {
err = msgp.WrapError(err, "EncryptionConfigUpdatedAt")
return
}
case "TaggingConfigUpdatedAt":
z.TaggingConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
if err != nil {
err = msgp.WrapError(err, "TaggingConfigUpdatedAt")
return
}
case "QuotaConfigUpdatedAt":
z.QuotaConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
if err != nil {
err = msgp.WrapError(err, "QuotaConfigUpdatedAt")
return
}
case "ReplicationConfigUpdatedAt":
z.ReplicationConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
if err != nil {
err = msgp.WrapError(err, "ReplicationConfigUpdatedAt")
return
}
default: default:
bts, err = msgp.Skip(bts) bts, err = msgp.Skip(bts)
if err != nil { if err != nil {
@ -430,6 +580,6 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *BucketMetadata) Msgsize() (s int) { func (z *BucketMetadata) Msgsize() (s int) {
s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON) s = 3 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON) + 22 + msgp.TimeSize + 26 + msgp.TimeSize + 26 + msgp.TimeSize + 23 + msgp.TimeSize + 21 + msgp.TimeSize + 27 + msgp.TimeSize
return return
} }

View File

@ -44,7 +44,7 @@ func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention,
return r, nil return r, nil
} }
config, err := globalBucketMetadataSys.GetObjectLockConfig(bucketName) config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucketName)
if err != nil { if err != nil {
if _, ok := err.(BucketObjectLockConfigNotFound); ok { if _, ok := err.(BucketObjectLockConfigNotFound); ok {
return r, nil return r, nil

View File

@ -42,8 +42,8 @@ func (sys *BucketQuotaSys) Get(ctx context.Context, bucketName string) (*madmin.
} }
return &madmin.BucketQuota{}, nil return &madmin.BucketQuota{}, nil
} }
qCfg, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucketName)
return globalBucketMetadataSys.GetQuotaConfig(ctx, bucketName) return qCfg, err
} }
// NewBucketQuotaSys returns initialized BucketQuotaSys // NewBucketQuotaSys returns initialized BucketQuotaSys

View File

@ -81,7 +81,8 @@ func getReplicationConfig(ctx context.Context, bucketName string) (rc *replicati
return rc, BucketReplicationConfigNotFound{Bucket: bucketName} return rc, BucketReplicationConfigNotFound{Bucket: bucketName}
} }
return globalBucketMetadataSys.GetReplicationConfig(ctx, bucketName) rCfg, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucketName)
return rCfg, err
} }
// validateReplicationDestination returns error if replication destination bucket missing or not configured // validateReplicationDestination returns error if replication destination bucket missing or not configured

View File

@ -127,27 +127,30 @@ func getMappedPolicyPath(name string, userType IAMUserType, isGroup bool) string
type UserIdentity struct { type UserIdentity struct {
Version int `json:"version"` Version int `json:"version"`
Credentials auth.Credentials `json:"credentials"` Credentials auth.Credentials `json:"credentials"`
UpdatedAt time.Time `json:"updatedAt,omitempty"`
} }
func newUserIdentity(cred auth.Credentials) UserIdentity { func newUserIdentity(cred auth.Credentials) UserIdentity {
return UserIdentity{Version: 1, Credentials: cred} return UserIdentity{Version: 1, Credentials: cred, UpdatedAt: UTCNow()}
} }
// GroupInfo contains info about a group // GroupInfo contains info about a group
type GroupInfo struct { type GroupInfo struct {
Version int `json:"version"` Version int `json:"version"`
Status string `json:"status"` Status string `json:"status"`
Members []string `json:"members"` Members []string `json:"members"`
UpdatedAt time.Time `json:"updatedAt,omitempty"`
} }
func newGroupInfo(members []string) GroupInfo { func newGroupInfo(members []string) GroupInfo {
return GroupInfo{Version: 1, Status: statusEnabled, Members: members} return GroupInfo{Version: 1, Status: statusEnabled, Members: members, UpdatedAt: UTCNow()}
} }
// MappedPolicy represents a policy name mapped to a user or group // MappedPolicy represents a policy name mapped to a user or group
type MappedPolicy struct { type MappedPolicy struct {
Version int `json:"version"` Version int `json:"version"`
Policies string `json:"policy"` Policies string `json:"policy"`
UpdatedAt time.Time `json:"updatedAt,omitempty"`
} }
// converts a mapped policy into a slice of distinct policies // converts a mapped policy into a slice of distinct policies
@ -168,7 +171,7 @@ func (mp MappedPolicy) policySet() set.StringSet {
} }
func newMappedPolicy(policy string) MappedPolicy { func newMappedPolicy(policy string) MappedPolicy {
return MappedPolicy{Version: 1, Policies: policy} return MappedPolicy{Version: 1, Policies: policy, UpdatedAt: UTCNow()}
} }
// PolicyDoc represents an IAM policy with some metadata. // PolicyDoc represents an IAM policy with some metadata.
@ -314,26 +317,26 @@ func (c *iamCache) removeGroupFromMembershipsMap(group string) {
// information in IAM (i.e sys.iam*Map) - this info is stored only in the STS // information in IAM (i.e sys.iam*Map) - this info is stored only in the STS
// generated credentials. Thus we skip looking up group memberships, user map, // generated credentials. Thus we skip looking up group memberships, user map,
// and group map and check the appropriate policy maps directly. // and group map and check the appropriate policy maps directly.
func (c *iamCache) policyDBGet(mode UsersSysType, name string, isGroup bool) ([]string, error) { func (c *iamCache) policyDBGet(mode UsersSysType, name string, isGroup bool) ([]string, time.Time, error) {
if isGroup { if isGroup {
if mode == MinIOUsersSysType { if mode == MinIOUsersSysType {
g, ok := c.iamGroupsMap[name] g, ok := c.iamGroupsMap[name]
if !ok { if !ok {
return nil, errNoSuchGroup return nil, time.Time{}, errNoSuchGroup
} }
// Group is disabled, so we return no policy - this // Group is disabled, so we return no policy - this
// ensures the request is denied. // ensures the request is denied.
if g.Status == statusDisabled { if g.Status == statusDisabled {
return nil, nil return nil, time.Time{}, nil
} }
} }
return c.iamGroupPolicyMap[name].toSlice(), nil return c.iamGroupPolicyMap[name].toSlice(), c.iamGroupPolicyMap[name].UpdatedAt, nil
} }
if name == globalActiveCred.AccessKey { if name == globalActiveCred.AccessKey {
return []string{"consoleAdmin"}, nil return []string{"consoleAdmin"}, time.Time{}, nil
} }
// When looking for a user's policies, we also check if the user // When looking for a user's policies, we also check if the user
@ -342,7 +345,7 @@ func (c *iamCache) policyDBGet(mode UsersSysType, name string, isGroup bool) ([]
u, ok := c.iamUsersMap[name] u, ok := c.iamUsersMap[name]
if ok { if ok {
if !u.IsValid() { if !u.IsValid() {
return nil, nil return nil, time.Time{}, nil
} }
parentName = u.ParentUser parentName = u.ParentUser
} }
@ -353,7 +356,7 @@ func (c *iamCache) policyDBGet(mode UsersSysType, name string, isGroup bool) ([]
if parentName == globalActiveCred.AccessKey && u.IsServiceAccount() { if parentName == globalActiveCred.AccessKey && u.IsServiceAccount() {
// even if this is set, the claims present in the service // even if this is set, the claims present in the service
// accounts apply the final permissions if any. // accounts apply the final permissions if any.
return []string{"consoleAdmin"}, nil return []string{"consoleAdmin"}, mp.UpdatedAt, nil
} }
if parentName != "" { if parentName != "" {
mp = c.iamUserPolicyMap[parentName] mp = c.iamUserPolicyMap[parentName]
@ -373,7 +376,7 @@ func (c *iamCache) policyDBGet(mode UsersSysType, name string, isGroup bool) ([]
policies = append(policies, c.iamGroupPolicyMap[group].toSlice()...) policies = append(policies, c.iamGroupPolicyMap[group].toSlice()...)
} }
return policies, nil return policies, mp.UpdatedAt, nil
} }
// IAMStorageAPI defines an interface for the IAM persistence layer // IAMStorageAPI defines an interface for the IAM persistence layer
@ -572,14 +575,14 @@ func (store *IAMStoreSys) PolicyDBGet(name string, isGroup bool, groups ...strin
cache := store.rlock() cache := store.rlock()
defer store.runlock() defer store.runlock()
policies, err := cache.policyDBGet(store.getUsersSysType(), name, isGroup) policies, _, err := cache.policyDBGet(store.getUsersSysType(), name, isGroup)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if !isGroup { if !isGroup {
for _, group := range groups { for _, group := range groups {
ps, err := cache.policyDBGet(store.getUsersSysType(), group, true) ps, _, err := cache.policyDBGet(store.getUsersSysType(), group, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -755,7 +758,7 @@ func (store *IAMStoreSys) GetGroupDescription(group string) (gd madmin.GroupDesc
cache := store.rlock() cache := store.rlock()
defer store.runlock() defer store.runlock()
ps, err := cache.policyDBGet(store.getUsersSysType(), group, true) ps, updatedAt, err := cache.policyDBGet(store.getUsersSysType(), group, true)
if err != nil { if err != nil {
return gd, err return gd, err
} }
@ -764,8 +767,9 @@ func (store *IAMStoreSys) GetGroupDescription(group string) (gd madmin.GroupDesc
if store.getUsersSysType() != MinIOUsersSysType { if store.getUsersSysType() != MinIOUsersSysType {
return madmin.GroupDesc{ return madmin.GroupDesc{
Name: group, Name: group,
Policy: policy, Policy: policy,
UpdatedAt: updatedAt,
}, nil }, nil
} }
@ -775,10 +779,11 @@ func (store *IAMStoreSys) GetGroupDescription(group string) (gd madmin.GroupDesc
} }
return madmin.GroupDesc{ return madmin.GroupDesc{
Name: group, Name: group,
Status: gi.Status, Status: gi.Status,
Members: gi.Members, Members: gi.Members,
Policy: policy, Policy: policy,
UpdatedAt: gi.UpdatedAt,
}, nil }, nil
} }
@ -1082,6 +1087,33 @@ func (store *IAMStoreSys) ListPolicies(ctx context.Context, bucketName string) (
return ret, nil return ret, nil
} }
// ListPolicyDocs - fetches all policy docs from storage and updates cache as well.
// If bucketName is non-empty, returns policy docs matching the bucket.
func (store *IAMStoreSys) ListPolicyDocs(ctx context.Context, bucketName string) (map[string]PolicyDoc, error) {
cache := store.lock()
defer store.unlock()
m := map[string]PolicyDoc{}
err := store.loadPolicyDocs(ctx, m)
if err != nil {
return nil, err
}
// Sets default canned policies
setDefaultCannedPolicies(m)
cache.iamPolicyDocsMap = m
ret := map[string]PolicyDoc{}
for k, v := range m {
if bucketName == "" || v.Policy.MatchResource(bucketName) {
ret[k] = v
}
}
return ret, nil
}
// helper function - does not take locks. // helper function - does not take locks.
func filterPolicies(cache *iamCache, policyName string, bucketName string) (string, iampolicy.Policy) { func filterPolicies(cache *iamCache, policyName string, bucketName string) (string, iampolicy.Policy) {
var policies []string var policies []string
@ -1177,7 +1209,8 @@ func (store *IAMStoreSys) GetUsers() map[string]madmin.UserInfo {
} }
return madmin.AccountDisabled return madmin.AccountDisabled
}(), }(),
MemberOf: cache.iamUserGroupMemberships[k].ToSlice(), MemberOf: cache.iamUserGroupMemberships[k].ToSlice(),
UpdatedAt: cache.iamUserPolicyMap[k].UpdatedAt,
} }
} }
@ -1222,6 +1255,7 @@ func (store *IAMStoreSys) GetUserInfo(name string) (u madmin.UserInfo, err error
return madmin.UserInfo{ return madmin.UserInfo{
PolicyName: mappedPolicy.Policies, PolicyName: mappedPolicy.Policies,
MemberOf: groups, MemberOf: groups,
UpdatedAt: mappedPolicy.UpdatedAt,
}, nil }, nil
} }
@ -1242,7 +1276,8 @@ func (store *IAMStoreSys) GetUserInfo(name string) (u madmin.UserInfo, err error
} }
return madmin.AccountDisabled return madmin.AccountDisabled
}(), }(),
MemberOf: cache.iamUserGroupMemberships[name].ToSlice(), MemberOf: cache.iamUserGroupMemberships[name].ToSlice(),
UpdatedAt: cache.iamUserPolicyMap[name].UpdatedAt,
}, nil }, nil
} }

View File

@ -546,6 +546,20 @@ func (sys *IAMSys) ListPolicies(ctx context.Context, bucketName string) (map[str
} }
} }
// ListPolicyDocs - lists all canned policy docs.
func (sys *IAMSys) ListPolicyDocs(ctx context.Context, bucketName string) (map[string]PolicyDoc, error) {
if !sys.Initialized() {
return nil, errServerNotInitialized
}
select {
case <-sys.configLoaded:
return sys.store.ListPolicyDocs(ctx, bucketName)
case <-ctx.Done():
return nil, ctx.Err()
}
}
// SetPolicy - sets a new named policy. // SetPolicy - sets a new named policy.
func (sys *IAMSys) SetPolicy(ctx context.Context, policyName string, p iampolicy.Policy) error { func (sys *IAMSys) SetPolicy(ctx context.Context, policyName string, p iampolicy.Policy) error {
if !sys.Initialized() { if !sys.Initialized() {

File diff suppressed because it is too large Load Diff

View File

@ -428,7 +428,7 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
} }
// Check if the current bucket has replication configuration // Check if the current bucket has replication configuration
if rcfg, err := globalBucketMetadataSys.GetReplicationConfig(ctx, cache.Info.Name); err == nil { if rcfg, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, cache.Info.Name); err == nil {
if rcfg.HasActiveRules("", true) { if rcfg.HasActiveRules("", true) {
tgts, err := globalBucketTargetSys.ListBucketTargets(ctx, cache.Info.Name) tgts, err := globalBucketTargetSys.ListBucketTargets(ctx, cache.Info.Name)
if err == nil { if err == nil {

2
go.mod
View File

@ -51,7 +51,7 @@ require (
github.com/minio/dperf v0.3.6 github.com/minio/dperf v0.3.6
github.com/minio/highwayhash v1.0.2 github.com/minio/highwayhash v1.0.2
github.com/minio/kes v0.19.0 github.com/minio/kes v0.19.0
github.com/minio/madmin-go v1.3.11 github.com/minio/madmin-go v1.3.12
github.com/minio/minio-go/v7 v7.0.24 github.com/minio/minio-go/v7 v7.0.24
github.com/minio/pkg v1.1.20 github.com/minio/pkg v1.1.20
github.com/minio/selfupdate v0.4.0 github.com/minio/selfupdate v0.4.0

19
go.sum
View File

@ -116,7 +116,6 @@ github.com/apache/thrift v0.15.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
github.com/armon/go-metrics v0.3.0/go.mod h1:zXjbSimjXTd7vOpY8B0/2LpvNvDoXBuplAD+gJD3GYs= github.com/armon/go-metrics v0.3.0/go.mod h1:zXjbSimjXTd7vOpY8B0/2LpvNvDoXBuplAD+gJD3GYs=
@ -201,7 +200,6 @@ github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kw
github.com/coredns/coredns v1.9.0 h1:M1EF1uups4CYcQGb1z8A97mfoq4BYCw3+xCYcJkOSDc= github.com/coredns/coredns v1.9.0 h1:M1EF1uups4CYcQGb1z8A97mfoq4BYCw3+xCYcJkOSDc=
github.com/coredns/coredns v1.9.0/go.mod h1:czzy6Ofs15Mzn1PXpWoplBCZxoWdGoQUInL9uPSiYME= github.com/coredns/coredns v1.9.0/go.mod h1:czzy6Ofs15Mzn1PXpWoplBCZxoWdGoQUInL9uPSiYME=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom8DBE9so9EBsM= github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom8DBE9so9EBsM=
github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
@ -281,7 +279,6 @@ github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09
github.com/fraugster/parquet-go v0.10.0 h1:whX91AO3dkkOnbH9MqD53DZ3rISw+Tnnj5yiqXjSv9Q= github.com/fraugster/parquet-go v0.10.0 h1:whX91AO3dkkOnbH9MqD53DZ3rISw+Tnnj5yiqXjSv9Q=
github.com/fraugster/parquet-go v0.10.0/go.mod h1:asQOKX0K/j+F3Xyj87kw7gKrU3yXo9M2hb8STSQKIIw= github.com/fraugster/parquet-go v0.10.0/go.mod h1:asQOKX0K/j+F3Xyj87kw7gKrU3yXo9M2hb8STSQKIIw=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko= github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko=
github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
@ -710,8 +707,8 @@ github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLT
github.com/minio/kes v0.19.0 h1:rKzkDXT4ay7FBW34KgXK+y85bie4x4Oiq29ONRuMzh0= github.com/minio/kes v0.19.0 h1:rKzkDXT4ay7FBW34KgXK+y85bie4x4Oiq29ONRuMzh0=
github.com/minio/kes v0.19.0/go.mod h1:e9YGKbwFCV7LbqNPMfZBazfNUsFGJ5LG4plSeWL8mmg= github.com/minio/kes v0.19.0/go.mod h1:e9YGKbwFCV7LbqNPMfZBazfNUsFGJ5LG4plSeWL8mmg=
github.com/minio/madmin-go v1.3.5/go.mod h1:vGKGboQgGIWx4DuDUaXixjlIEZOCIp6ivJkQoiVaACc= github.com/minio/madmin-go v1.3.5/go.mod h1:vGKGboQgGIWx4DuDUaXixjlIEZOCIp6ivJkQoiVaACc=
github.com/minio/madmin-go v1.3.11 h1:Cj02kzG2SD1pnZW2n1joe00yqb6NFE40Jt2gp+5mWFQ= github.com/minio/madmin-go v1.3.12 h1:7SmK/KtT7+d3hn3VcYBqI/c4yETfXV9gRT1j+g/U1jE=
github.com/minio/madmin-go v1.3.11/go.mod h1:ez87VmMtsxP7DRxjKJKD4RDNW+nhO2QF9KSzwxBDQ98= github.com/minio/madmin-go v1.3.12/go.mod h1:ez87VmMtsxP7DRxjKJKD4RDNW+nhO2QF9KSzwxBDQ98=
github.com/minio/mc v0.0.0-20220415073849-792c6c1475bd h1:kAqiTMvTBGQq57Mr+lHIh0tZdMP80dbDDFMspANC6ic= github.com/minio/mc v0.0.0-20220415073849-792c6c1475bd h1:kAqiTMvTBGQq57Mr+lHIh0tZdMP80dbDDFMspANC6ic=
github.com/minio/mc v0.0.0-20220415073849-792c6c1475bd/go.mod h1:Z8lzJkjqgORAzT+ux12BZrov2HdPresY/gD0w+SCsIk= github.com/minio/mc v0.0.0-20220415073849-792c6c1475bd/go.mod h1:Z8lzJkjqgORAzT+ux12BZrov2HdPresY/gD0w+SCsIk=
github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
@ -746,7 +743,6 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
@ -938,27 +934,16 @@ github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
github.com/streadway/amqp v1.0.0 h1:kuuDrUJFZL1QYL9hUNuCxNObNzB0bV/ZG5jV3RWAQgo= github.com/streadway/amqp v1.0.0 h1:kuuDrUJFZL1QYL9hUNuCxNObNzB0bV/ZG5jV3RWAQgo=