mirror of
https://github.com/minio/minio.git
synced 2025-11-21 10:16:03 -05:00
Reuse types for key-value, notification and retry (#17936)
This commit is contained in:
Krishnan Parthasarathi
committed by
GitHub
GitHub
parent
1067dd3011
commit
6a67c277eb
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) 2015-2022 MinIO, Inc.
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
@@ -42,7 +42,6 @@ import (
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio-go/v7/pkg/encrypt"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
@@ -51,222 +50,10 @@ import (
|
||||
"github.com/minio/pkg/console"
|
||||
"github.com/minio/pkg/env"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
"github.com/minio/pkg/wildcard"
|
||||
"github.com/minio/pkg/workers"
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
// replicate:
|
||||
// # source of the objects to be replicated
|
||||
// source:
|
||||
// type: "minio"
|
||||
// bucket: "testbucket"
|
||||
// prefix: "spark/"
|
||||
//
|
||||
// # optional flags based filtering criteria
|
||||
// # for source objects
|
||||
// flags:
|
||||
// filter:
|
||||
// newerThan: "7d"
|
||||
// olderThan: "7d"
|
||||
// createdAfter: "date"
|
||||
// createdBefore: "date"
|
||||
// tags:
|
||||
// - key: "name"
|
||||
// value: "value*"
|
||||
// metadata:
|
||||
// - key: "content-type"
|
||||
// value: "image/*"
|
||||
// notify:
|
||||
// endpoint: "https://splunk-hec.dev.com"
|
||||
// token: "Splunk ..." # e.g. "Bearer token"
|
||||
//
|
||||
// # target where the objects must be replicated
|
||||
// target:
|
||||
// type: "minio"
|
||||
// bucket: "testbucket1"
|
||||
// endpoint: "https://play.min.io"
|
||||
// path: "on"
|
||||
// credentials:
|
||||
// accessKey: "minioadmin"
|
||||
// secretKey: "minioadmin"
|
||||
// sessionToken: ""
|
||||
|
||||
// BatchJobReplicateKV is a datatype that holds key and values for filtering of objects
|
||||
// used by metadata filter as well as tags based filtering.
|
||||
type BatchJobReplicateKV struct {
|
||||
Key string `yaml:"key" json:"key"`
|
||||
Value string `yaml:"value" json:"value"`
|
||||
}
|
||||
|
||||
// Validate returns an error if key is empty
|
||||
func (kv BatchJobReplicateKV) Validate() error {
|
||||
if kv.Key == "" {
|
||||
return errInvalidArgument
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Empty indicates if kv is not set
|
||||
func (kv BatchJobReplicateKV) Empty() bool {
|
||||
return kv.Key == "" && kv.Value == ""
|
||||
}
|
||||
|
||||
// Match matches input kv with kv, value will be wildcard matched depending on the user input
|
||||
func (kv BatchJobReplicateKV) Match(ikv BatchJobReplicateKV) bool {
|
||||
if kv.Empty() {
|
||||
return true
|
||||
}
|
||||
if strings.EqualFold(kv.Key, ikv.Key) {
|
||||
return wildcard.Match(kv.Value, ikv.Value)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// BatchReplicateRetry datatype represents total retry attempts and delay between each retries.
|
||||
type BatchReplicateRetry struct {
|
||||
Attempts int `yaml:"attempts" json:"attempts"` // number of retry attempts
|
||||
Delay time.Duration `yaml:"delay" json:"delay"` // delay between each retries
|
||||
}
|
||||
|
||||
// Validate validates input replicate retries.
|
||||
func (r BatchReplicateRetry) Validate() error {
|
||||
if r.Attempts < 0 {
|
||||
return errInvalidArgument
|
||||
}
|
||||
|
||||
if r.Delay < 0 {
|
||||
return errInvalidArgument
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// BatchReplicateFilter holds all the filters currently supported for batch replication
|
||||
type BatchReplicateFilter struct {
|
||||
NewerThan time.Duration `yaml:"newerThan,omitempty" json:"newerThan"`
|
||||
OlderThan time.Duration `yaml:"olderThan,omitempty" json:"olderThan"`
|
||||
CreatedAfter time.Time `yaml:"createdAfter,omitempty" json:"createdAfter"`
|
||||
CreatedBefore time.Time `yaml:"createdBefore,omitempty" json:"createdBefore"`
|
||||
Tags []BatchJobReplicateKV `yaml:"tags,omitempty" json:"tags"`
|
||||
Metadata []BatchJobReplicateKV `yaml:"metadata,omitempty" json:"metadata"`
|
||||
}
|
||||
|
||||
// BatchReplicateNotification success or failure notification endpoint for each job attempts
|
||||
type BatchReplicateNotification struct {
|
||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||
Token string `yaml:"token" json:"token"`
|
||||
}
|
||||
|
||||
// BatchJobReplicateFlags various configurations for replication job definition currently includes
|
||||
// - filter
|
||||
// - notify
|
||||
// - retry
|
||||
type BatchJobReplicateFlags struct {
|
||||
Filter BatchReplicateFilter `yaml:"filter" json:"filter"`
|
||||
Notify BatchReplicateNotification `yaml:"notify" json:"notify"`
|
||||
Retry BatchReplicateRetry `yaml:"retry" json:"retry"`
|
||||
}
|
||||
|
||||
// BatchJobReplicateResourceType defines the type of batch jobs
|
||||
type BatchJobReplicateResourceType string
|
||||
|
||||
// Validate validates if the replicate resource type is recognized and supported
|
||||
func (t BatchJobReplicateResourceType) Validate() error {
|
||||
switch t {
|
||||
case BatchJobReplicateResourceMinIO:
|
||||
case BatchJobReplicateResourceS3:
|
||||
default:
|
||||
return errInvalidArgument
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t BatchJobReplicateResourceType) isMinio() bool {
|
||||
return t == BatchJobReplicateResourceMinIO
|
||||
}
|
||||
|
||||
// Different types of batch jobs..
|
||||
const (
|
||||
BatchJobReplicateResourceMinIO BatchJobReplicateResourceType = "minio"
|
||||
BatchJobReplicateResourceS3 BatchJobReplicateResourceType = "s3"
|
||||
|
||||
// add future targets
|
||||
)
|
||||
|
||||
// BatchJobReplicateCredentials access credentials for batch replication it may
|
||||
// be either for target or source.
|
||||
type BatchJobReplicateCredentials struct {
|
||||
AccessKey string `xml:"AccessKeyId" json:"accessKey,omitempty" yaml:"accessKey"`
|
||||
SecretKey string `xml:"SecretAccessKey" json:"secretKey,omitempty" yaml:"secretKey"`
|
||||
SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty" yaml:"sessionToken"`
|
||||
}
|
||||
|
||||
// Empty indicates if credentials are not set
|
||||
func (c BatchJobReplicateCredentials) Empty() bool {
|
||||
return c.AccessKey == "" && c.SecretKey == "" && c.SessionToken == ""
|
||||
}
|
||||
|
||||
// Validate validates if credentials are valid
|
||||
func (c BatchJobReplicateCredentials) Validate() error {
|
||||
if !auth.IsAccessKeyValid(c.AccessKey) || !auth.IsSecretKeyValid(c.SecretKey) {
|
||||
return errInvalidArgument
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BatchJobReplicateTarget describes target element of the replication job that receives
|
||||
// the filtered data from source
|
||||
type BatchJobReplicateTarget struct {
|
||||
Type BatchJobReplicateResourceType `yaml:"type" json:"type"`
|
||||
Bucket string `yaml:"bucket" json:"bucket"`
|
||||
Prefix string `yaml:"prefix" json:"prefix"`
|
||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||
Path string `yaml:"path" json:"path"`
|
||||
Creds BatchJobReplicateCredentials `yaml:"credentials" json:"credentials"`
|
||||
}
|
||||
|
||||
// ValidPath returns true if path is valid
|
||||
func (t BatchJobReplicateTarget) ValidPath() bool {
|
||||
return t.Path == "on" || t.Path == "off" || t.Path == "auto" || t.Path == ""
|
||||
}
|
||||
|
||||
// BatchJobReplicateSource describes source element of the replication job that is
|
||||
// the source of the data for the target
|
||||
type BatchJobReplicateSource struct {
|
||||
Type BatchJobReplicateResourceType `yaml:"type" json:"type"`
|
||||
Bucket string `yaml:"bucket" json:"bucket"`
|
||||
Prefix string `yaml:"prefix" json:"prefix"`
|
||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||
Path string `yaml:"path" json:"path"`
|
||||
Creds BatchJobReplicateCredentials `yaml:"credentials" json:"credentials"`
|
||||
}
|
||||
|
||||
// ValidPath returns true if path is valid
|
||||
func (s BatchJobReplicateSource) ValidPath() bool {
|
||||
switch s.Path {
|
||||
case "on", "off", "auto", "":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// BatchJobReplicateV1 v1 of batch job replication
|
||||
type BatchJobReplicateV1 struct {
|
||||
APIVersion string `yaml:"apiVersion" json:"apiVersion"`
|
||||
Flags BatchJobReplicateFlags `yaml:"flags" json:"flags"`
|
||||
Target BatchJobReplicateTarget `yaml:"target" json:"target"`
|
||||
Source BatchJobReplicateSource `yaml:"source" json:"source"`
|
||||
|
||||
clnt *miniogo.Core `msg:"-"`
|
||||
}
|
||||
|
||||
// RemoteToLocal returns true if source is remote and target is local
|
||||
func (r BatchJobReplicateV1) RemoteToLocal() bool {
|
||||
return !r.Source.Creds.Empty()
|
||||
}
|
||||
|
||||
// BatchJobRequest this is an internal data structure not for external consumption.
|
||||
type BatchJobRequest struct {
|
||||
ID string `yaml:"-" json:"name"`
|
||||
@@ -516,7 +303,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
|
||||
}
|
||||
for _, kv := range r.Flags.Filter.Tags {
|
||||
for t, v := range tagMap {
|
||||
if kv.Match(BatchJobReplicateKV{Key: t, Value: v}) {
|
||||
if kv.Match(BatchJobKV{Key: t, Value: v}) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -533,7 +320,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
|
||||
continue
|
||||
}
|
||||
// We only need to match x-amz-meta or standardHeaders
|
||||
if kv.Match(BatchJobReplicateKV{Key: k, Value: v}) {
|
||||
if kv.Match(BatchJobKV{Key: k, Value: v}) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -1070,7 +857,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
|
||||
|
||||
for _, kv := range r.Flags.Filter.Tags {
|
||||
for t, v := range tagMap {
|
||||
if kv.Match(BatchJobReplicateKV{Key: t, Value: v}) {
|
||||
if kv.Match(BatchJobKV{Key: t, Value: v}) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -1087,7 +874,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
|
||||
continue
|
||||
}
|
||||
// We only need to match x-amz-meta or standardHeaders
|
||||
if kv.Match(BatchJobReplicateKV{Key: k, Value: v}) {
|
||||
if kv.Match(BatchJobKV{Key: k, Value: v}) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user