MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 12:52:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

Support persistent queue store for loggers (#17121)

Browse Source
This commit is contained in:
Praveen raj Mani
2023-05-09 09:50:31 +05:30
committed by GitHub
parent 42fb3cd95e
commit 57acacd5a7
25 changed files with 516 additions and 300 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
internal/event/target/amqp.go
View File

@@ -30,6 +30,7 @@ import (
"github.com/minio/minio/internal/event"
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
xnet "github.com/minio/pkg/net"
"github.com/rabbitmq/amqp091-go"
@@ -112,7 +113,7 @@ func (a *AMQPArgs) Validate() error {
// AMQPTarget - AMQP target
type AMQPTarget struct {
lazyInit lazyInit
initOnce once.Init
id event.TargetID
args AMQPArgs
@@ -289,8 +290,8 @@ func (target *AMQPTarget) Save(eventData event.Event) error {
return target.send(eventData, ch, confirms)
}
// Send - sends event to AMQP091.
func (target *AMQPTarget) Send(eventKey string) error {
// SendFromStore - reads an event from store and sends it to AMQP091.
func (target *AMQPTarget) SendFromStore(eventKey string) error {
if err := target.init(); err != nil {
return err
}
@@ -329,7 +330,7 @@ func (target *AMQPTarget) Close() error {
}
func (target *AMQPTarget) init() error {
return target.lazyInit.Do(target.initAMQP)
return target.initOnce.Do(target.initAMQP)
}
func (target *AMQPTarget) initAMQP() error {

9
internal/event/target/elasticsearch.go
View File

@@ -36,6 +36,7 @@ import (
"github.com/minio/highwayhash"
"github.com/minio/minio/internal/event"
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
xnet "github.com/minio/pkg/net"
"github.com/pkg/errors"
@@ -153,7 +154,7 @@ func (a ElasticsearchArgs) Validate() error {
// ElasticsearchTarget - Elasticsearch target.
type ElasticsearchTarget struct {
lazyInit lazyInit
initOnce once.Init
id event.TargetID
args ElasticsearchArgs
@@ -263,8 +264,8 @@ func (target *ElasticsearchTarget) send(eventData event.Event) error {
return nil
}
// Send - reads an event from store and sends it to Elasticsearch.
func (target *ElasticsearchTarget) Send(eventKey string) error {
// SendFromStore - reads an event from store and sends it to Elasticsearch.
func (target *ElasticsearchTarget) SendFromStore(eventKey string) error {
if err := target.init(); err != nil {
return err
}
@@ -344,7 +345,7 @@ func (target *ElasticsearchTarget) checkAndInitClient(ctx context.Context) error
}
func (target *ElasticsearchTarget) init() error {
return target.lazyInit.Do(target.initElasticsearch)
return target.initOnce.Do(target.initElasticsearch)
}
func (target *ElasticsearchTarget) initElasticsearch() error {

9
internal/event/target/kafka.go
View File

@@ -32,6 +32,7 @@ import (
"github.com/minio/minio/internal/event"
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
xnet "github.com/minio/pkg/net"
@@ -125,7 +126,7 @@ func (k KafkaArgs) Validate() error {
// KafkaTarget - Kafka target.
type KafkaTarget struct {
lazyInit lazyInit
initOnce once.Init
id event.TargetID
args KafkaArgs
@@ -208,8 +209,8 @@ func (target *KafkaTarget) send(eventData event.Event) error {
return err
}
// Send - reads an event from store and sends it to Kafka.
func (target *KafkaTarget) Send(eventKey string) error {
// SendFromStore - reads an event from store and sends it to Kafka.
func (target *KafkaTarget) SendFromStore(eventKey string) error {
if err := target.init(); err != nil {
return err
}
@@ -278,7 +279,7 @@ func (k KafkaArgs) pingBrokers() bool {
}
func (target *KafkaTarget) init() error {
return target.lazyInit.Do(target.initKafka)
return target.initOnce.Do(target.initKafka)
}
func (target *KafkaTarget) initKafka() error {

51
internal/event/target/lazyinit.go
View File

@@ -1,51 +0,0 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package target
import (
"sync"
"sync/atomic"
)
// Inspired from Golang sync.Once but it is only marked
// initialized when the provided function returns nil.
type lazyInit struct {
done uint32
m sync.Mutex
}
func (l *lazyInit) Do(f func() error) error {
if atomic.LoadUint32(&l.done) == 0 {
return l.doSlow(f)
}
return nil
}
func (l *lazyInit) doSlow(f func() error) error {
l.m.Lock()
defer l.m.Unlock()
if atomic.LoadUint32(&l.done) == 0 {
if err := f(); err != nil {
return err
}
// Mark as done only when f() is successful
atomic.StoreUint32(&l.done, 1)
}
return nil
}

9
internal/event/target/mqtt.go
View File

@@ -31,6 +31,7 @@ import (
mqtt "github.com/eclipse/paho.mqtt.golang"
"github.com/minio/minio/internal/event"
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
xnet "github.com/minio/pkg/net"
)
@@ -107,7 +108,7 @@ func (m MQTTArgs) Validate() error {
// MQTTTarget - MQTT target.
type MQTTTarget struct {
lazyInit lazyInit
initOnce once.Init
id event.TargetID
args MQTTArgs
@@ -167,8 +168,8 @@ func (target *MQTTTarget) send(eventData event.Event) error {
return token.Error()
}
// Send - reads an event from store and sends it to MQTT.
func (target *MQTTTarget) Send(eventKey string) error {
// SendFromStore - reads an event from store and sends it to MQTT.
func (target *MQTTTarget) SendFromStore(eventKey string) error {
if err := target.init(); err != nil {
return err
}
@@ -224,7 +225,7 @@ func (target *MQTTTarget) Close() error {
}
func (target *MQTTTarget) init() error {
return target.lazyInit.Do(target.initMQTT)
return target.initOnce.Do(target.initMQTT)
}
func (target *MQTTTarget) initMQTT() error {

9
internal/event/target/mysql.go
View File

@@ -33,6 +33,7 @@ import (
"github.com/go-sql-driver/mysql"
"github.com/minio/minio/internal/event"
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
xnet "github.com/minio/pkg/net"
)
@@ -146,7 +147,7 @@ func (m MySQLArgs) Validate() error {
// MySQLTarget - MySQL target.
type MySQLTarget struct {
lazyInit lazyInit
initOnce once.Init
id event.TargetID
args MySQLArgs
@@ -252,8 +253,8 @@ func (target *MySQLTarget) send(eventData event.Event) error {
return nil
}
// Send - reads an event from store and sends it to MySQL.
func (target *MySQLTarget) Send(eventKey string) error {
// SendFromStore - reads an event from store and sends it to MySQL.
func (target *MySQLTarget) SendFromStore(eventKey string) error {
if err := target.init(); err != nil {
return err
}
@@ -349,7 +350,7 @@ func (target *MySQLTarget) executeStmts() error {
}
func (target *MySQLTarget) init() error {
return target.lazyInit.Do(target.initMySQL)
return target.initOnce.Do(target.initMySQL)
}
func (target *MySQLTarget) initMySQL() error {

9
internal/event/target/nats.go
View File

@@ -31,6 +31,7 @@ import (
"github.com/google/uuid"
"github.com/minio/minio/internal/event"
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
xnet "github.com/minio/pkg/net"
"github.com/nats-io/nats.go"
@@ -216,7 +217,7 @@ func (n NATSArgs) connectStan() (stan.Conn, error) {
// NATSTarget - NATS target.
type NATSTarget struct {
lazyInit lazyInit
initOnce once.Init
id event.TargetID
args NATSArgs
@@ -333,8 +334,8 @@ func (target *NATSTarget) send(eventData event.Event) error {
return err
}
// Send - sends event to Nats.
func (target *NATSTarget) Send(eventKey string) error {
// SendFromStore - reads an event from store and sends it to Nats.
func (target *NATSTarget) SendFromStore(eventKey string) error {
if err := target.init(); err != nil {
return err
}
@@ -380,7 +381,7 @@ func (target *NATSTarget) Close() (err error) {
}
func (target *NATSTarget) init() error {
return target.lazyInit.Do(target.initNATS)
return target.initOnce.Do(target.initNATS)
}
func (target *NATSTarget) initNATS() error {

9
internal/event/target/nsq.go
View File

@@ -31,6 +31,7 @@ import (
"github.com/minio/minio/internal/event"
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
xnet "github.com/minio/pkg/net"
)
@@ -90,7 +91,7 @@ func (n NSQArgs) Validate() error {
// NSQTarget - NSQ target.
type NSQTarget struct {
lazyInit lazyInit
initOnce once.Init
id event.TargetID
args NSQArgs
@@ -176,8 +177,8 @@ func (target *NSQTarget) send(eventData event.Event) error {
return target.producer.Publish(target.args.Topic, data)
}
// Send - reads an event from store and sends it to NSQ.
func (target *NSQTarget) Send(eventKey string) error {
// SendFromStore - reads an event from store and sends it to NSQ.
func (target *NSQTarget) SendFromStore(eventKey string) error {
if err := target.init(); err != nil {
return err
}
@@ -216,7 +217,7 @@ func (target *NSQTarget) Close() (err error) {
}
func (target *NSQTarget) init() error {
return target.lazyInit.Do(target.initNSQ)
return target.initOnce.Do(target.initNSQ)
}
func (target *NSQTarget) initNSQ() error {

9
internal/event/target/postgresql.go
View File

@@ -34,6 +34,7 @@ import (
"github.com/minio/minio/internal/event"
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
xnet "github.com/minio/pkg/net"
)
@@ -138,7 +139,7 @@ func (p PostgreSQLArgs) Validate() error {
// PostgreSQLTarget - PostgreSQL target.
type PostgreSQLTarget struct {
lazyInit lazyInit
initOnce once.Init
id event.TargetID
args PostgreSQLArgs
@@ -249,8 +250,8 @@ func (target *PostgreSQLTarget) send(eventData event.Event) error {
return nil
}
// Send - reads an event from store and sends it to PostgreSQL.
func (target *PostgreSQLTarget) Send(eventKey string) error {
// SendFromStore - reads an event from store and sends it to PostgreSQL.
func (target *PostgreSQLTarget) SendFromStore(eventKey string) error {
if err := target.init(); err != nil {
return err
}
@@ -345,7 +346,7 @@ func (target *PostgreSQLTarget) executeStmts() error {
}
func (target *PostgreSQLTarget) init() error {
return target.lazyInit.Do(target.initPostgreSQL)
return target.initOnce.Do(target.initPostgreSQL)
}
func (target *PostgreSQLTarget) initPostgreSQL() error {

9
internal/event/target/redis.go
View File

@@ -31,6 +31,7 @@ import (
"github.com/gomodule/redigo/redis"
"github.com/minio/minio/internal/event"
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
xnet "github.com/minio/pkg/net"
)
@@ -118,7 +119,7 @@ func (r RedisArgs) validateFormat(c redis.Conn) error {
// RedisTarget - Redis target.
type RedisTarget struct {
lazyInit lazyInit
initOnce once.Init
id event.TargetID
args RedisArgs
@@ -221,8 +222,8 @@ func (target *RedisTarget) send(eventData event.Event) error {
return nil
}
// Send - reads an event from store and sends it to redis.
func (target *RedisTarget) Send(eventKey string) error {
// SendFromStore - reads an event from store and sends it to redis.
func (target *RedisTarget) SendFromStore(eventKey string) error {
if err := target.init(); err != nil {
return err
}
@@ -276,7 +277,7 @@ func (target *RedisTarget) Close() error {
}
func (target *RedisTarget) init() error {
return target.lazyInit.Do(target.initRedis)
return target.initOnce.Do(target.initRedis)
}
func (target *RedisTarget) initRedis() error {

9
internal/event/target/webhook.go
View File

@@ -35,6 +35,7 @@ import (
"github.com/minio/minio/internal/event"
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
"github.com/minio/pkg/certs"
xnet "github.com/minio/pkg/net"
@@ -91,7 +92,7 @@ func (w WebhookArgs) Validate() error {
// WebhookTarget - Webhook target.
type WebhookTarget struct {
lazyInit lazyInit
initOnce once.Init
id event.TargetID
args WebhookArgs
@@ -222,8 +223,8 @@ func (target *WebhookTarget) send(eventData event.Event) error {
return nil
}
// Send - reads an event from store and sends it to webhook.
func (target *WebhookTarget) Send(eventKey string) error {
// SendFromStore - reads an event from store and sends it to webhook.
func (target *WebhookTarget) SendFromStore(eventKey string) error {
if err := target.init(); err != nil {
return err
}
@@ -256,7 +257,7 @@ func (target *WebhookTarget) Close() error {
}
func (target *WebhookTarget) init() error {
return target.lazyInit.Do(target.initWebhook)
return target.initOnce.Do(target.initWebhook)
}
// Only called from init()

2
internal/event/targetlist.go
View File

@@ -34,7 +34,7 @@ type Target interface {
ID() TargetID
IsActive() (bool, error)
Save(Event) error
Send(string) error
SendFromStore(string) error
Close() error
Store() TargetStore
}

4
internal/event/targetlist_test.go
View File

@@ -60,8 +60,8 @@ func (target ExampleTarget) send(eventData Event) error {
return nil
}
// Send - interface compatible method does no-op.
func (target ExampleTarget) Send(eventKey string) error {
// SendFromStore - interface compatible method does no-op.
func (target *ExampleTarget) SendFromStore(eventKey string) error {
return nil
}