Kafka notify: support batched commits for queue store (#20377)

The items will be saved per target batch and will
be committed to the queue store when the batch is full

Also, periodically commit the batched items to the queue store
based on configured commit_timeout; default is 30s;

Bonus: compress queue store multi writes

internal/event/target/amqp.go

@@ -276,7 +276,8 @@ func (target *AMQPTarget) send(eventData event.Event, ch *amqp091.Channel, confi
 // Save - saves the events to the store which will be replayed when the amqp connection is active.
 func (target *AMQPTarget) Save(eventData event.Event) error {
 	if target.store != nil {
-		return target.store.Put(eventData)
+		_, err := target.store.Put(eventData)
+		return err
 	}
 	if err := target.init(); err != nil {
 		return err
@@ -302,7 +303,7 @@ func (target *AMQPTarget) SendFromStore(key store.Key) error {
 	}
 	defer ch.Close()
 
-	eventData, eErr := target.store.Get(key.Name)
+	eventData, eErr := target.store.Get(key)
 	if eErr != nil {
 		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
 		// Such events will not exist and wouldve been already been sent successfully.
@@ -317,7 +318,7 @@ func (target *AMQPTarget) SendFromStore(key store.Key) error {
 	}
 
 	// Delete the event from store.
-	return target.store.Del(key.Name)
+	return target.store.Del(key)
 }
 
 // Close - does nothing and available for interface compatibility.

internal/event/target/elasticsearch.go

@@ -202,7 +202,8 @@ func (target *ElasticsearchTarget) isActive() (bool, error) {
 // Save - saves the events to the store if queuestore is configured, which will be replayed when the elasticsearch connection is active.
 func (target *ElasticsearchTarget) Save(eventData event.Event) error {
 	if target.store != nil {
-		return target.store.Put(eventData)
+		_, err := target.store.Put(eventData)
+		return err
 	}
 	if err := target.init(); err != nil {
 		return err
@@ -278,7 +279,7 @@ func (target *ElasticsearchTarget) SendFromStore(key store.Key) error {
 		return err
 	}
 
-	eventData, eErr := target.store.Get(key.Name)
+	eventData, eErr := target.store.Get(key)
 	if eErr != nil {
 		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
 		// Such events will not exist and wouldve been already been sent successfully.
@@ -296,7 +297,7 @@ func (target *ElasticsearchTarget) SendFromStore(key store.Key) error {
 	}
 
 	// Delete the event from store.
-	return target.store.Del(key.Name)
+	return target.store.Del(key)
 }
 
 // Close - does nothing and available for interface compatibility.

internal/event/target/kafka.go

@@ -43,23 +43,24 @@ import (
 
 // Kafka input constants
 const (
-	KafkaBrokers          = "brokers"
-	KafkaTopic            = "topic"
-	KafkaQueueDir         = "queue_dir"
-	KafkaQueueLimit       = "queue_limit"
-	KafkaTLS              = "tls"
-	KafkaTLSSkipVerify    = "tls_skip_verify"
-	KafkaTLSClientAuth    = "tls_client_auth"
-	KafkaSASL             = "sasl"
-	KafkaSASLUsername     = "sasl_username"
-	KafkaSASLPassword     = "sasl_password"
-	KafkaSASLMechanism    = "sasl_mechanism"
-	KafkaClientTLSCert    = "client_tls_cert"
-	KafkaClientTLSKey     = "client_tls_key"
-	KafkaVersion          = "version"
-	KafkaBatchSize        = "batch_size"
-	KafkaCompressionCodec = "compression_codec"
-	KafkaCompressionLevel = "compression_level"
+	KafkaBrokers            = "brokers"
+	KafkaTopic              = "topic"
+	KafkaQueueDir           = "queue_dir"
+	KafkaQueueLimit         = "queue_limit"
+	KafkaTLS                = "tls"
+	KafkaTLSSkipVerify      = "tls_skip_verify"
+	KafkaTLSClientAuth      = "tls_client_auth"
+	KafkaSASL               = "sasl"
+	KafkaSASLUsername       = "sasl_username"
+	KafkaSASLPassword       = "sasl_password"
+	KafkaSASLMechanism      = "sasl_mechanism"
+	KafkaClientTLSCert      = "client_tls_cert"
+	KafkaClientTLSKey       = "client_tls_key"
+	KafkaVersion            = "version"
+	KafkaBatchSize          = "batch_size"
+	KafkaBatchCommitTimeout = "batch_commit_timeout"
+	KafkaCompressionCodec   = "compression_codec"
+	KafkaCompressionLevel   = "compression_level"
 
 	EnvKafkaEnable                   = "MINIO_NOTIFY_KAFKA_ENABLE"
 	EnvKafkaBrokers                  = "MINIO_NOTIFY_KAFKA_BROKERS"
@@ -77,6 +78,7 @@ const (
 	EnvKafkaClientTLSKey             = "MINIO_NOTIFY_KAFKA_CLIENT_TLS_KEY"
 	EnvKafkaVersion                  = "MINIO_NOTIFY_KAFKA_VERSION"
 	EnvKafkaBatchSize                = "MINIO_NOTIFY_KAFKA_BATCH_SIZE"
+	EnvKafkaBatchCommitTimeout       = "MINIO_NOTIFY_KAFKA_BATCH_COMMIT_TIMEOUT"
 	EnvKafkaProducerCompressionCodec = "MINIO_NOTIFY_KAFKA_PRODUCER_COMPRESSION_CODEC"
 	EnvKafkaProducerCompressionLevel = "MINIO_NOTIFY_KAFKA_PRODUCER_COMPRESSION_LEVEL"
 )
@@ -91,14 +93,15 @@ var codecs = map[string]sarama.CompressionCodec{
 
 // KafkaArgs - Kafka target arguments.
 type KafkaArgs struct {
-	Enable     bool        `json:"enable"`
-	Brokers    []xnet.Host `json:"brokers"`
-	Topic      string      `json:"topic"`
-	QueueDir   string      `json:"queueDir"`
-	QueueLimit uint64      `json:"queueLimit"`
-	Version    string      `json:"version"`
-	BatchSize  uint32      `json:"batchSize"`
-	TLS        struct {
+	Enable             bool          `json:"enable"`
+	Brokers            []xnet.Host   `json:"brokers"`
+	Topic              string        `json:"topic"`
+	QueueDir           string        `json:"queueDir"`
+	QueueLimit         uint64        `json:"queueLimit"`
+	Version            string        `json:"version"`
+	BatchSize          uint32        `json:"batchSize"`
+	BatchCommitTimeout time.Duration `json:"batchCommitTimeout"`
+	TLS                struct {
 		Enable        bool               `json:"enable"`
 		RootCAs       *x509.CertPool     `json:"-"`
 		SkipVerify    bool               `json:"skipVerify"`
@@ -146,6 +149,11 @@ func (k KafkaArgs) Validate() error {
 			return errors.New("batch should be enabled only if queue dir is enabled")
 		}
 	}
+	if k.BatchCommitTimeout > 0 {
+		if k.QueueDir == "" || k.BatchSize <= 1 {
+			return errors.New("batch commit timeout should be set only if queue dir is enabled and batch size > 1")
+		}
+	}
 	return nil
 }
 
@@ -159,7 +167,7 @@ type KafkaTarget struct {
 	producer   sarama.SyncProducer
 	config     *sarama.Config
 	store      store.Store[event.Event]
-	batch      *store.Batch[string, *sarama.ProducerMessage]
+	batch      *store.Batch[event.Event]
 	loggerOnce logger.LogOnce
 	quitCh     chan struct{}
 }
@@ -199,7 +207,11 @@ func (target *KafkaTarget) isActive() (bool, error) {
 // Save - saves the events to the store which will be replayed when the Kafka connection is active.
 func (target *KafkaTarget) Save(eventData event.Event) error {
 	if target.store != nil {
-		return target.store.Put(eventData)
+		if target.batch != nil {
+			return target.batch.Add(eventData)
+		}
+		_, err := target.store.Put(eventData)
+		return err
 	}
 	if err := target.init(); err != nil {
 		return err
@@ -220,80 +232,59 @@ func (target *KafkaTarget) send(eventData event.Event) error {
 	return err
 }
 
-// SendFromStore - reads an event from store and sends it to Kafka.
-func (target *KafkaTarget) SendFromStore(key store.Key) error {
-	if err := target.init(); err != nil {
-		return err
+// sendMultiple sends multiple messages to the kafka.
+func (target *KafkaTarget) sendMultiple(events []event.Event) error {
+	if target.producer == nil {
+		return store.ErrNotConnected
 	}
-
-	// If batch is enabled, the event will be batched in memory
-	// and will be committed once the batch is full.
-	if target.batch != nil {
-		return target.addToBatch(key)
-	}
-
-	eventData, eErr := target.store.Get(key.Name)
-	if eErr != nil {
-		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
-		// Such events will not exist and wouldve been already been sent successfully.
-		if os.IsNotExist(eErr) {
-			return nil
-		}
-		return eErr
-	}
-
-	if err := target.send(eventData); err != nil {
-		if isKafkaConnErr(err) {
-			return store.ErrNotConnected
-		}
-		return err
-	}
-
-	// Delete the event from store.
-	return target.store.Del(key.Name)
-}
-
-func (target *KafkaTarget) addToBatch(key store.Key) error {
-	if target.batch.IsFull() {
-		if err := target.commitBatch(); err != nil {
+	var msgs []*sarama.ProducerMessage
+	for _, event := range events {
+		msg, err := target.toProducerMessage(event)
+		if err != nil {
 			return err
 		}
+		msgs = append(msgs, msg)
 	}
-	if _, ok := target.batch.GetByKey(key.Name); !ok {
-		eventData, err := target.store.Get(key.Name)
+	return target.producer.SendMessages(msgs)
+}
+
+// SendFromStore - reads an event from store and sends it to Kafka.
+func (target *KafkaTarget) SendFromStore(key store.Key) (err error) {
+	if err = target.init(); err != nil {
+		return err
+	}
+	switch {
+	case key.ItemCount == 1:
+		var event event.Event
+		event, err = target.store.Get(key)
+		if err != nil {
+			// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
+			// Such events will not exist and wouldve been already been sent successfully.
+			if os.IsNotExist(err) {
+				return nil
+			}
+			return err
+		}
+		err = target.send(event)
+	case key.ItemCount > 1:
+		var events []event.Event
+		events, err = target.store.GetMultiple(key)
 		if err != nil {
 			if os.IsNotExist(err) {
 				return nil
 			}
 			return err
 		}
-		msg, err := target.toProducerMessage(eventData)
-		if err != nil {
-			return err
-		}
-		if err = target.batch.Add(key.Name, msg); err != nil {
-			return err
-		}
+		err = target.sendMultiple(events)
 	}
-	// commit the batch if the key is the last one present in the store.
-	if key.IsLast || target.batch.IsFull() {
-		return target.commitBatch()
-	}
-	return nil
-}
-
-func (target *KafkaTarget) commitBatch() error {
-	keys, msgs, err := target.batch.GetAll()
 	if err != nil {
-		return err
-	}
-	if err = target.producer.SendMessages(msgs); err != nil {
 		if isKafkaConnErr(err) {
 			return store.ErrNotConnected
 		}
 		return err
 	}
-	return target.store.DelList(keys)
+	// Delete the event from store.
+	return target.store.Del(key)
 }
 
 func (target *KafkaTarget) toProducerMessage(eventData event.Event) (*sarama.ProducerMessage, error) {
@@ -319,7 +310,18 @@ func (target *KafkaTarget) toProducerMessage(eventData event.Event) (*sarama.Pro
 func (target *KafkaTarget) Close() error {
 	close(target.quitCh)
 
+	if target.batch != nil {
+		target.batch.Close()
+	}
+
 	if target.producer != nil {
+		if target.store != nil {
+			// It is safe to abort the current transaction if
+			// queue_dir is configured
+			target.producer.AbortTxn()
+		} else {
+			target.producer.CommitTxn()
+		}
 		target.producer.Close()
 		return target.client.Close()
 	}
@@ -442,10 +444,14 @@ func NewKafkaTarget(id string, args KafkaArgs, loggerOnce logger.LogOnce) (*Kafk
 		loggerOnce: loggerOnce,
 		quitCh:     make(chan struct{}),
 	}
-
 	if target.store != nil {
 		if args.BatchSize > 1 {
-			target.batch = store.NewBatch[string, *sarama.ProducerMessage](args.BatchSize)
+			target.batch = store.NewBatch[event.Event](store.BatchConfig[event.Event]{
+				Limit:         args.BatchSize,
+				Log:           loggerOnce,
+				Store:         queueStore,
+				CommitTimeout: args.BatchCommitTimeout,
+			})
 		}
 		store.StreamItems(target.store, target, target.quitCh, target.loggerOnce)
 	}

internal/event/target/mqtt.go

@@ -180,7 +180,7 @@ func (target *MQTTTarget) SendFromStore(key store.Key) error {
 		return err
 	}
 
-	eventData, err := target.store.Get(key.Name)
+	eventData, err := target.store.Get(key)
 	if err != nil {
 		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
 		// Such events will not exist and wouldve been already been sent successfully.
@@ -195,14 +195,15 @@ func (target *MQTTTarget) SendFromStore(key store.Key) error {
 	}
 
 	// Delete the event from store.
-	return target.store.Del(key.Name)
+	return target.store.Del(key)
 }
 
 // Save - saves the events to the store if queuestore is configured, which will
 // be replayed when the mqtt connection is active.
 func (target *MQTTTarget) Save(eventData event.Event) error {
 	if target.store != nil {
-		return target.store.Put(eventData)
+		_, err := target.store.Put(eventData)
+		return err
 	}
 	if err := target.init(); err != nil {
 		return err

internal/event/target/mysql.go

@@ -198,7 +198,8 @@ func (target *MySQLTarget) isActive() (bool, error) {
 // Save - saves the events to the store which will be replayed when the SQL connection is active.
 func (target *MySQLTarget) Save(eventData event.Event) error {
 	if target.store != nil {
-		return target.store.Put(eventData)
+		_, err := target.store.Put(eventData)
+		return err
 	}
 	if err := target.init(); err != nil {
 		return err
@@ -273,7 +274,7 @@ func (target *MySQLTarget) SendFromStore(key store.Key) error {
 		}
 	}
 
-	eventData, eErr := target.store.Get(key.Name)
+	eventData, eErr := target.store.Get(key)
 	if eErr != nil {
 		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
 		// Such events will not exist and wouldve been already been sent successfully.
@@ -291,7 +292,7 @@ func (target *MySQLTarget) SendFromStore(key store.Key) error {
 	}
 
 	// Delete the event from store.
-	return target.store.Del(key.Name)
+	return target.store.Del(key)
 }
 
 // Close - closes underneath connections to MySQL database.

internal/event/target/nats.go

@@ -299,7 +299,8 @@ func (target *NATSTarget) isActive() (bool, error) {
 // Save - saves the events to the store which will be replayed when the Nats connection is active.
 func (target *NATSTarget) Save(eventData event.Event) error {
 	if target.store != nil {
-		return target.store.Put(eventData)
+		_, err := target.store.Put(eventData)
+		return err
 	}
 
 	if err := target.init(); err != nil {
@@ -353,7 +354,7 @@ func (target *NATSTarget) SendFromStore(key store.Key) error {
 		return err
 	}
 
-	eventData, eErr := target.store.Get(key.Name)
+	eventData, eErr := target.store.Get(key)
 	if eErr != nil {
 		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
 		// Such events will not exist and wouldve been already been sent successfully.
@@ -367,7 +368,7 @@ func (target *NATSTarget) SendFromStore(key store.Key) error {
 		return err
 	}
 
-	return target.store.Del(key.Name)
+	return target.store.Del(key)
 }
 
 // Close - closes underneath connections to NATS server.

internal/event/target/nsq.go

@@ -147,7 +147,8 @@ func (target *NSQTarget) isActive() (bool, error) {
 // Save - saves the events to the store which will be replayed when the nsq connection is active.
 func (target *NSQTarget) Save(eventData event.Event) error {
 	if target.store != nil {
-		return target.store.Put(eventData)
+		_, err := target.store.Put(eventData)
+		return err
 	}
 
 	if err := target.init(); err != nil {
@@ -188,7 +189,7 @@ func (target *NSQTarget) SendFromStore(key store.Key) error {
 		return err
 	}
 
-	eventData, eErr := target.store.Get(key.Name)
+	eventData, eErr := target.store.Get(key)
 	if eErr != nil {
 		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
 		// Such events will not exist and wouldve been already been sent successfully.
@@ -203,7 +204,7 @@ func (target *NSQTarget) SendFromStore(key store.Key) error {
 	}
 
 	// Delete the event from store.
-	return target.store.Del(key.Name)
+	return target.store.Del(key)
 }
 
 // Close - closes underneath connections to NSQD server.

internal/event/target/postgresql.go

@@ -196,7 +196,8 @@ func (target *PostgreSQLTarget) isActive() (bool, error) {
 // Save - saves the events to the store if questore is configured, which will be replayed when the PostgreSQL connection is active.
 func (target *PostgreSQLTarget) Save(eventData event.Event) error {
 	if target.store != nil {
-		return target.store.Put(eventData)
+		_, err := target.store.Put(eventData)
+		return err
 	}
 
 	if err := target.init(); err != nil {
@@ -275,7 +276,7 @@ func (target *PostgreSQLTarget) SendFromStore(key store.Key) error {
 		}
 	}
 
-	eventData, eErr := target.store.Get(key.Name)
+	eventData, eErr := target.store.Get(key)
 	if eErr != nil {
 		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
 		// Such events will not exist and wouldve been already been sent successfully.
@@ -293,7 +294,7 @@ func (target *PostgreSQLTarget) SendFromStore(key store.Key) error {
 	}
 
 	// Delete the event from store.
-	return target.store.Del(key.Name)
+	return target.store.Del(key)
 }
 
 // Close - closes underneath connections to PostgreSQL database.

internal/event/target/redis.go

@@ -173,7 +173,8 @@ func (target *RedisTarget) isActive() (bool, error) {
 // Save - saves the events to the store if questore is configured, which will be replayed when the redis connection is active.
 func (target *RedisTarget) Save(eventData event.Event) error {
 	if target.store != nil {
-		return target.store.Put(eventData)
+		_, err := target.store.Put(eventData)
+		return err
 	}
 	if err := target.init(); err != nil {
 		return err
@@ -252,7 +253,7 @@ func (target *RedisTarget) SendFromStore(key store.Key) error {
 		target.firstPing = true
 	}
 
-	eventData, eErr := target.store.Get(key.Name)
+	eventData, eErr := target.store.Get(key)
 	if eErr != nil {
 		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
 		// Such events will not exist and would've been already been sent successfully.
@@ -270,7 +271,7 @@ func (target *RedisTarget) SendFromStore(key store.Key) error {
 	}
 
 	// Delete the event from store.
-	return target.store.Del(key.Name)
+	return target.store.Del(key)
 }
 
 // Close - releases the resources used by the pool.

internal/event/target/webhook.go

@@ -146,7 +146,8 @@ func (target *WebhookTarget) isActive() (bool, error) {
 // which will be replayed when the webhook connection is active.
 func (target *WebhookTarget) Save(eventData event.Event) error {
 	if target.store != nil {
-		return target.store.Put(eventData)
+		_, err := target.store.Put(eventData)
+		return err
 	}
 	if err := target.init(); err != nil {
 		return err
@@ -213,7 +214,7 @@ func (target *WebhookTarget) SendFromStore(key store.Key) error {
 		return err
 	}
 
-	eventData, eErr := target.store.Get(key.Name)
+	eventData, eErr := target.store.Get(key)
 	if eErr != nil {
 		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
 		// Such events will not exist and would've been already been sent successfully.
@@ -231,7 +232,7 @@ func (target *WebhookTarget) SendFromStore(key store.Key) error {
 	}
 
 	// Delete the event from store.
-	return target.store.Del(key.Name)
+	return target.store.Del(key)
 }
 
 // Close - does nothing and available for interface compatibility.