mirror of
https://github.com/minio/minio.git
synced 2025-11-09 13:39:46 -05:00
Kafka notify: support batched commits for queue store (#20377)
The items will be saved per target batch and will be committed to the queue store when the batch is full Also, periodically commit the batched items to the queue store based on configured commit_timeout; default is 30s; Bonus: compress queue store multi writes
This commit is contained in:
@@ -173,7 +173,8 @@ func (target *RedisTarget) isActive() (bool, error) {
|
||||
// Save - saves the events to the store if questore is configured, which will be replayed when the redis connection is active.
|
||||
func (target *RedisTarget) Save(eventData event.Event) error {
|
||||
if target.store != nil {
|
||||
return target.store.Put(eventData)
|
||||
_, err := target.store.Put(eventData)
|
||||
return err
|
||||
}
|
||||
if err := target.init(); err != nil {
|
||||
return err
|
||||
@@ -252,7 +253,7 @@ func (target *RedisTarget) SendFromStore(key store.Key) error {
|
||||
target.firstPing = true
|
||||
}
|
||||
|
||||
eventData, eErr := target.store.Get(key.Name)
|
||||
eventData, eErr := target.store.Get(key)
|
||||
if eErr != nil {
|
||||
// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
|
||||
// Such events will not exist and would've been already been sent successfully.
|
||||
@@ -270,7 +271,7 @@ func (target *RedisTarget) SendFromStore(key store.Key) error {
|
||||
}
|
||||
|
||||
// Delete the event from store.
|
||||
return target.store.Del(key.Name)
|
||||
return target.store.Del(key)
|
||||
}
|
||||
|
||||
// Close - releases the resources used by the pool.
|
||||
|
||||
Reference in New Issue
Block a user