Kafka notify: support batched commits for queue store (#20377)

The items will be saved per target batch and will
be committed to the queue store when the batch is full

Also, periodically commit the batched items to the queue store
based on configured commit_timeout; default is 30s;

Bonus: compress queue store multi writes
This commit is contained in:
Praveen raj Mani
2024-09-07 04:36:30 +05:30
committed by GitHub
parent 0f1e8db4c5
commit 261111e728
20 changed files with 907 additions and 397 deletions

View File

@@ -173,7 +173,8 @@ func (target *RedisTarget) isActive() (bool, error) {
// Save - saves the events to the store if questore is configured, which will be replayed when the redis connection is active.
func (target *RedisTarget) Save(eventData event.Event) error {
if target.store != nil {
return target.store.Put(eventData)
_, err := target.store.Put(eventData)
return err
}
if err := target.init(); err != nil {
return err
@@ -252,7 +253,7 @@ func (target *RedisTarget) SendFromStore(key store.Key) error {
target.firstPing = true
}
eventData, eErr := target.store.Get(key.Name)
eventData, eErr := target.store.Get(key)
if eErr != nil {
// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
// Such events will not exist and would've been already been sent successfully.
@@ -270,7 +271,7 @@ func (target *RedisTarget) SendFromStore(key store.Key) error {
}
// Delete the event from store.
return target.store.Del(key.Name)
return target.store.Del(key)
}
// Close - releases the resources used by the pool.