MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 21:02:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

Kafka notify: support batched commits for queue store (#20377)

Browse Source 
The items will be saved per target batch and will
be committed to the queue store when the batch is full

Also, periodically commit the batched items to the queue store
based on configured commit_timeout; default is 30s;

Bonus: compress queue store multi writes
This commit is contained in:
Praveen raj Mani
2024-09-07 04:36:30 +05:30
committed by GitHub
parent 0f1e8db4c5
commit 261111e728
20 changed files with 907 additions and 397 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
internal/logger/target/http/http.go
View File

@@ -418,7 +418,7 @@ func (h *Target) startQueueProcessor(ctx context.Context, mainWorker bool) {
if !isDirQueue {
err = h.send(ctx, buf.Bytes(), count, h.payloadType, webhookCallTimeout)
} else {
err = h.store.PutMultiple(entries)
_, err = h.store.PutMultiple(entries)
}
if err != nil {
@@ -530,7 +530,7 @@ func New(config Config) (*Target, error) {
// SendFromStore - reads the log from store and sends it to webhook.
func (h *Target) SendFromStore(key store.Key) (err error) {
var eventData []byte
eventData, err = h.store.GetRaw(key.Name)
eventData, err = h.store.GetRaw(key)
if err != nil {
if os.IsNotExist(err) {
return nil
@@ -552,7 +552,7 @@ func (h *Target) SendFromStore(key store.Key) (err error) {
}
// Delete the event from store.
return h.store.Del(key.Name)
return h.store.Del(key)
}
// Send the log message 'entry' to the http target.

7
internal/logger/target/kafka/kafka.go
View File

@@ -315,7 +315,8 @@ func (h *Target) IsOnline(_ context.Context) bool {
func (h *Target) Send(ctx context.Context, entry interface{}) error {
if h.store != nil {
// save the entry to the queue store which will be replayed to the target.
return h.store.Put(entry)
_, err := h.store.Put(entry)
return err
}
h.logChMu.RLock()
defer h.logChMu.RUnlock()
@@ -344,7 +345,7 @@ func (h *Target) Send(ctx context.Context, entry interface{}) error {
// SendFromStore - reads the log from store and sends it to kafka.
func (h *Target) SendFromStore(key store.Key) (err error) {
auditEntry, err := h.store.Get(key.Name)
auditEntry, err := h.store.Get(key)
if err != nil {
if os.IsNotExist(err) {
return nil
@@ -358,7 +359,7 @@ func (h *Target) SendFromStore(key store.Key) (err error) {
return
}
// Delete the event from store.
return h.store.Del(key.Name)
return h.store.Del(key)
}
// Cancel - cancels the target