Webhook targets refactor and bug fixes (#19275)

- old version was unable to retain messages during config reload
- old version could not go from memory to disk during reload
- new version can batch disk queue entries to single for to reduce I/O load
- error logging has been improved, previous version would miss certain errors.
- logic for spawning/despawning additional workers has been adjusted to trigger when half capacity is reached, instead of when the log queue becomes full.
- old version would json marshall x2 and unmarshal 1x for every log item. Now we only do marshal x1 and then we GetRaw from the store and send it without having to re-marshal.

internal/store/queuestore.go

@@ -28,6 +28,8 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	jsoniter "github.com/json-iterator/go"
+	"github.com/valyala/bytebufferpool"
 )
 
 const (
@@ -84,6 +86,7 @@ func (store *QueueStore[_]) Open() error {
 	if uint64(len(files)) > store.entryLimit {
 		files = files[:store.entryLimit]
 	}
+
 	for _, file := range files {
 		if file.IsDir() {
 			continue
@@ -97,6 +100,54 @@ func (store *QueueStore[_]) Open() error {
 	return nil
 }
 
+// Delete - Remove the store directory from disk
+func (store *QueueStore[_]) Delete() error {
+	return os.Remove(store.directory)
+}
+
+// PutMultiple - puts an item to the store.
+func (store *QueueStore[I]) PutMultiple(item []I) error {
+	store.Lock()
+	defer store.Unlock()
+	if uint64(len(store.entries)) >= store.entryLimit {
+		return errLimitExceeded
+	}
+	// Generate a new UUID for the key.
+	key, err := uuid.NewRandom()
+	if err != nil {
+		return err
+	}
+	return store.multiWrite(key.String(), item)
+}
+
+// multiWrite - writes an item to the directory.
+func (store *QueueStore[I]) multiWrite(key string, item []I) error {
+	buf := bytebufferpool.Get()
+	defer bytebufferpool.Put(buf)
+
+	enc := jsoniter.ConfigCompatibleWithStandardLibrary.NewEncoder(buf)
+
+	for i := range item {
+		err := enc.Encode(item[i])
+		if err != nil {
+			return err
+		}
+	}
+	b := buf.Bytes()
+
+	path := filepath.Join(store.directory, key+store.fileExt)
+	err := os.WriteFile(path, b, os.FileMode(0o770))
+	buf.Reset()
+	if err != nil {
+		return err
+	}
+
+	// Increment the item count.
+	store.entries[key] = time.Now().UnixNano()
+
+	return nil
+}
+
 // write - writes an item to the directory.
 func (store *QueueStore[I]) write(key string, item I) error {
 	// Marshalls the item.
@@ -131,6 +182,30 @@ func (store *QueueStore[I]) Put(item I) error {
 	return store.write(key.String(), item)
 }
 
+// GetRaw - gets an item from the store.
+func (store *QueueStore[I]) GetRaw(key string) (raw []byte, err error) {
+	store.RLock()
+
+	defer func(store *QueueStore[I]) {
+		store.RUnlock()
+		if err != nil {
+			// Upon error we remove the entry.
+			store.Del(key)
+		}
+	}(store)
+
+	raw, err = os.ReadFile(filepath.Join(store.directory, key+store.fileExt))
+	if err != nil {
+		return
+	}
+
+	if len(raw) == 0 {
+		return raw, os.ErrNotExist
+	}
+
+	return
+}
+
 // Get - gets an item from the store.
 func (store *QueueStore[I]) Get(key string) (item I, err error) {
 	store.RLock()