Generalize the event store using go generics (#16910)

internal/store/queuestore.go

@@ -0,0 +1,234 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package store
+
+import (
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+const (
+	defaultLimit = 100000 // Default store limit.
+	defaultExt   = ".unknown"
+)
+
+// errLimitExceeded error is sent when the maximum limit is reached.
+var errLimitExceeded = errors.New("the maximum store limit reached")
+
+// QueueStore - Filestore for persisting items.
+type QueueStore[_ any] struct {
+	sync.RWMutex
+	entryLimit uint64
+	directory  string
+	fileExt    string
+
+	entries map[string]int64 // key -> modtime as unix nano
+}
+
+// NewQueueStore - Creates an instance for QueueStore.
+func NewQueueStore[I any](directory string, limit uint64, ext string) *QueueStore[I] {
+	if limit == 0 {
+		limit = defaultLimit
+	}
+
+	if ext == "" {
+		ext = defaultExt
+	}
+
+	return &QueueStore[I]{
+		directory:  directory,
+		entryLimit: limit,
+		fileExt:    ext,
+		entries:    make(map[string]int64, limit),
+	}
+}
+
+// Open - Creates the directory if not present.
+func (store *QueueStore[_]) Open() error {
+	store.Lock()
+	defer store.Unlock()
+
+	if err := os.MkdirAll(store.directory, os.FileMode(0o770)); err != nil {
+		return err
+	}
+
+	files, err := store.list()
+	if err != nil {
+		return err
+	}
+
+	// Truncate entries.
+	if uint64(len(files)) > store.entryLimit {
+		files = files[:store.entryLimit]
+	}
+	for _, file := range files {
+		if file.IsDir() {
+			continue
+		}
+		key := strings.TrimSuffix(file.Name(), store.fileExt)
+		if fi, err := file.Info(); err == nil {
+			store.entries[key] = fi.ModTime().UnixNano()
+		}
+	}
+
+	return nil
+}
+
+// write - writes an item to the directory.
+func (store *QueueStore[I]) write(key string, item I) error {
+	// Marshalls the item.
+	eventData, err := json.Marshal(item)
+	if err != nil {
+		return err
+	}
+
+	path := filepath.Join(store.directory, key+store.fileExt)
+	if err := os.WriteFile(path, eventData, os.FileMode(0o770)); err != nil {
+		return err
+	}
+
+	// Increment the item count.
+	store.entries[key] = time.Now().UnixNano()
+
+	return nil
+}
+
+// Put - puts an item to the store.
+func (store *QueueStore[I]) Put(item I) error {
+	store.Lock()
+	defer store.Unlock()
+	if uint64(len(store.entries)) >= store.entryLimit {
+		return errLimitExceeded
+	}
+	// Generate a new UUID for the key.
+	key, err := uuid.NewRandom()
+	if err != nil {
+		return err
+	}
+	return store.write(key.String(), item)
+}
+
+// Get - gets an item from the store.
+func (store *QueueStore[I]) Get(key string) (item I, err error) {
+	store.RLock()
+
+	defer func(store *QueueStore[I]) {
+		store.RUnlock()
+		if err != nil {
+			// Upon error we remove the entry.
+			store.Del(key)
+		}
+	}(store)
+
+	var eventData []byte
+	eventData, err = os.ReadFile(filepath.Join(store.directory, key+store.fileExt))
+	if err != nil {
+		return item, err
+	}
+
+	if len(eventData) == 0 {
+		return item, os.ErrNotExist
+	}
+
+	if err = json.Unmarshal(eventData, &item); err != nil {
+		return item, err
+	}
+
+	return item, nil
+}
+
+// Del - Deletes an entry from the store.
+func (store *QueueStore[_]) Del(key string) error {
+	store.Lock()
+	defer store.Unlock()
+	return store.del(key)
+}
+
+// Len returns the entry count.
+func (store *QueueStore[_]) Len() int {
+	store.RLock()
+	l := len(store.entries)
+	defer store.RUnlock()
+	return l
+}
+
+// lockless call
+func (store *QueueStore[_]) del(key string) error {
+	err := os.Remove(filepath.Join(store.directory, key+store.fileExt))
+
+	// Delete as entry no matter the result
+	delete(store.entries, key)
+
+	return err
+}
+
+// List - lists all files registered in the store.
+func (store *QueueStore[_]) List() ([]string, error) {
+	store.RLock()
+	l := make([]string, 0, len(store.entries))
+	for k := range store.entries {
+		l = append(l, k)
+	}
+
+	// Sort entries...
+	sort.Slice(l, func(i, j int) bool {
+		return store.entries[l[i]] < store.entries[l[j]]
+	})
+	store.RUnlock()
+
+	return l, nil
+}
+
+// list will read all entries from disk.
+// Entries are returned sorted by modtime, oldest first.
+// Underlying entry list in store is *not* updated.
+func (store *QueueStore[_]) list() ([]os.DirEntry, error) {
+	files, err := os.ReadDir(store.directory)
+	if err != nil {
+		return nil, err
+	}
+
+	// Sort the entries.
+	sort.Slice(files, func(i, j int) bool {
+		ii, err := files[i].Info()
+		if err != nil {
+			return false
+		}
+		ji, err := files[j].Info()
+		if err != nil {
+			return true
+		}
+		return ii.ModTime().Before(ji.ModTime())
+	})
+
+	return files, nil
+}
+
+// Extension will return the file extension used
+// for the items stored in the queue.
+func (store *QueueStore[_]) Extension() string {
+	return store.fileExt
+}

internal/store/queuestore_test.go

@@ -0,0 +1,240 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package store
+
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+type TestItem struct {
+	Name     string `json:"Name"`
+	Property string `json:"property"`
+}
+
+var (
+	// TestDir
+	queueDir = filepath.Join(os.TempDir(), "minio_test")
+	// Sample test item.
+	testItem = TestItem{Name: "test-item", Property: "property"}
+	// Ext for test item
+	testItemExt = ".test"
+)
+
+// Initialize the queue store.
+func setUpQueueStore(directory string, limit uint64) (Store[TestItem], error) {
+	queueStore := NewQueueStore[TestItem](queueDir, limit, testItemExt)
+	if oErr := queueStore.Open(); oErr != nil {
+		return nil, oErr
+	}
+	return queueStore, nil
+}
+
+// Tear down queue store.
+func tearDownQueueStore() error {
+	return os.RemoveAll(queueDir)
+}
+
+// TestQueueStorePut - tests for store.Put
+func TestQueueStorePut(t *testing.T) {
+	defer func() {
+		if err := tearDownQueueStore(); err != nil {
+			t.Fatal("Failed to tear down store ", err)
+		}
+	}()
+	store, err := setUpQueueStore(queueDir, 100)
+	if err != nil {
+		t.Fatal("Failed to create a queue store ", err)
+	}
+	// Put 100 items.
+	for i := 0; i < 100; i++ {
+		if err := store.Put(testItem); err != nil {
+			t.Fatal("Failed to put to queue store ", err)
+		}
+	}
+	// Count the items.
+	names, err := store.List()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(names) != 100 {
+		t.Fatalf("List() Expected: 100, got %d", len(names))
+	}
+}
+
+// TestQueueStoreGet - tests for store.Get
+func TestQueueStoreGet(t *testing.T) {
+	defer func() {
+		if err := tearDownQueueStore(); err != nil {
+			t.Fatal("Failed to tear down store ", err)
+		}
+	}()
+	store, err := setUpQueueStore(queueDir, 10)
+	if err != nil {
+		t.Fatal("Failed to create a queue store ", err)
+	}
+	// Put 10 items
+	for i := 0; i < 10; i++ {
+		if err := store.Put(testItem); err != nil {
+			t.Fatal("Failed to put to queue store ", err)
+		}
+	}
+	itemKeys, err := store.List()
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Get 10 items.
+	if len(itemKeys) == 10 {
+		for _, key := range itemKeys {
+			item, eErr := store.Get(strings.TrimSuffix(key, testItemExt))
+			if eErr != nil {
+				t.Fatal("Failed to Get the item from the queue store ", eErr)
+			}
+			if !reflect.DeepEqual(testItem, item) {
+				t.Fatalf("Failed to read the item: error: expected = %v, got = %v", testItem, item)
+			}
+		}
+	} else {
+		t.Fatalf("List() Expected: 10, got %d", len(itemKeys))
+	}
+}
+
+// TestQueueStoreDel - tests for store.Del
+func TestQueueStoreDel(t *testing.T) {
+	defer func() {
+		if err := tearDownQueueStore(); err != nil {
+			t.Fatal("Failed to tear down store ", err)
+		}
+	}()
+	store, err := setUpQueueStore(queueDir, 20)
+	if err != nil {
+		t.Fatal("Failed to create a queue store ", err)
+	}
+	// Put 20 items.
+	for i := 0; i < 20; i++ {
+		if err := store.Put(testItem); err != nil {
+			t.Fatal("Failed to put to queue store ", err)
+		}
+	}
+	itemKeys, err := store.List()
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Remove all the items.
+	if len(itemKeys) == 20 {
+		for _, key := range itemKeys {
+			err := store.Del(strings.TrimSuffix(key, testItemExt))
+			if err != nil {
+				t.Fatal("queue store Del failed with ", err)
+			}
+		}
+	} else {
+		t.Fatalf("List() Expected: 20, got %d", len(itemKeys))
+	}
+
+	names, err := store.List()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(names) != 0 {
+		t.Fatalf("List() Expected: 0, got %d", len(names))
+	}
+}
+
+// TestQueueStoreLimit - tests the item limit for the store.
+func TestQueueStoreLimit(t *testing.T) {
+	defer func() {
+		if err := tearDownQueueStore(); err != nil {
+			t.Fatal("Failed to tear down store ", err)
+		}
+	}()
+	// The max limit is set to 5.
+	store, err := setUpQueueStore(queueDir, 5)
+	if err != nil {
+		t.Fatal("Failed to create a queue store ", err)
+	}
+	for i := 0; i < 5; i++ {
+		if err := store.Put(testItem); err != nil {
+			t.Fatal("Failed to put to queue store ", err)
+		}
+	}
+	// Should not allow 6th Put.
+	if err := store.Put(testItem); err == nil {
+		t.Fatalf("Expected to fail with %s, but passes", errLimitExceeded)
+	}
+}
+
+// TestQueueStoreLimit - tests for store.LimitN.
+func TestQueueStoreListN(t *testing.T) {
+	defer func() {
+		if err := tearDownQueueStore(); err != nil {
+			t.Fatal("Failed to tear down store ", err)
+		}
+	}()
+	store, err := setUpQueueStore(queueDir, 10)
+	if err != nil {
+		t.Fatal("Failed to create a queue store ", err)
+	}
+	for i := 0; i < 10; i++ {
+		if err := store.Put(testItem); err != nil {
+			t.Fatal("Failed to put to queue store ", err)
+		}
+	}
+	// Should return all the item keys in the store.
+	names, err := store.List()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(names) != 10 {
+		t.Fatalf("List() Expected: 10, got %d", len(names))
+	}
+
+	// re-open
+	store, err = setUpQueueStore(queueDir, 10)
+	if err != nil {
+		t.Fatal("Failed to create a queue store ", err)
+	}
+	names, err = store.List()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(names) != 10 {
+		t.Fatalf("List() Expected: 10, got %d", len(names))
+	}
+	if len(names) != store.Len() {
+		t.Fatalf("List() Expected: 10, got %d", len(names))
+	}
+
+	// Delete all
+	for _, key := range names {
+		err := store.Del(key)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	// Re-list
+	lst, err := store.List()
+	if len(lst) > 0 || err != nil {
+		t.Fatalf("Expected List() to return empty list and no error, got %v err: %v", lst, err)
+	}
+}

internal/store/store.go

@@ -0,0 +1,145 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package store
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/minio/minio/internal/logger"
+	xnet "github.com/minio/pkg/net"
+)
+
+const (
+	retryInterval = 3 * time.Second
+)
+
+// ErrNotConnected - indicates that the target connection is not active.
+var ErrNotConnected = errors.New("not connected to target server/service")
+
+// Target - store target interface
+type Target interface {
+	Name() string
+	Send(key string) error
+}
+
+// Store - Used to persist items.
+type Store[I any] interface {
+	Put(item I) error
+	Get(key string) (I, error)
+	Len() int
+	List() ([]string, error)
+	Del(key string) error
+	Open() error
+	Extension() string
+}
+
+// replayItems - Reads the items from the store and replays.
+func replayItems[I any](store Store[I], doneCh <-chan struct{}, loggerOnce logger.LogOnce, id string) <-chan string {
+	itemKeyCh := make(chan string)
+
+	go func() {
+		defer close(itemKeyCh)
+
+		retryTicker := time.NewTicker(retryInterval)
+		defer retryTicker.Stop()
+
+		for {
+			names, err := store.List()
+			if err != nil {
+				loggerOnce(context.Background(), fmt.Errorf("store.List() failed with: %w", err), id)
+			} else {
+				for _, name := range names {
+					select {
+					case itemKeyCh <- strings.TrimSuffix(name, store.Extension()):
+					// Get next key.
+					case <-doneCh:
+						return
+					}
+				}
+			}
+
+			select {
+			case <-retryTicker.C:
+			case <-doneCh:
+				return
+			}
+		}
+	}()
+
+	return itemKeyCh
+}
+
+// sendItems - Reads items from the store and re-plays.
+func sendItems(target Target, itemKeyCh <-chan string, doneCh <-chan struct{}, loggerOnce logger.LogOnce) {
+	retryTicker := time.NewTicker(retryInterval)
+	defer retryTicker.Stop()
+
+	send := func(itemKey string) bool {
+		for {
+			err := target.Send(itemKey)
+			if err == nil {
+				break
+			}
+
+			if err != ErrNotConnected && !xnet.IsConnResetErr(err) {
+				loggerOnce(context.Background(),
+					fmt.Errorf("target.Send() failed with '%w'", err),
+					target.Name())
+			}
+
+			// Retrying after 3secs back-off
+
+			select {
+			case <-retryTicker.C:
+			case <-doneCh:
+				return false
+			}
+		}
+		return true
+	}
+
+	for {
+		select {
+		case itemKey, ok := <-itemKeyCh:
+			if !ok {
+				// closed channel.
+				return
+			}
+
+			if !send(itemKey) {
+				return
+			}
+		case <-doneCh:
+			return
+		}
+	}
+}
+
+// StreamItems reads the keys from the store and replays the corresponding item to the target.
+func StreamItems[I any](store Store[I], target Target, doneCh <-chan struct{}, loggerOnce logger.LogOnce) {
+	go func() {
+		// Replays the items from the store.
+		itemKeyCh := replayItems(store, doneCh, loggerOnce, target.Name())
+		// Send items from the store.
+		sendItems(target, itemKeyCh, doneCh, loggerOnce)
+	}()
+}