Improve expiration of tiered objects (#18926)

- Use a shared worker pool for all ILM expiry tasks
- Free version cleanup executes in a separate goroutine
- Add a free version only if removing the remote object fails
- Add ILM expiry metrics to the node namespace
- Move tier journal tasks to expiryState
- Remove unused on-disk journal for tiered objects pending deletion
- Distribute expiry tasks across workers such that the expiry of versions of
  the same object serialized
- Ability to resize worker pool without server restart
- Make scaling down of expiryState workers' concurrency safe; Thanks
  @klauspost
- Add error logs when expiryState and transition state are not
  initialized (yet)
* metrics: Add missed tier journal entry tasks
* Initialize the ILM worker pool after the object layer

cmd/tier-sweeper.go

@@ -18,6 +18,8 @@
 package cmd
 
 import (
+	"context"
+
 	"github.com/minio/minio/internal/bucket/lifecycle"
 )
 
@@ -128,9 +130,26 @@ func (os *objSweeper) shouldRemoveRemoteObject() (jentry, bool) {
 }
 
 // Sweep removes the transitioned object if it's no longer referred to.
-func (os *objSweeper) Sweep() error {
+func (os *objSweeper) Sweep() {
 	if je, ok := os.shouldRemoveRemoteObject(); ok {
-		return globalTierJournal.AddEntry(je)
+		globalExpiryState.enqueueTierJournalEntry(je)
+	}
+}
+
+type jentry struct {
+	ObjName   string
+	VersionID string
+	TierName  string
+}
+
+func deleteObjectFromRemoteTier(ctx context.Context, objName, rvID, tierName string) error {
+	w, err := globalTierConfigMgr.getDriver(tierName)
+	if err != nil {
+		return err
+	}
+	err = w.Remove(ctx, objName, remoteVersionID(rvID))
+	if err != nil {
+		return err
 	}
 	return nil
 }