feat: bring new HDD related performance enhancements (#18239)

Optionally allows customers to enable 

- Enable an external cache to catch GET/HEAD responses 
- Enable skipping disks that are slow to respond in GET/HEAD 
  when we have already achieved a quorum

cmd/bucket-handlers.go

@@ -51,6 +51,7 @@ import (
 	sse "github.com/minio/minio/internal/bucket/encryption"
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/config/cache"
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/event"
@@ -668,6 +669,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			continue
 		}
 
+		defer globalCacheConfig.Delete(bucket, dobj.ObjectName)
+
 		if replicateDeletes && (dobj.DeleteMarkerReplicationStatus() == replication.Pending || dobj.VersionPurgeStatus() == Pending) {
 			// copy so we can re-add null ID.
 			dobj := dobj
@@ -1343,6 +1346,22 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 					continue
 				}
 
+				asize, err := objInfo.GetActualSize()
+				if err != nil {
+					asize = objInfo.Size
+				}
+
+				globalCacheConfig.Set(&cache.ObjectInfo{
+					Key:          objInfo.Name,
+					Bucket:       objInfo.Bucket,
+					ETag:         getDecryptedETag(formValues, objInfo, false),
+					ModTime:      objInfo.ModTime,
+					Expires:      objInfo.Expires.UTC().Format(http.TimeFormat),
+					CacheControl: objInfo.CacheControl,
+					Metadata:     cleanReservedKeys(objInfo.UserDefined),
+					Size:         asize,
+				})
+
 				fanOutResp = append(fanOutResp, minio.PutObjectFanOutResponse{
 					Key:          objInfo.Name,
 					ETag:         getDecryptedETag(formValues, objInfo, false),
@@ -1399,10 +1418,12 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
+	etag := getDecryptedETag(formValues, objInfo, false)
+
 	// We must not use the http.Header().Set method here because some (broken)
 	// clients expect the ETag header key to be literally "ETag" - not "Etag" (case-sensitive).
 	// Therefore, we have to set the ETag directly as map entry.
-	w.Header()[xhttp.ETag] = []string{`"` + objInfo.ETag + `"`}
+	w.Header()[xhttp.ETag] = []string{`"` + etag + `"`}
 
 	// Set the relevant version ID as part of the response header.
 	if objInfo.VersionID != "" && objInfo.VersionID != nullVersionID {
@@ -1413,6 +1434,22 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		w.Header().Set(xhttp.Location, obj)
 	}
 
+	asize, err := objInfo.GetActualSize()
+	if err != nil {
+		asize = objInfo.Size
+	}
+
+	defer globalCacheConfig.Set(&cache.ObjectInfo{
+		Key:          objInfo.Name,
+		Bucket:       objInfo.Bucket,
+		ETag:         etag,
+		ModTime:      objInfo.ModTime,
+		Expires:      objInfo.ExpiresStr(),
+		CacheControl: objInfo.CacheControl,
+		Metadata:     cleanReservedKeys(objInfo.UserDefined),
+		Size:         asize,
+	})
+
 	// Notify object created event.
 	defer sendEvent(eventArgs{
 		EventName:    event.ObjectCreatedPost,