fix: replication metadata comparsion and other fixes (#11410)

- using miniogo.ObjectInfo.UserMetadata is not correct
- using UserTags from Map->String() can change order
- ContentType comparison needs to be removed.
- Compare both lowercase and uppercase key names.
- do not silently error out constructing PutObjectOptions
  if tag parsing fails
- avoid notification for empty object info, failed operations
  should rely on valid objInfo for notification in all
  situations
- optimize copyObject implementation, also introduce a new 
  replication event
- clone ObjectInfo() before scheduling for replication
- add additional headers for comparison
- remove strings.EqualFold comparison avoid unexpected bugs
- fix pool based proxying with multiple pools
- compare only specific metadata

Co-authored-by: Poorna Krishnamoorthy <poornas@users.noreply.github.com>

cmd/handler-utils.go

@@ -26,6 +26,7 @@ import (
 	"mime/multipart"
 	"net"
 	"net/http"
+	"net/textproto"
 	"net/url"
 	"regexp"
 	"strings"
@@ -74,6 +75,7 @@ var supportedHeaders = []string{
 	"content-language",
 	"content-encoding",
 	"content-disposition",
+	"x-amz-storage-class",
 	xhttp.AmzStorageClass,
 	xhttp.AmzObjectTagging,
 	"expires",
@@ -103,8 +105,6 @@ func isDirectiveReplace(value string) bool {
 // All values stored with a key starting with one of the following prefixes
 // must be extracted from the header.
 var userMetadataKeyPrefixes = []string{
-	"X-Amz-Meta-",
-	"X-Minio-Meta-",
 	"x-amz-meta-",
 	"x-minio-meta-",
 }
@@ -115,13 +115,13 @@ func extractMetadata(ctx context.Context, r *http.Request) (metadata map[string]
 	header := r.Header
 	metadata = make(map[string]string)
 	// Extract all query values.
-	err = extractMetadataFromMap(ctx, query, metadata)
+	err = extractMetadataFromMime(ctx, textproto.MIMEHeader(query), metadata)
 	if err != nil {
 		return nil, err
 	}
 
 	// Extract all header values.
-	err = extractMetadataFromMap(ctx, header, metadata)
+	err = extractMetadataFromMime(ctx, textproto.MIMEHeader(header), metadata)
 	if err != nil {
 		return nil, err
 	}
@@ -133,7 +133,7 @@ func extractMetadata(ctx context.Context, r *http.Request) (metadata map[string]
 
 	// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
 	for k := range metadata {
-		if strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentLength) || strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentMD5) {
+		if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
 			delete(metadata, k)
 		}
 	}
@@ -161,25 +161,32 @@ func extractMetadata(ctx context.Context, r *http.Request) (metadata map[string]
 }
 
 // extractMetadata extracts metadata from map values.
-func extractMetadataFromMap(ctx context.Context, v map[string][]string, m map[string]string) error {
+func extractMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[string]string) error {
 	if v == nil {
 		logger.LogIf(ctx, errInvalidArgument)
 		return errInvalidArgument
 	}
+
+	nv := make(textproto.MIMEHeader, len(v))
+	for k, kv := range v {
+		// Canonicalize all headers, to remove any duplicates.
+		nv[http.CanonicalHeaderKey(k)] = kv
+	}
+
 	// Save all supported headers.
 	for _, supportedHeader := range supportedHeaders {
-		if value, ok := v[http.CanonicalHeaderKey(supportedHeader)]; ok {
-			m[supportedHeader] = value[0]
-		} else if value, ok := v[supportedHeader]; ok {
-			m[supportedHeader] = value[0]
+		value, ok := nv[http.CanonicalHeaderKey(supportedHeader)]
+		if ok {
+			m[supportedHeader] = strings.Join(value, ",")
 		}
 	}
+
 	for key := range v {
 		for _, prefix := range userMetadataKeyPrefixes {
 			if !strings.HasPrefix(strings.ToLower(key), strings.ToLower(prefix)) {
 				continue
 			}
-			value, ok := v[key]
+			value, ok := nv[http.CanonicalHeaderKey(key)]
 			if ok {
 				m[key] = strings.Join(value, ",")
 				break