fix: replication metadata comparsion and other fixes (#11410)

- using miniogo.ObjectInfo.UserMetadata is not correct
- using UserTags from Map->String() can change order
- ContentType comparison needs to be removed.
- Compare both lowercase and uppercase key names.
- do not silently error out constructing PutObjectOptions
  if tag parsing fails
- avoid notification for empty object info, failed operations
  should rely on valid objInfo for notification in all
  situations
- optimize copyObject implementation, also introduce a new 
  replication event
- clone ObjectInfo() before scheduling for replication
- add additional headers for comparison
- remove strings.EqualFold comparison avoid unexpected bugs
- fix pool based proxying with multiple pools
- compare only specific metadata

Co-authored-by: Poorna Krishnamoorthy <poornas@users.noreply.github.com>

cmd/erasure-server-pool.go

@@ -259,13 +259,15 @@ func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object stri
 	for i, pool := range z.serverPools {
 		objInfo, err := pool.GetObjectInfo(ctx, bucket, object, opts)
 		switch err.(type) {
+		case VersionNotFound:
+			// VersionId not found, versionId was specified
 		case ObjectNotFound:
 			// VersionId was not specified but found delete marker or no versions exist.
 		case MethodNotAllowed:
 			// VersionId was specified but found delete marker
 		default:
+			// All other unhandled errors return right here.
 			if err != nil {
-				// any other un-handled errors return right here.
 				return -1, err
 			}
 		}
@@ -531,6 +533,13 @@ func (z *erasureServerPools) GetObjectNInfo(ctx context.Context, bucket, object
 		}
 		return gr, nil
 	}
+	if isProxyable(ctx, bucket) {
+		// proxy to replication target if active-active replication is in place.
+		reader, proxy := proxyGetToReplicationTarget(ctx, bucket, object, rs, h, opts)
+		if reader != nil && proxy {
+			return reader, nil
+		}
+	}
 	if opts.VersionID != "" {
 		return gr, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID}
 	}
@@ -576,6 +585,13 @@ func (z *erasureServerPools) GetObjectInfo(ctx context.Context, bucket, object s
 		return objInfo, nil
 	}
 	object = decodeDirObject(object)
+	// proxy HEAD to replication target if active-active replication configured on bucket
+	if isProxyable(ctx, bucket) {
+		oi, proxy, err := proxyHeadToReplicationTarget(ctx, bucket, object, opts)
+		if proxy {
+			return oi, err
+		}
+	}
 	if opts.VersionID != "" {
 		return objInfo, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID}
 	}