Various improvements in replication (#11949)

- collect real time replication metrics for prometheus.
- add pending_count, failed_count metric for total pending/failed replication operations.

- add API to get replication metrics

- add MRF worker to handle spill-over replication operations

- multiple issues found with replication
- fixes an issue when client sends a bucket
 name with `/` at the end from SetRemoteTarget
 API call make sure to trim the bucket name to 
 avoid any extra `/`.

- hold write locks in GetObjectNInfo during replication
  to ensure that object version stack is not overwritten
  while reading the content.

- add additional protection during WriteMetadata() to
  ensure that we always write a valid FileInfo{} and avoid
  ever writing empty FileInfo{} to the lowest layers.

Co-authored-by: Poorna Krishnamoorthy <poorna@minio.io>
Co-authored-by: Harshavardhana <harsha@minio.io>

cmd/erasure-metadata.go

@@ -322,8 +322,12 @@ func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, bucket, prefix
 				return errDiskNotFound
 			}
 			// Pick one FileInfo for a disk at index.
-			files[index].Erasure.Index = index + 1
-			return disks[index].WriteMetadata(ctx, bucket, prefix, files[index])
+			fi := files[index]
+			fi.Erasure.Index = index + 1
+			if fi.IsValid() {
+				return disks[index].WriteMetadata(ctx, bucket, prefix, fi)
+			}
+			return errCorruptedFormat
 		}, index)
 	}