Various improvements in replication (#11949)

- collect real time replication metrics for prometheus.
- add pending_count, failed_count metric for total pending/failed replication operations.

- add API to get replication metrics

- add MRF worker to handle spill-over replication operations

- multiple issues found with replication
- fixes an issue when client sends a bucket
 name with `/` at the end from SetRemoteTarget
 API call make sure to trim the bucket name to 
 avoid any extra `/`.

- hold write locks in GetObjectNInfo during replication
  to ensure that object version stack is not overwritten
  while reading the content.

- add additional protection during WriteMetadata() to
  ensure that we always write a valid FileInfo{} and avoid
  ever writing empty FileInfo{} to the lowest layers.

Co-authored-by: Poorna Krishnamoorthy <poorna@minio.io>
Co-authored-by: Harshavardhana <harsha@minio.io>

cmd/config/api/api.go

@@ -29,14 +29,15 @@ import (
 
 // API sub-system constants
 const (
-	apiRequestsMax                = "requests_max"
-	apiRequestsDeadline           = "requests_deadline"
-	apiClusterDeadline            = "cluster_deadline"
-	apiCorsAllowOrigin            = "cors_allow_origin"
-	apiRemoteTransportDeadline    = "remote_transport_deadline"
-	apiListQuorum                 = "list_quorum"
-	apiExtendListCacheLife        = "extend_list_cache_life"
-	apiReplicationWorkers         = "replication_workers"
+	apiRequestsMax             = "requests_max"
+	apiRequestsDeadline        = "requests_deadline"
+	apiClusterDeadline         = "cluster_deadline"
+	apiCorsAllowOrigin         = "cors_allow_origin"
+	apiRemoteTransportDeadline = "remote_transport_deadline"
+	apiListQuorum              = "list_quorum"
+	apiExtendListCacheLife     = "extend_list_cache_life"
+	apiReplicationWorkers      = "replication_workers"
+
 	EnvAPIRequestsMax             = "MINIO_API_REQUESTS_MAX"
 	EnvAPIRequestsDeadline        = "MINIO_API_REQUESTS_DEADLINE"
 	EnvAPIClusterDeadline         = "MINIO_API_CLUSTER_DEADLINE"
@@ -87,7 +88,7 @@ var (
 		},
 		config.KV{
 			Key:   apiReplicationWorkers,
-			Value: "100",
+			Value: "500",
 		},
 	}
 )