Various improvements in replication (#11949)

- collect real time replication metrics for prometheus.
- add pending_count, failed_count metric for total pending/failed replication operations.

- add API to get replication metrics

- add MRF worker to handle spill-over replication operations

- multiple issues found with replication
- fixes an issue when client sends a bucket
 name with `/` at the end from SetRemoteTarget
 API call make sure to trim the bucket name to 
 avoid any extra `/`.

- hold write locks in GetObjectNInfo during replication
  to ensure that object version stack is not overwritten
  while reading the content.

- add additional protection during WriteMetadata() to
  ensure that we always write a valid FileInfo{} and avoid
  ever writing empty FileInfo{} to the lowest layers.

Co-authored-by: Poorna Krishnamoorthy <poorna@minio.io>
Co-authored-by: Harshavardhana <harsha@minio.io>

pkg/madmin/README.md

@@ -258,15 +258,15 @@ Fetches accounting usage information for the current authenticated user
 | Param                          | Type                 | Description             |
 |--------------------------------|----------------------|-------------------------|
 | `AccountInfo.AccountName` | _string_             | Account name.           |
-| `AccountInfo.Buckets`     | _[]BucketUsageInfo_  | Bucket usage info.      |
+| `AccountInfo.Buckets`     | _[]BucketAccessInfo_  | Bucket usage info.      |
 
 
 | Param                      | Type            | Description                             |
 |----------------------------|-----------------|-----------------------------------------|
-| `BucketUsageInfo.Name`     | _string_        | The name of the current bucket
-| `BucketUsageInfo.Size`     | _uint64_        | The total size of the current bucket
-| `BucketUsageInfo.Created`  | _time.Time_     | Bucket creation time
-| `BucketUsageInfo.Access`   | _AccountAccess_ | Type of access of the current account
+| `BucketAccessInfo.Name`     | _string_        | The name of the current bucket
+| `BucketAccessInfo.Size`     | _uint64_        | The total size of the current bucket
+| `BucketAccessInfo.Created`  | _time.Time_     | Bucket creation time
+| `BucketAccessInfo.Access`   | _AccountAccess_ | Type of access of the current account
 
 
 | Param                  | Type    | Description                                                                                                              |

pkg/madmin/info-commands.go

@@ -20,7 +20,6 @@ package madmin
 import (
 	"context"
 	"encoding/json"
-	"io/ioutil"
 	"net/http"
 	"time"
 )
@@ -124,36 +123,71 @@ func (adm *AdminClient) StorageInfo(ctx context.Context) (StorageInfo, error) {
 
 	// Unmarshal the server's json response
 	var storageInfo StorageInfo
-
-	respBytes, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return StorageInfo{}, err
-	}
-
-	err = json.Unmarshal(respBytes, &storageInfo)
-	if err != nil {
+	if err = json.NewDecoder(resp.Body).Decode(&storageInfo); err != nil {
 		return StorageInfo{}, err
 	}
 
 	return storageInfo, nil
 }
 
-// DataUsageInfo represents data usage of an Object API
+// BucketUsageInfo - bucket usage info provides
+// - total size of the bucket
+// - total objects in a bucket
+// - object size histogram per bucket
+type BucketUsageInfo struct {
+	Size                    uint64 `json:"size"`
+	ReplicationPendingSize  uint64 `json:"objectsPendingReplicationTotalSize"`
+	ReplicationFailedSize   uint64 `json:"objectsFailedReplicationTotalSize"`
+	ReplicatedSize          uint64 `json:"objectsReplicatedTotalSize"`
+	ReplicaSize             uint64 `json:"objectReplicaTotalSize"`
+	ReplicationPendingCount uint64 `json:"objectsPendingReplicationCount"`
+	ReplicationFailedCount  uint64 `json:"objectsFailedReplicationCount"`
+
+	ObjectsCount         uint64            `json:"objectsCount"`
+	ObjectSizesHistogram map[string]uint64 `json:"objectsSizesHistogram"`
+}
+
+// DataUsageInfo represents data usage stats of the underlying Object API
 type DataUsageInfo struct {
 	// LastUpdate is the timestamp of when the data usage info was last updated.
 	// This does not indicate a full scan.
-	LastUpdate       time.Time `json:"lastUpdate"`
-	ObjectsCount     uint64    `json:"objectsCount"`
-	ObjectsTotalSize uint64    `json:"objectsTotalSize"`
+	LastUpdate time.Time `json:"lastUpdate"`
 
-	// ObjectsSizesHistogram contains information on objects across all buckets.
-	// See ObjectsHistogramIntervals.
-	ObjectsSizesHistogram map[string]uint64 `json:"objectsSizesHistogram"`
+	// Objects total count across all buckets
+	ObjectsTotalCount uint64 `json:"objectsCount"`
 
+	// Objects total size across all buckets
+	ObjectsTotalSize uint64 `json:"objectsTotalSize"`
+
+	// Total Size for objects that have not yet been replicated
+	ReplicationPendingSize uint64 `json:"objectsPendingReplicationTotalSize"`
+
+	// Total size for objects that have witness one or more failures and will be retried
+	ReplicationFailedSize uint64 `json:"objectsFailedReplicationTotalSize"`
+
+	// Total size for objects that have been replicated to destination
+	ReplicatedSize uint64 `json:"objectsReplicatedTotalSize"`
+
+	// Total size for objects that are replicas
+	ReplicaSize uint64 `json:"objectsReplicaTotalSize"`
+
+	// Total number of objects pending replication
+	ReplicationPendingCount uint64 `json:"objectsPendingReplicationCount"`
+
+	// Total number of objects that failed replication
+	ReplicationFailedCount uint64 `json:"objectsFailedReplicationCount"`
+
+	// Total number of buckets in this cluster
 	BucketsCount uint64 `json:"bucketsCount"`
 
-	// BucketsSizes is "bucket name" -> size.
-	BucketsSizes map[string]uint64 `json:"bucketsSizes"`
+	// Buckets usage info provides following information across all buckets
+	// - total size of the bucket
+	// - total objects in a bucket
+	// - object size histogram per bucket
+	BucketsUsage map[string]BucketUsageInfo `json:"bucketsUsageInfo"`
+
+	// Deprecated kept here for backward compatibility reasons.
+	BucketSizes map[string]uint64 `json:"bucketsSizes"`
 }
 
 // DataUsageInfo - returns data usage of the current object API
@@ -171,14 +205,7 @@ func (adm *AdminClient) DataUsageInfo(ctx context.Context) (DataUsageInfo, error
 
 	// Unmarshal the server's json response
 	var dataUsageInfo DataUsageInfo
-
-	respBytes, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return DataUsageInfo{}, err
-	}
-
-	err = json.Unmarshal(respBytes, &dataUsageInfo)
-	if err != nil {
+	if err = json.NewDecoder(resp.Body).Decode(&dataUsageInfo); err != nil {
 		return DataUsageInfo{}, err
 	}
 
@@ -344,14 +371,7 @@ func (adm *AdminClient) ServerInfo(ctx context.Context) (InfoMessage, error) {
 
 	// Unmarshal the server's json response
 	var message InfoMessage
-
-	respBytes, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return InfoMessage{}, err
-	}
-
-	err = json.Unmarshal(respBytes, &message)
-	if err != nil {
+	if err = json.NewDecoder(resp.Body).Decode(&message); err != nil {
 		return InfoMessage{}, err
 	}
 

pkg/madmin/update-commands.go

@@ -20,7 +20,6 @@ package madmin
 import (
 	"context"
 	"encoding/json"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 )
@@ -53,10 +52,9 @@ func (adm *AdminClient) ServerUpdate(ctx context.Context, updateURL string) (us
 		return us, httpRespToErrorResponse(resp)
 	}
 
-	buf, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
+	if err = json.NewDecoder(resp.Body).Decode(&us); err != nil {
 		return us, err
 	}
-	err = json.Unmarshal(buf, &us)
-	return us, err
+
+	return us, nil
 }

pkg/madmin/user-commands.go

@@ -35,9 +35,9 @@ type AccountAccess struct {
 	Write bool `json:"write"`
 }
 
-// BucketUsageInfo represents bucket usage of a bucket, and its relevant
+// BucketAccessInfo represents bucket usage of a bucket, and its relevant
 // access type for an account
-type BucketUsageInfo struct {
+type BucketAccessInfo struct {
 	Name    string        `json:"name"`
 	Size    uint64        `json:"size"`
 	Created time.Time     `json:"created"`
@@ -49,7 +49,7 @@ type BucketUsageInfo struct {
 type AccountInfo struct {
 	AccountName string
 	Policy      iampolicy.Policy
-	Buckets     []BucketUsageInfo
+	Buckets     []BucketAccessInfo
 }
 
 // AccountInfo returns the usage info for the authenticating account.