Add large bucket support for erasure coded backend (#5160)

This PR implements an object layer which
combines input erasure sets of XL layers
into a unified namespace.

This object layer extends the existing
erasure coded implementation, it is assumed
in this design that providing > 16 disks is
a static configuration as well i.e if you started
the setup with 32 disks with 4 sets 8 disks per
pack then you would need to provide 4 sets always.

Some design details and restrictions:

- Objects are distributed using consistent ordering
  to a unique erasure coded layer.
- Each pack has its own dsync so locks are synchronized
  properly at pack (erasure layer).
- Each pack still has a maximum of 16 disks
  requirement, you can start with multiple
  such sets statically.
- Static sets set of disks and cannot be
  changed, there is no elastic expansion allowed.
- Static sets set of disks and cannot be
  changed, there is no elastic removal allowed.
- ListObjects() across sets can be noticeably
  slower since List happens on all servers,
  and is merged at this sets layer.

Fixes #5465
Fixes #5464
Fixes #5461
Fixes #5460
Fixes #5459
Fixes #5458
Fixes #5460
Fixes #5488
Fixes #5489
Fixes #5497
Fixes #5496

pkg/madmin/heal-commands.go

@@ -1,10 +1,8 @@
 /*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ * Minio Cloud Storage, (C) 2017, 2018 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
-
-
  * You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
@@ -75,6 +73,13 @@ const (
 	DriveStateMissing        = "missing"
 )
 
+// HealDriveInfo - struct for an individual drive info item.
+type HealDriveInfo struct {
+	UUID     string `json:"uuid"`
+	Endpoint string `json:"endpoint"`
+	State    string `json:"state"`
+}
+
 // HealResultItem - struct for an individual heal result item
 type HealResultItem struct {
 	ResultIndex  int64        `json:"resultId"`
@@ -85,33 +90,87 @@ type HealResultItem struct {
 	ParityBlocks int          `json:"parityBlocks,omitempty"`
 	DataBlocks   int          `json:"dataBlocks,omitempty"`
 	DiskCount    int          `json:"diskCount"`
-	DriveInfo    struct {
-		// below maps are from drive endpoint to drive state
-		Before map[string]string `json:"before"`
-		After  map[string]string `json:"after"`
-	} `json:"drives"`
+	SetCount     int          `json:"setCount"`
+	// below slices are from drive info.
+	Before struct {
+		Drives []HealDriveInfo `json:"drives"`
+	} `json:"before"`
+	After struct {
+		Drives []HealDriveInfo `json:"drives"`
+	} `json:"after"`
 	ObjectSize int64 `json:"objectSize"`
 }
 
-// InitDrives - initialize maps used to represent drive info
-func (hri *HealResultItem) InitDrives() {
-	hri.DriveInfo.Before = make(map[string]string)
-	hri.DriveInfo.After = make(map[string]string)
+// GetMissingCounts - returns the number of missing disks before
+// and after heal
+func (hri *HealResultItem) GetMissingCounts() (b, a int) {
+	if hri == nil {
+		return
+	}
+	for _, v := range hri.Before.Drives {
+		if v.State == DriveStateMissing {
+			b++
+		}
+	}
+	for _, v := range hri.After.Drives {
+		if v.State == DriveStateMissing {
+			a++
+		}
+	}
+	return
 }
 
-// GetOnlineCounts - returns the number of online disks before and
-// after heal
+// GetOfflineCounts - returns the number of offline disks before
+// and after heal
+func (hri *HealResultItem) GetOfflineCounts() (b, a int) {
+	if hri == nil {
+		return
+	}
+	for _, v := range hri.Before.Drives {
+		if v.State == DriveStateOffline {
+			b++
+		}
+	}
+	for _, v := range hri.After.Drives {
+		if v.State == DriveStateOffline {
+			a++
+		}
+	}
+	return
+}
+
+// GetCorruptedCounts - returns the number of corrupted disks before
+// and after heal
+func (hri *HealResultItem) GetCorruptedCounts() (b, a int) {
+	if hri == nil {
+		return
+	}
+	for _, v := range hri.Before.Drives {
+		if v.State == DriveStateCorrupt {
+			b++
+		}
+	}
+	for _, v := range hri.After.Drives {
+		if v.State == DriveStateCorrupt {
+			a++
+		}
+	}
+	return
+}
+
+// GetOnlineCounts - returns the number of online disks before
+// and after heal
 func (hri *HealResultItem) GetOnlineCounts() (b, a int) {
 	if hri == nil {
 		return
 	}
-	for _, v := range hri.DriveInfo.Before {
-		if v == DriveStateOk {
+	for _, v := range hri.Before.Drives {
+		if v.State == DriveStateOk {
 			b++
 		}
 	}
-	for _, v := range hri.DriveInfo.After {
-		if v == DriveStateOk {
+	for _, v := range hri.After.Drives {
+		if v.State == DriveStateOk {
 			a++
 		}
 	}

pkg/madmin/heal-commands_test.go

@@ -0,0 +1,73 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package madmin
+
+import (
+	"testing"
+)
+
+// Tests heal drives missing and offline counts.
+func TestHealDriveCounts(t *testing.T) {
+	rs := HealResultItem{}
+	rs.Before.Drives = make([]HealDriveInfo, 20)
+	rs.After.Drives = make([]HealDriveInfo, 20)
+	for i := range rs.Before.Drives {
+		if i < 4 {
+			rs.Before.Drives[i] = HealDriveInfo{State: DriveStateMissing}
+			rs.After.Drives[i] = HealDriveInfo{State: DriveStateMissing}
+		} else if i > 4 && i < 15 {
+			rs.Before.Drives[i] = HealDriveInfo{State: DriveStateOffline}
+			rs.After.Drives[i] = HealDriveInfo{State: DriveStateOffline}
+		} else if i > 15 {
+			rs.Before.Drives[i] = HealDriveInfo{State: DriveStateCorrupt}
+			rs.After.Drives[i] = HealDriveInfo{State: DriveStateCorrupt}
+		} else {
+			rs.Before.Drives[i] = HealDriveInfo{State: DriveStateOk}
+			rs.After.Drives[i] = HealDriveInfo{State: DriveStateOk}
+		}
+	}
+
+	i, j := rs.GetOnlineCounts()
+	if i > 2 {
+		t.Errorf("Expected '2', got %d before online disks", i)
+	}
+	if j > 2 {
+		t.Errorf("Expected '2', got %d after online disks", j)
+	}
+	i, j = rs.GetOfflineCounts()
+	if i > 10 {
+		t.Errorf("Expected '10', got %d before offline disks", i)
+	}
+	if j > 10 {
+		t.Errorf("Expected '10', got %d after offline disks", j)
+	}
+	i, j = rs.GetCorruptedCounts()
+	if i > 4 {
+		t.Errorf("Expected '4', got %d before corrupted disks", i)
+	}
+	if j > 4 {
+		t.Errorf("Expected '4', got %d after corrupted disks", j)
+	}
+	i, j = rs.GetMissingCounts()
+	if i > 4 {
+		t.Errorf("Expected '4', got %d before missing disks", i)
+	}
+	if j > 4 {
+		t.Errorf("Expected '4', got %d after missing disks", i)
+	}
+}

pkg/madmin/info-commands.go

@@ -38,6 +38,10 @@ const (
 	// Add your own backend.
 )
 
+// DriveInfo - represents each drive info, describing
+// status, uuid and endpoint.
+type DriveInfo HealDriveInfo
+
 // StorageInfo - represents total capacity of underlying storage.
 type StorageInfo struct {
 	// Total disk space.
@@ -52,8 +56,13 @@ type StorageInfo struct {
 		// Following fields are only meaningful if BackendType is Erasure.
 		OnlineDisks      int // Online disks during server startup.
 		OfflineDisks     int // Offline disks during server startup.
+		StandardSCData   int // Data disks for currently configured Standard storage class.
 		StandardSCParity int // Parity disks for currently configured Standard storage class.
+		RRSCData         int // Data disks for currently configured Reduced Redundancy storage class.
 		RRSCParity       int // Parity disks for currently configured Reduced Redundancy storage class.
+
+		// List of all disk status, this is only meaningful if BackendType is Erasure.
+		Sets [][]DriveInfo
 	}
 }