Add large bucket support for erasure coded backend (#5160)

This PR implements an object layer which
combines input erasure sets of XL layers
into a unified namespace.

This object layer extends the existing
erasure coded implementation, it is assumed
in this design that providing > 16 disks is
a static configuration as well i.e if you started
the setup with 32 disks with 4 sets 8 disks per
pack then you would need to provide 4 sets always.

Some design details and restrictions:

- Objects are distributed using consistent ordering
  to a unique erasure coded layer.
- Each pack has its own dsync so locks are synchronized
  properly at pack (erasure layer).
- Each pack still has a maximum of 16 disks
  requirement, you can start with multiple
  such sets statically.
- Static sets set of disks and cannot be
  changed, there is no elastic expansion allowed.
- Static sets set of disks and cannot be
  changed, there is no elastic removal allowed.
- ListObjects() across sets can be noticeably
  slower since List happens on all servers,
  and is merged at this sets layer.

Fixes #5465
Fixes #5464
Fixes #5461
Fixes #5460
Fixes #5459
Fixes #5458
Fixes #5460
Fixes #5488
Fixes #5489
Fixes #5497
Fixes #5496

pkg/sync/errgroup/errgroup.go

@@ -0,0 +1,59 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package errgroup
+
+import (
+	"sync"
+)
+
+// A Group is a collection of goroutines working on subtasks that are part of
+// the same overall task.
+//
+// A zero Group is valid and does not cancel on error.
+type Group struct {
+	wg   sync.WaitGroup
+	errs []error
+}
+
+// WithNErrs returns a new Group with length of errs slice upto nerrs,
+// upon Wait() errors are returned collected from all tasks.
+func WithNErrs(nerrs int) *Group {
+	return &Group{errs: make([]error, nerrs)}
+}
+
+// Wait blocks until all function calls from the Go method have returned, then
+// returns the slice of errors from all function calls.
+func (g *Group) Wait() []error {
+	g.wg.Wait()
+	return g.errs
+}
+
+// Go calls the given function in a new goroutine.
+//
+// The first call to return a non-nil error will be
+// collected in errs slice and returned by Wait().
+func (g *Group) Go(f func() error, index int) {
+	g.wg.Add(1)
+
+	go func() {
+		defer g.wg.Done()
+
+		if err := f(); err != nil {
+			g.errs[index] = err
+		}
+	}()
+}

pkg/sync/errgroup/errgroup_test.go

@@ -0,0 +1,52 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package errgroup
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+func TestGroupWithNErrs(t *testing.T) {
+	err1 := fmt.Errorf("errgroup_test: 1")
+	err2 := fmt.Errorf("errgroup_test: 2")
+
+	cases := []struct {
+		errs []error
+	}{
+		{errs: []error{nil}},
+		{errs: []error{err1}},
+		{errs: []error{err1, nil}},
+		{errs: []error{err1, nil, err2}},
+	}
+
+	for j, tc := range cases {
+		t.Run(fmt.Sprintf("Test%d", j+1), func(t *testing.T) {
+			g := WithNErrs(len(tc.errs))
+			for i, err := range tc.errs {
+				err := err
+				g.Go(func() error { return err }, i)
+			}
+
+			gotErrs := g.Wait()
+			if !reflect.DeepEqual(gotErrs, tc.errs) {
+				t.Errorf("Expected %#v, got %#v", tc.errs, gotErrs)
+			}
+		})
+	}
+}