Add large bucket support for erasure coded backend (#5160)

This PR implements an object layer which
combines input erasure sets of XL layers
into a unified namespace.

This object layer extends the existing
erasure coded implementation, it is assumed
in this design that providing > 16 disks is
a static configuration as well i.e if you started
the setup with 32 disks with 4 sets 8 disks per
pack then you would need to provide 4 sets always.

Some design details and restrictions:

- Objects are distributed using consistent ordering
  to a unique erasure coded layer.
- Each pack has its own dsync so locks are synchronized
  properly at pack (erasure layer).
- Each pack still has a maximum of 16 disks
  requirement, you can start with multiple
  such sets statically.
- Static sets set of disks and cannot be
  changed, there is no elastic expansion allowed.
- Static sets set of disks and cannot be
  changed, there is no elastic removal allowed.
- ListObjects() across sets can be noticeably
  slower since List happens on all servers,
  and is merged at this sets layer.

Fixes #5465
Fixes #5464
Fixes #5461
Fixes #5460
Fixes #5459
Fixes #5458
Fixes #5460
Fixes #5488
Fixes #5489
Fixes #5497
Fixes #5496

cmd/fs-v1.go

@@ -164,7 +164,8 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) {
 	fs.fsFormatRlk = rlk
 
 	// Initialize and load bucket policies.
-	if err = initBucketPolicies(fs); err != nil {
+	fs.bucketPolicies, err = initBucketPolicies(fs)
+	if err != nil {
 		return nil, fmt.Errorf("Unable to load all bucket policies. %s", err)
 	}
 
@@ -174,6 +175,7 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) {
 	}
 
 	go fs.cleanupStaleMultipartUploads(multipartCleanupInterval, multipartExpiry, globalServiceDoneCh)
+
 	// Return successfully initialized object layer.
 	return fs, nil
 }
@@ -1048,6 +1050,11 @@ func (fs *fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKe
 	return result, nil
 }
 
+// HealFormat - no-op for fs, Valid only for XL.
+func (fs *fsObjects) HealFormat(dryRun bool) (madmin.HealResultItem, error) {
+	return madmin.HealResultItem{}, errors.Trace(NotImplemented{})
+}
+
 // HealObject - no-op for fs. Valid only for XL.
 func (fs *fsObjects) HealObject(bucket, object string, dryRun bool) (
 	res madmin.HealResultItem, err error) {