Add large bucket support for erasure coded backend (#5160)

This PR implements an object layer which
combines input erasure sets of XL layers
into a unified namespace.

This object layer extends the existing
erasure coded implementation, it is assumed
in this design that providing > 16 disks is
a static configuration as well i.e if you started
the setup with 32 disks with 4 sets 8 disks per
pack then you would need to provide 4 sets always.

Some design details and restrictions:

- Objects are distributed using consistent ordering
  to a unique erasure coded layer.
- Each pack has its own dsync so locks are synchronized
  properly at pack (erasure layer).
- Each pack still has a maximum of 16 disks
  requirement, you can start with multiple
  such sets statically.
- Static sets set of disks and cannot be
  changed, there is no elastic expansion allowed.
- Static sets set of disks and cannot be
  changed, there is no elastic removal allowed.
- ListObjects() across sets can be noticeably
  slower since List happens on all servers,
  and is merged at this sets layer.

Fixes #5465
Fixes #5464
Fixes #5461
Fixes #5460
Fixes #5459
Fixes #5458
Fixes #5460
Fixes #5488
Fixes #5489
Fixes #5497
Fixes #5496

cmd/bucket-policy.go

@@ -76,14 +76,15 @@ func (bp *bucketPolicies) DeleteBucketPolicy(bucket string) error {
 }
 
 // Intialize all bucket policies.
-func initBucketPolicies(objAPI ObjectLayer) error {
+func initBucketPolicies(objAPI ObjectLayer) (*bucketPolicies, error) {
 	if objAPI == nil {
-		return errInvalidArgument
+		return nil, errInvalidArgument
 	}
+
 	// List buckets to proceed loading all notification configuration.
 	buckets, err := objAPI.ListBuckets()
 	if err != nil {
-		return errors.Cause(err)
+		return nil, errors.Cause(err)
 	}
 
 	policies := make(map[string]policy.BucketAccessPolicy)
@@ -95,7 +96,7 @@ func initBucketPolicies(objAPI ObjectLayer) error {
 			// other unexpected errors during net.Dial.
 			if !errors.IsErrIgnored(pErr, errDiskNotFound) {
 				if !isErrBucketPolicyNotFound(pErr) {
-					return errors.Cause(pErr)
+					return nil, errors.Cause(pErr)
 				}
 			}
 			// Continue to load other bucket policies if possible.
@@ -103,20 +104,12 @@ func initBucketPolicies(objAPI ObjectLayer) error {
 		}
 		policies[bucket.Name] = bp
 	}
-	// Populate global bucket collection.
-	bPolicies := &bucketPolicies{
+
+	// Return all bucket policies.
+	return &bucketPolicies{
 		rwMutex:             &sync.RWMutex{},
 		bucketPolicyConfigs: policies,
-	}
-	switch objAPI.(type) {
-	case *fsObjects:
-		objAPI.(*fsObjects).bucketPolicies = bPolicies
-	case *xlObjects:
-		objAPI.(*xlObjects).bucketPolicies = bPolicies
-	}
-
-	// Success.
-	return nil
+	}, nil
 }
 
 // readBucketPolicyJSON - reads bucket policy for an input bucket, returns BucketPolicyNotFound