Add large bucket support for erasure coded backend (#5160)

This PR implements an object layer which
combines input erasure sets of XL layers
into a unified namespace.

This object layer extends the existing
erasure coded implementation, it is assumed
in this design that providing > 16 disks is
a static configuration as well i.e if you started
the setup with 32 disks with 4 sets 8 disks per
pack then you would need to provide 4 sets always.

Some design details and restrictions:

- Objects are distributed using consistent ordering
  to a unique erasure coded layer.
- Each pack has its own dsync so locks are synchronized
  properly at pack (erasure layer).
- Each pack still has a maximum of 16 disks
  requirement, you can start with multiple
  such sets statically.
- Static sets set of disks and cannot be
  changed, there is no elastic expansion allowed.
- Static sets set of disks and cannot be
  changed, there is no elastic removal allowed.
- ListObjects() across sets can be noticeably
  slower since List happens on all servers,
  and is merged at this sets layer.

Fixes #5465
Fixes #5464
Fixes #5461
Fixes #5460
Fixes #5459
Fixes #5458
Fixes #5460
Fixes #5488
Fixes #5489
Fixes #5497
Fixes #5496

cmd/bucket-handlers_test.go

@@ -35,7 +35,6 @@ func TestGetBucketLocationHandler(t *testing.T) {
 
 func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
 	credentials auth.Credentials, t *testing.T) {
-	initBucketPolicies(obj)
 
 	// test cases with sample input and expected output.
 	testCases := []struct {
@@ -180,7 +179,6 @@ func TestHeadBucketHandler(t *testing.T) {
 
 func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
 	credentials auth.Credentials, t *testing.T) {
-	initBucketPolicies(obj)
 
 	// test cases with sample input and expected output.
 	testCases := []struct {
@@ -287,7 +285,6 @@ func TestListMultipartUploadsHandler(t *testing.T) {
 // testListMultipartUploadsHandler - Tests validate listing of multipart uploads.
 func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
 	credentials auth.Credentials, t *testing.T) {
-	initBucketPolicies(obj)
 
 	// Collection of non-exhaustive ListMultipartUploads test cases, valid errors
 	// and success responses.
@@ -618,7 +615,6 @@ func TestAPIDeleteMultipleObjectsHandler(t *testing.T) {
 
 func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
 	credentials auth.Credentials, t *testing.T) {
-	initBucketPolicies(obj)
 
 	var err error
 	// register event notifier.
@@ -822,7 +818,6 @@ func testIsBucketActionAllowedHandler(obj ObjectLayer, instanceType, bucketName
 		{"s3:ListObject", "mybucket", "abc", false, false},
 	}
 	for i, testCase := range testCases {
-		initBucketPolicies(obj)
 		isAllowed := isBucketActionAllowed(testCase.action, testCase.bucket, testCase.prefix, obj)
 		if isAllowed != testCase.shouldPass {
 			t.Errorf("Case %d: Expected the response status to be `%t`, but instead found `%t`", i+1, testCase.shouldPass, isAllowed)