Unify gateway and object layer. (#5487)

* Unify gateway and object layer. Bring bucket policies into
object layer.

cmd/gateway/b2/gateway-b2-anonymous.go

@@ -1,136 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package b2
-
-import (
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/minio/minio/pkg/errors"
-
-	minio "github.com/minio/minio/cmd"
-)
-
-// mkRange converts offset, size into Range header equivalent.
-func mkRange(offset, size int64) string {
-	if offset == 0 && size == 0 {
-		return ""
-	}
-	if size == 0 {
-		return fmt.Sprintf("bytes=%d-", offset)
-	}
-	return fmt.Sprintf("bytes=%d-%d", offset, offset+size-1)
-}
-
-// AnonGetObject - performs a plain http GET request on a public resource,
-// fails if the resource is not public.
-func (l *b2Objects) AnonGetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
-	uri := fmt.Sprintf("%s/file/%s/%s", l.b2Client.DownloadURI, bucket, object)
-	req, err := http.NewRequest("GET", uri, nil)
-	if err != nil {
-		return b2ToObjectError(errors.Trace(err), bucket, object)
-	}
-	rng := mkRange(startOffset, length)
-	if rng != "" {
-		req.Header.Set("Range", rng)
-	}
-	resp, err := l.anonClient.Do(req)
-	if err != nil {
-		return b2ToObjectError(errors.Trace(err), bucket, object)
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		return b2ToObjectError(errors.Trace(fmt.Errorf(resp.Status)), bucket, object)
-	}
-	_, err = io.Copy(writer, resp.Body)
-	return b2ToObjectError(errors.Trace(err), bucket, object)
-}
-
-// Converts http Header into ObjectInfo. This function looks for all the
-// standard Backblaze B2 headers to convert into ObjectInfo.
-//
-// Content-Length is converted to Size.
-// X-Bz-Upload-Timestamp is converted to ModTime.
-// X-Bz-Info-<header>:<value> is converted to <header>:<value>
-// Content-Type is converted to ContentType.
-// X-Bz-Content-Sha1 is converted to ETag.
-func headerToObjectInfo(bucket, object string, header http.Header) (objInfo minio.ObjectInfo, err error) {
-	clen, err := strconv.ParseInt(header.Get("Content-Length"), 10, 64)
-	if err != nil {
-		return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
-	}
-
-	// Converting upload timestamp in milliseconds to a time.Time value for ObjectInfo.ModTime.
-	timeStamp, err := strconv.ParseInt(header.Get("X-Bz-Upload-Timestamp"), 10, 64)
-	if err != nil {
-		return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
-	}
-
-	// Populate user metadata by looking for all the X-Bz-Info-<name>
-	// HTTP headers, ignore other headers since they have their own
-	// designated meaning, for more details refer B2 API documentation.
-	userMetadata := make(map[string]string)
-	for key := range header {
-		if strings.HasPrefix(key, "X-Bz-Info-") {
-			var name string
-			name, err = url.QueryUnescape(strings.TrimPrefix(key, "X-Bz-Info-"))
-			if err != nil {
-				return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
-			}
-			var val string
-			val, err = url.QueryUnescape(header.Get(key))
-			if err != nil {
-				return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
-			}
-			userMetadata[name] = val
-		}
-	}
-
-	return minio.ObjectInfo{
-		Bucket:      bucket,
-		Name:        object,
-		ContentType: header.Get("Content-Type"),
-		ModTime:     time.Unix(0, 0).Add(time.Duration(timeStamp) * time.Millisecond),
-		Size:        clen,
-		ETag:        header.Get("X-Bz-File-Id"),
-		UserDefined: userMetadata,
-	}, nil
-}
-
-// AnonGetObjectInfo - performs a plain http HEAD request on a public resource,
-// fails if the resource is not public.
-func (l *b2Objects) AnonGetObjectInfo(bucket string, object string) (objInfo minio.ObjectInfo, err error) {
-	uri := fmt.Sprintf("%s/file/%s/%s", l.b2Client.DownloadURI, bucket, object)
-	req, err := http.NewRequest("HEAD", uri, nil)
-	if err != nil {
-		return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
-	}
-	resp, err := l.anonClient.Do(req)
-	if err != nil {
-		return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		return objInfo, b2ToObjectError(errors.Trace(fmt.Errorf(resp.Status)), bucket, object)
-	}
-	return headerToObjectInfo(bucket, object, resp.Header)
-}

cmd/gateway/b2/gateway-b2.go

@@ -24,7 +24,6 @@ import (
 	"io"
 	"io/ioutil"
 
-	"net/http"
 	"strings"
 	"sync"
 	"time"
@@ -96,9 +95,9 @@ func (g *B2) Name() string {
 	return b2Backend
 }
 
-// NewGatewayLayer returns b2 gateway layer, implements GatewayLayer interface to
+// NewGatewayLayer returns b2 gateway layer, implements ObjectLayer interface to
 // talk to B2 remote backend.
-func (g *B2) NewGatewayLayer(creds auth.Credentials) (minio.GatewayLayer, error) {
+func (g *B2) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
 	ctx := context.Background()
 	client, err := b2.AuthorizeAccount(ctx, creds.AccessKey, creds.SecretKey, b2.Transport(minio.NewCustomHTTPTransport()))
 	if err != nil {
@@ -108,10 +107,7 @@ func (g *B2) NewGatewayLayer(creds auth.Credentials) (minio.GatewayLayer, error)
 	return &b2Objects{
 		creds:    creds,
 		b2Client: client,
-		anonClient: &http.Client{
-			Transport: minio.NewCustomHTTPTransport(),
-		},
-		ctx: ctx,
+		ctx:      ctx,
 	}, nil
 }
 
@@ -124,11 +120,10 @@ func (g *B2) Production() bool {
 // b2Object implements gateway for Minio and BackBlaze B2 compatible object storage servers.
 type b2Objects struct {
 	minio.GatewayUnsupported
-	mu         sync.Mutex
-	creds      auth.Credentials
-	b2Client   *b2.B2
-	anonClient *http.Client
-	ctx        context.Context
+	mu       sync.Mutex
+	creds    auth.Credentials
+	b2Client *b2.B2
+	ctx      context.Context
 }
 
 // Convert B2 errors to minio object layer errors.
@@ -694,11 +689,11 @@ func (l *b2Objects) CompleteMultipartUpload(bucket string, object string, upload
 	return l.GetObjectInfo(bucket, object)
 }
 
-// SetBucketPolicies - B2 supports 2 types of bucket policies:
+// SetBucketPolicy - B2 supports 2 types of bucket policies:
 // bucketType.AllPublic - bucketTypeReadOnly means that anybody can download the files is the bucket;
 // bucketType.AllPrivate - bucketTypePrivate means that you need an authorization token to download them.
 // Default is AllPrivate for all buckets.
-func (l *b2Objects) SetBucketPolicies(bucket string, policyInfo policy.BucketAccessPolicy) error {
+func (l *b2Objects) SetBucketPolicy(bucket string, policyInfo policy.BucketAccessPolicy) error {
 	var policies []minio.BucketAccessPolicy
 
 	for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket) {
@@ -726,9 +721,9 @@ func (l *b2Objects) SetBucketPolicies(bucket string, policyInfo policy.BucketAcc
 	return b2ToObjectError(errors.Trace(err))
 }
 
-// GetBucketPolicies, returns the current bucketType from B2 backend and convert
+// GetBucketPolicy, returns the current bucketType from B2 backend and convert
 // it into S3 compatible bucket policy info.
-func (l *b2Objects) GetBucketPolicies(bucket string) (policy.BucketAccessPolicy, error) {
+func (l *b2Objects) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, error) {
 	policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"}
 	bkt, err := l.Bucket(bucket)
 	if err != nil {
@@ -744,8 +739,8 @@ func (l *b2Objects) GetBucketPolicies(bucket string) (policy.BucketAccessPolicy,
 	return policy.BucketAccessPolicy{}, errors.Trace(minio.PolicyNotFound{Bucket: bucket})
 }
 
-// DeleteBucketPolicies - resets the bucketType of bucket on B2 to 'allPrivate'.
-func (l *b2Objects) DeleteBucketPolicies(bucket string) error {
+// DeleteBucketPolicy - resets the bucketType of bucket on B2 to 'allPrivate'.
+func (l *b2Objects) DeleteBucketPolicy(bucket string) error {
 	bkt, err := l.Bucket(bucket)
 	if err != nil {
 		return err

cmd/gateway/b2/gateway-b2_test.go

@@ -18,7 +18,6 @@ package b2
 
 import (
 	"fmt"
-	"net/http"
 	"testing"
 
 	b2 "github.com/minio/blazer/base"
@@ -27,88 +26,6 @@ import (
 	minio "github.com/minio/minio/cmd"
 )
 
-// Tests headerToObjectInfo
-func TestHeaderToObjectInfo(t *testing.T) {
-	testCases := []struct {
-		bucket, object string
-		header         http.Header
-		objInfo        minio.ObjectInfo
-	}{
-		{
-			bucket: "bucket",
-			object: "object",
-			header: http.Header{
-				"Content-Length":         []string{"10"},
-				"Content-Type":           []string{"application/javascript"},
-				"X-Bz-Upload-Timestamp":  []string{"1000"},
-				"X-Bz-Info-X-Amz-Meta-1": []string{"test1"},
-				"X-Bz-File-Id":           []string{"xxxxx"},
-			},
-			objInfo: minio.ObjectInfo{
-				Bucket:      "bucket",
-				Name:        "object",
-				ContentType: "application/javascript",
-				Size:        10,
-				UserDefined: map[string]string{
-					"X-Amz-Meta-1": "test1",
-				},
-				ETag: "xxxxx",
-			},
-		},
-	}
-	for i, testCase := range testCases {
-		gotObjInfo, err := headerToObjectInfo(testCase.bucket, testCase.object, testCase.header)
-		if err != nil {
-			t.Fatalf("Test %d: %s", i+1, err)
-		}
-		if gotObjInfo.Bucket != testCase.objInfo.Bucket {
-			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.objInfo.Bucket, gotObjInfo.Bucket)
-		}
-		if gotObjInfo.Name != testCase.objInfo.Name {
-			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.objInfo.Name, gotObjInfo.Name)
-		}
-		if gotObjInfo.ContentType != testCase.objInfo.ContentType {
-			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.objInfo.ContentType, gotObjInfo.ContentType)
-		}
-		if gotObjInfo.ETag != testCase.objInfo.ETag {
-			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.objInfo.ETag, gotObjInfo.ETag)
-		}
-	}
-}
-
-// Tests mkRange test.
-func TestMkRange(t *testing.T) {
-	testCases := []struct {
-		offset, size int64
-		expectedRng  string
-	}{
-		// No offset set, size not set.
-		{
-			offset:      0,
-			size:        0,
-			expectedRng: "",
-		},
-		// Offset set, size not set.
-		{
-			offset:      10,
-			size:        0,
-			expectedRng: "bytes=10-",
-		},
-		// Offset set, size set.
-		{
-			offset:      10,
-			size:        11,
-			expectedRng: "bytes=10-20",
-		},
-	}
-	for i, testCase := range testCases {
-		gotRng := mkRange(testCase.offset, testCase.size)
-		if gotRng != testCase.expectedRng {
-			t.Errorf("Test %d: expected %s, got %s", i+1, testCase.expectedRng, gotRng)
-		}
-	}
-}
-
 // Test b2 object error.
 func TestB2ObjectError(t *testing.T) {
 	testCases := []struct {