Unify gateway and object layer. (#5487)

* Unify gateway and object layer. Bring bucket policies into
object layer.

cmd/gateway/gcs/gateway-gcs-anonymous.go

@@ -1,146 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package gcs
-
-import (
-	"fmt"
-	"io"
-	"net/http"
-	"strconv"
-	"time"
-
-	"github.com/minio/minio/pkg/errors"
-
-	minio "github.com/minio/minio/cmd"
-)
-
-func toGCSPublicURL(bucket, object string) string {
-	return fmt.Sprintf("https://storage.googleapis.com/%s/%s", bucket, object)
-}
-
-// AnonGetObject - Get object anonymously
-func (l *gcsGateway) AnonGetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
-	req, err := http.NewRequest("GET", toGCSPublicURL(bucket, object), nil)
-	if err != nil {
-		return gcsToObjectError(errors.Trace(err), bucket, object)
-	}
-
-	if length > 0 && startOffset > 0 {
-		req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", startOffset, startOffset+length-1))
-	} else if startOffset > 0 {
-		req.Header.Add("Range", fmt.Sprintf("bytes=%d-", startOffset))
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return gcsToObjectError(errors.Trace(err), bucket, object)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusPartialContent && resp.StatusCode != http.StatusOK {
-		return gcsToObjectError(errors.Trace(minio.AnonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
-	}
-
-	_, err = io.Copy(writer, resp.Body)
-	return gcsToObjectError(errors.Trace(err), bucket, object)
-}
-
-// AnonGetObjectInfo - Get object info anonymously
-func (l *gcsGateway) AnonGetObjectInfo(bucket string, object string) (objInfo minio.ObjectInfo, err error) {
-	resp, err := http.Head(toGCSPublicURL(bucket, object))
-	if err != nil {
-		return objInfo, gcsToObjectError(errors.Trace(err), bucket, object)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return objInfo, gcsToObjectError(errors.Trace(minio.AnonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
-	}
-
-	var contentLength int64
-	contentLengthStr := resp.Header.Get("Content-Length")
-	if contentLengthStr != "" {
-		contentLength, err = strconv.ParseInt(contentLengthStr, 0, 64)
-		if err != nil {
-			return objInfo, gcsToObjectError(errors.Trace(fmt.Errorf("Unexpected error")), bucket, object)
-		}
-	}
-
-	t, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
-	if err != nil {
-		return objInfo, errors.Trace(err)
-	}
-
-	objInfo.ModTime = t
-	objInfo.Bucket = bucket
-	objInfo.UserDefined = make(map[string]string)
-	if resp.Header.Get("Content-Encoding") != "" {
-		objInfo.UserDefined["Content-Encoding"] = resp.Header.Get("Content-Encoding")
-	}
-	objInfo.UserDefined["Content-Type"] = resp.Header.Get("Content-Type")
-	objInfo.ETag = resp.Header.Get("Etag")
-	objInfo.ModTime = t
-	objInfo.Name = object
-	objInfo.Size = contentLength
-	return
-}
-
-// AnonListObjects - List objects anonymously
-func (l *gcsGateway) AnonListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (minio.ListObjectsInfo, error) {
-	result, err := l.anonClient.ListObjects(bucket, prefix, marker, delimiter, maxKeys)
-	if err != nil {
-		return minio.ListObjectsInfo{}, minio.ErrorRespToObjectError(errors.Trace(err), bucket)
-	}
-
-	return minio.FromMinioClientListBucketResult(bucket, result), nil
-}
-
-// AnonListObjectsV2 - List objects in V2 mode, anonymously
-func (l *gcsGateway) AnonListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (minio.ListObjectsV2Info, error) {
-	// Request V1 List Object to the backend
-	result, err := l.anonClient.ListObjects(bucket, prefix, continuationToken, delimiter, maxKeys)
-	if err != nil {
-		return minio.ListObjectsV2Info{}, minio.ErrorRespToObjectError(errors.Trace(err), bucket)
-	}
-	// translate V1 Result to V2Info
-	return minio.FromMinioClientListBucketResultToV2Info(bucket, result), nil
-}
-
-// AnonGetBucketInfo - Get bucket metadata anonymously.
-func (l *gcsGateway) AnonGetBucketInfo(bucket string) (bucketInfo minio.BucketInfo, err error) {
-	resp, err := http.Head(toGCSPublicURL(bucket, ""))
-	if err != nil {
-		return bucketInfo, gcsToObjectError(errors.Trace(err))
-	}
-
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return bucketInfo, gcsToObjectError(errors.Trace(minio.AnonErrToObjectErr(resp.StatusCode, bucket)), bucket)
-	}
-
-	t, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
-	if err != nil {
-		return bucketInfo, errors.Trace(err)
-	}
-
-	// Last-Modified date being returned by GCS
-	return minio.BucketInfo{
-		Name:    bucket,
-		Created: t,
-	}, nil
-}

cmd/gateway/gcs/gateway-gcs.go

@@ -42,7 +42,6 @@ import (
 	"google.golang.org/api/iterator"
 	"google.golang.org/api/option"
 
-	miniogo "github.com/minio/minio-go"
 	minio "github.com/minio/minio/cmd"
 )
 
@@ -155,13 +154,13 @@ type GCS struct {
 	projectID string
 }
 
-// Name returns the name of gcs gatewaylayer.
+// Name returns the name of gcs ObjectLayer.
 func (g *GCS) Name() string {
 	return gcsBackend
 }
 
-// NewGatewayLayer returns gcs gatewaylayer.
-func (g *GCS) NewGatewayLayer(creds auth.Credentials) (minio.GatewayLayer, error) {
+// NewGatewayLayer returns gcs ObjectLayer.
+func (g *GCS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
 	ctx := context.Background()
 
 	var err error
@@ -182,18 +181,10 @@ func (g *GCS) NewGatewayLayer(creds auth.Credentials) (minio.GatewayLayer, error
 		return nil, err
 	}
 
-	// Initialize a anonymous client with minio core APIs.
-	anonClient, err := miniogo.NewCore(googleStorageEndpoint, "", "", true)
-	if err != nil {
-		return nil, err
-	}
-	anonClient.SetCustomTransport(minio.NewCustomHTTPTransport())
-
 	gcs := &gcsGateway{
-		client:     client,
-		projectID:  g.projectID,
-		ctx:        ctx,
-		anonClient: anonClient,
+		client:    client,
+		projectID: g.projectID,
+		ctx:       ctx,
 	}
 
 	// Start background process to cleanup old files in minio.sys.tmp
@@ -349,10 +340,9 @@ func isValidGCSProjectIDFormat(projectID string) bool {
 // gcsGateway - Implements gateway for Minio and GCS compatible object storage servers.
 type gcsGateway struct {
 	minio.GatewayUnsupported
-	client     *storage.Client
-	anonClient *miniogo.Core
-	projectID  string
-	ctx        context.Context
+	client    *storage.Client
+	projectID string
+	ctx       context.Context
 }
 
 const googleStorageEndpoint = "storage.googleapis.com"
@@ -1057,8 +1047,8 @@ func (l *gcsGateway) CompleteMultipartUpload(bucket string, key string, uploadID
 	return fromGCSAttrsToObjectInfo(attrs), nil
 }
 
-// SetBucketPolicies - Set policy on bucket
-func (l *gcsGateway) SetBucketPolicies(bucket string, policyInfo policy.BucketAccessPolicy) error {
+// SetBucketPolicy - Set policy on bucket
+func (l *gcsGateway) SetBucketPolicy(bucket string, policyInfo policy.BucketAccessPolicy) error {
 	var policies []minio.BucketAccessPolicy
 
 	for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket) {
@@ -1102,8 +1092,8 @@ func (l *gcsGateway) SetBucketPolicies(bucket string, policyInfo policy.BucketAc
 	return nil
 }
 
-// GetBucketPolicies - Get policy on bucket
-func (l *gcsGateway) GetBucketPolicies(bucket string) (policy.BucketAccessPolicy, error) {
+// GetBucketPolicy - Get policy on bucket
+func (l *gcsGateway) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, error) {
 	rules, err := l.client.Bucket(bucket).ACL().List(l.ctx)
 	if err != nil {
 		return policy.BucketAccessPolicy{}, gcsToObjectError(errors.Trace(err), bucket)
@@ -1127,8 +1117,8 @@ func (l *gcsGateway) GetBucketPolicies(bucket string) (policy.BucketAccessPolicy
 	return policyInfo, nil
 }
 
-// DeleteBucketPolicies - Delete all policies on bucket
-func (l *gcsGateway) DeleteBucketPolicies(bucket string) error {
+// DeleteBucketPolicy - Delete all policies on bucket
+func (l *gcsGateway) DeleteBucketPolicy(bucket string) error {
 	// This only removes the storage.AllUsers policies
 	if err := l.client.Bucket(bucket).ACL().Delete(l.ctx, storage.AllUsers); err != nil {
 		return gcsToObjectError(errors.Trace(err), bucket)

cmd/gateway/gcs/gateway-gcs_test.go

@@ -223,13 +223,6 @@ func TestGCSParseProjectID(t *testing.T) {
 	}
 }
 
-func TestGCSPublicURL(t *testing.T) {
-	gcsURL := toGCSPublicURL("bucket", "testing")
-	if gcsURL != "https://storage.googleapis.com/bucket/testing" {
-		t.Errorf(`Expected "https://storage.googleapis.com/bucket/testing", got %s"`, gcsURL)
-	}
-}
-
 func TestGCSToObjectError(t *testing.T) {
 	testCases := []struct {
 		params      []string