MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 04:42:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

Create logger package and rename errorIf to LogIf (#5678)

Browse Source 
Removing message from error logging
Replace errors.Trace with LogIf
This commit is contained in:
kannappanr
2018-04-05 15:04:40 -07:00
committed by GitHub
parent 91fd8ffeb7
commit f8a3fd0c2a
119 changed files with 2608 additions and 1860 deletions
Download Patch File Download Diff File Expand all files Collapse all files

184
cmd/gateway/azure/gateway-azure.go
View File

@@ -35,6 +35,7 @@ import (
humanize "github.com/dustin/go-humanize"
"github.com/minio/cli"
"github.com/minio/minio-go/pkg/policy"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/errors"
"github.com/minio/minio/pkg/hash"
@@ -118,7 +119,7 @@ func azureGatewayMain(ctx *cli.Context) {
// Validate gateway arguments.
host := ctx.Args().First()
// Validate gateway arguments.
minio.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
minio.StartGateway(ctx, &Azure{host})
}
@@ -181,11 +182,12 @@ func (g *Azure) Production() bool {
// copied into BlobProperties.
//
// Header names are canonicalized as in http.Header.
func s3MetaToAzureProperties(s3Metadata map[string]string) (storage.BlobMetadata,
func s3MetaToAzureProperties(ctx context.Context, s3Metadata map[string]string) (storage.BlobMetadata,
storage.BlobProperties, error) {
for k := range s3Metadata {
if strings.Contains(k, "--") {
return storage.BlobMetadata{}, storage.BlobProperties{}, errors.Trace(minio.UnsupportedMetadata{})
logger.LogIf(ctx, minio.UnsupportedMetadata{})
return storage.BlobMetadata{}, storage.BlobProperties{}, minio.UnsupportedMetadata{}
}
}
@@ -300,15 +302,6 @@ func azureToObjectError(err error, params ...string) error {
return nil
}
e, ok := err.(*errors.Error)
if !ok {
// Code should be fixed if this function is called without doing errors.Trace()
// Else handling different situations in this function makes this function complicated.
minio.ErrorIf(err, "Expected type *Error")
return err
}
err = e.Cause
bucket := ""
object := ""
if len(params) >= 1 {
@@ -322,7 +315,7 @@ func azureToObjectError(err error, params ...string) error {
if !ok {
// We don't interpret non Azure errors. As azure errors will
// have StatusCode to help to convert to object errors.
return e
return err
}
switch azureErr.Code {
@@ -349,8 +342,7 @@ func azureToObjectError(err error, params ...string) error {
err = minio.BucketNameInvalid{Bucket: bucket}
}
}
e.Cause = err
return e
return err
}
// mustGetAzureUploadID - returns new upload ID which is hex encoded 8 bytes random value.
@@ -371,17 +363,23 @@ func mustGetAzureUploadID() string {
}
// checkAzureUploadID - returns error in case of given string is upload ID.
func checkAzureUploadID(uploadID string) (err error) {
func checkAzureUploadID(ctx context.Context, uploadID string) (err error) {
if len(uploadID) != 16 {
return errors.Trace(minio.MalformedUploadID{
logger.LogIf(ctx, minio.MalformedUploadID{
UploadID: uploadID,
})
return minio.MalformedUploadID{
UploadID: uploadID,
}
}
if _, err = hex.DecodeString(uploadID); err != nil {
return errors.Trace(minio.MalformedUploadID{
logger.LogIf(ctx, minio.MalformedUploadID{
UploadID: uploadID,
})
return minio.MalformedUploadID{
UploadID: uploadID,
}
}
return nil
@@ -438,7 +436,8 @@ func (a *azureObjects) MakeBucketWithLocation(ctx context.Context, bucket, locat
err := container.Create(&storage.CreateContainerOptions{
Access: storage.ContainerAccessTypePrivate,
})
return azureToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return azureToObjectError(err, bucket)
}
// GetBucketInfo - Get bucket metadata..
@@ -448,7 +447,8 @@ func (a *azureObjects) GetBucketInfo(ctx context.Context, bucket string) (bi min
// in azure documentation, so we will simply use the same function here.
// Ref - https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata
if !minio.IsValidBucketName(bucket) {
return bi, errors.Trace(minio.BucketNameInvalid{Bucket: bucket})
logger.LogIf(ctx, minio.BucketNameInvalid{Bucket: bucket})
return bi, minio.BucketNameInvalid{Bucket: bucket}
}
// Azure does not have an equivalent call, hence use
@@ -457,7 +457,8 @@ func (a *azureObjects) GetBucketInfo(ctx context.Context, bucket string) (bi min
Prefix: bucket,
})
if err != nil {
return bi, azureToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return bi, azureToObjectError(err, bucket)
}
for _, container := range resp.Containers {
if container.Name == bucket {
@@ -470,19 +471,22 @@ func (a *azureObjects) GetBucketInfo(ctx context.Context, bucket string) (bi min
} // else continue
}
}
return bi, errors.Trace(minio.BucketNotFound{Bucket: bucket})
logger.LogIf(ctx, minio.BucketNotFound{Bucket: bucket})
return bi, minio.BucketNotFound{Bucket: bucket}
}
// ListBuckets - Lists all azure containers, uses Azure equivalent ListContainers.
func (a *azureObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketInfo, err error) {
resp, err := a.client.ListContainers(storage.ListContainersParameters{})
if err != nil {
return nil, azureToObjectError(errors.Trace(err))
logger.LogIf(ctx, err)
return nil, azureToObjectError(err)
}
for _, container := range resp.Containers {
t, e := time.Parse(time.RFC1123, container.Properties.LastModified)
if e != nil {
return nil, errors.Trace(e)
logger.LogIf(ctx, e)
return nil, e
}
buckets = append(buckets, minio.BucketInfo{
Name: container.Name,
@@ -495,7 +499,9 @@ func (a *azureObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketI
// DeleteBucket - delete a container on azure, uses Azure equivalent DeleteContainer.
func (a *azureObjects) DeleteBucket(ctx context.Context, bucket string) error {
container := a.client.GetContainerReference(bucket)
return azureToObjectError(errors.Trace(container.Delete(nil)), bucket)
err := container.Delete(nil)
logger.LogIf(ctx, err)
return azureToObjectError(err, bucket)
}
// ListObjects - lists all blobs on azure with in a container filtered by prefix
@@ -512,7 +518,8 @@ func (a *azureObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
MaxResults: uint(maxKeys),
})
if err != nil {
return result, azureToObjectError(errors.Trace(err), bucket, prefix)
logger.LogIf(ctx, err)
return result, azureToObjectError(err, bucket, prefix)
}
for _, object := range resp.Blobs {
@@ -580,7 +587,8 @@ func (a *azureObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin
func (a *azureObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
// startOffset cannot be negative.
if startOffset < 0 {
return azureToObjectError(errors.Trace(minio.InvalidRange{}), bucket, object)
logger.LogIf(ctx, minio.InvalidRange{})
return azureToObjectError(minio.InvalidRange{}, bucket, object)
}
blobRange := &storage.BlobRange{Start: uint64(startOffset)}
@@ -599,11 +607,13 @@ func (a *azureObjects) GetObject(ctx context.Context, bucket, object string, sta
})
}
if err != nil {
return azureToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return azureToObjectError(err, bucket, object)
}
_, err = io.Copy(writer, rc)
rc.Close()
return errors.Trace(err)
logger.LogIf(ctx, err)
return err
}
// GetObjectInfo - reads blob metadata properties and replies back minio.ObjectInfo,
@@ -612,7 +622,8 @@ func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string)
blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
err = blob.GetProperties(nil)
if err != nil {
return objInfo, azureToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return objInfo, azureToObjectError(err, bucket, object)
}
return minio.ObjectInfo{
@@ -631,13 +642,14 @@ func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string)
// uses Azure equivalent CreateBlockBlobFromReader.
func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
blob.Metadata, blob.Properties, err = s3MetaToAzureProperties(metadata)
blob.Metadata, blob.Properties, err = s3MetaToAzureProperties(ctx, metadata)
if err != nil {
return objInfo, azureToObjectError(err, bucket, object)
}
err = blob.CreateBlockBlobFromReader(data, nil)
if err != nil {
return objInfo, azureToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return objInfo, azureToObjectError(err, bucket, object)
}
return a.GetObjectInfo(ctx, bucket, object)
}
@@ -647,19 +659,21 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, dat
func (a *azureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) {
srcBlobURL := a.client.GetContainerReference(srcBucket).GetBlobReference(srcObject).GetURL()
destBlob := a.client.GetContainerReference(destBucket).GetBlobReference(destObject)
azureMeta, props, err := s3MetaToAzureProperties(srcInfo.UserDefined)
azureMeta, props, err := s3MetaToAzureProperties(ctx, srcInfo.UserDefined)
if err != nil {
return objInfo, azureToObjectError(err, srcBucket, srcObject)
}
destBlob.Metadata = azureMeta
err = destBlob.Copy(srcBlobURL, nil)
if err != nil {
return objInfo, azureToObjectError(errors.Trace(err), srcBucket, srcObject)
logger.LogIf(ctx, err)
return objInfo, azureToObjectError(err, srcBucket, srcObject)
}
destBlob.Properties = props
err = destBlob.SetProperties(nil)
if err != nil {
return objInfo, azureToObjectError(errors.Trace(err), srcBucket, srcObject)
logger.LogIf(ctx, err)
return objInfo, azureToObjectError(err, srcBucket, srcObject)
}
return a.GetObjectInfo(ctx, destBucket, destObject)
}
@@ -670,7 +684,7 @@ func (a *azureObjects) DeleteObject(ctx context.Context, bucket, object string)
blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
err := blob.Delete(nil)
if err != nil {
return azureToObjectError(errors.Trace(err), bucket, object)
return azureToObjectError(err, bucket, object)
}
return nil
}
@@ -690,19 +704,21 @@ func getAzureMetadataObjectName(objectName, uploadID string) string {
return fmt.Sprintf(metadataObjectNameTemplate, uploadID, sha256.Sum256([]byte(objectName)))
}
func (a *azureObjects) checkUploadIDExists(bucketName, objectName, uploadID string) (err error) {
func (a *azureObjects) checkUploadIDExists(ctx context.Context, bucketName, objectName, uploadID string) (err error) {
blob := a.client.GetContainerReference(bucketName).GetBlobReference(
getAzureMetadataObjectName(objectName, uploadID))
err = blob.GetMetadata(nil)
err = azureToObjectError(errors.Trace(err), bucketName, objectName)
logger.LogIf(ctx, err)
err = azureToObjectError(err, bucketName, objectName)
oerr := minio.ObjectNotFound{
Bucket: bucketName,
Object: objectName,
}
if errors.Cause(err) == oerr {
err = errors.Trace(minio.InvalidUploadID{
logger.LogIf(ctx, minio.InvalidUploadID{UploadID: uploadID})
err = minio.InvalidUploadID{
UploadID: uploadID,
})
}
}
return err
}
@@ -714,13 +730,15 @@ func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object st
var jsonData []byte
if jsonData, err = json.Marshal(azureMultipartMetadata{Name: object, Metadata: metadata}); err != nil {
return "", errors.Trace(err)
logger.LogIf(ctx, err)
return "", err
}
blob := a.client.GetContainerReference(bucket).GetBlobReference(metadataObject)
err = blob.CreateBlockBlobFromReader(bytes.NewBuffer(jsonData), nil)
if err != nil {
return "", azureToObjectError(errors.Trace(err), bucket, metadataObject)
logger.LogIf(ctx, err)
return "", azureToObjectError(err, bucket, metadataObject)
}
return uploadID, nil
@@ -728,11 +746,11 @@ func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object st
// PutObjectPart - Use Azure equivalent PutBlockWithLength.
func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info minio.PartInfo, err error) {
if err = a.checkUploadIDExists(bucket, object, uploadID); err != nil {
if err = a.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
return info, err
}
if err = checkAzureUploadID(uploadID); err != nil {
if err = checkAzureUploadID(ctx, uploadID); err != nil {
return info, err
}
@@ -756,7 +774,8 @@ func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, upload
blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
err = blob.PutBlockWithLength(id, uint64(subPartSize), io.LimitReader(data, subPartSize), nil)
if err != nil {
return info, azureToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return info, azureToObjectError(err, bucket, object)
}
subPartNumber++
}
@@ -770,7 +789,7 @@ func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, upload
// ListObjectParts - Use Azure equivalent GetBlockList.
func (a *azureObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int) (result minio.ListPartsInfo, err error) {
if err = a.checkUploadIDExists(bucket, object, uploadID); err != nil {
if err = a.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
return result, err
}
@@ -787,7 +806,8 @@ func (a *azureObjects) ListObjectParts(ctx context.Context, bucket, object, uplo
return result, nil
}
if err != nil {
return result, azureToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return result, azureToObjectError(err, bucket, object)
}
// Build a sorted list of parts and return the requested entries.
partsMap := make(map[int]minio.PartInfo)
@@ -796,7 +816,8 @@ func (a *azureObjects) ListObjectParts(ctx context.Context, bucket, object, uplo
var parsedUploadID string
var md5Hex string
if partNumber, _, parsedUploadID, md5Hex, err = azureParseBlockID(block.Name); err != nil {
return result, azureToObjectError(errors.Trace(fmt.Errorf("Unexpected error")), bucket, object)
logger.LogIf(ctx, fmt.Errorf("Unexpected error"))
return result, azureToObjectError(fmt.Errorf("Unexpected error"), bucket, object)
}
if parsedUploadID != uploadID {
continue
@@ -813,7 +834,8 @@ func (a *azureObjects) ListObjectParts(ctx context.Context, bucket, object, uplo
if part.ETag != md5Hex {
// If two parts of same partNumber were uploaded with different contents
// return error as we won't be able to decide which the latest part is.
return result, azureToObjectError(errors.Trace(fmt.Errorf("Unexpected error")), bucket, object)
logger.LogIf(ctx, fmt.Errorf("Unexpected error"))
return result, azureToObjectError(fmt.Errorf("Unexpected error"), bucket, object)
}
part.Size += block.Size
partsMap[partNumber] = part
@@ -856,7 +878,7 @@ func (a *azureObjects) ListObjectParts(ctx context.Context, bucket, object, uplo
// There is no corresponding API in azure to abort an incomplete upload. The uncommmitted blocks
// gets deleted after one week.
func (a *azureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) (err error) {
if err = a.checkUploadIDExists(bucket, object, uploadID); err != nil {
if err = a.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
return err
}
@@ -868,23 +890,25 @@ func (a *azureObjects) AbortMultipartUpload(ctx context.Context, bucket, object,
// CompleteMultipartUpload - Use Azure equivalent PutBlockList.
func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []minio.CompletePart) (objInfo minio.ObjectInfo, err error) {
metadataObject := getAzureMetadataObjectName(object, uploadID)
if err = a.checkUploadIDExists(bucket, object, uploadID); err != nil {
if err = a.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
return objInfo, err
}
if err = checkAzureUploadID(uploadID); err != nil {
if err = checkAzureUploadID(ctx, uploadID); err != nil {
return objInfo, err
}
var metadataReader io.Reader
blob := a.client.GetContainerReference(bucket).GetBlobReference(metadataObject)
if metadataReader, err = blob.Get(nil); err != nil {
return objInfo, azureToObjectError(errors.Trace(err), bucket, metadataObject)
logger.LogIf(ctx, err)
return objInfo, azureToObjectError(err, bucket, metadataObject)
}
var metadata azureMultipartMetadata
if err = json.NewDecoder(metadataReader).Decode(&metadata); err != nil {
return objInfo, azureToObjectError(errors.Trace(err), bucket, metadataObject)
logger.LogIf(ctx, err)
return objInfo, azureToObjectError(err, bucket, metadataObject)
}
defer func() {
@@ -894,13 +918,15 @@ func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje
blob := a.client.GetContainerReference(bucket).GetBlobReference(metadataObject)
derr := blob.Delete(nil)
minio.ErrorIf(derr, "unable to remove meta data object for upload ID %s", uploadID)
logger.GetReqInfo(ctx).AppendTags("uploadID", uploadID)
logger.LogIf(ctx, derr)
}()
objBlob := a.client.GetContainerReference(bucket).GetBlobReference(object)
resp, err := objBlob.GetBlockList(storage.BlockListTypeUncommitted, nil)
if err != nil {
return objInfo, azureToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return objInfo, azureToObjectError(err, bucket, object)
}
getBlocks := func(partNumber int, etag string) (blocks []storage.Block, size int64, err error) {
@@ -936,7 +962,8 @@ func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje
var size int64
blocks, size, err = getBlocks(part.PartNumber, part.ETag)
if err != nil {
return objInfo, errors.Trace(err)
logger.LogIf(ctx, err)
return objInfo, err
}
allBlocks = append(allBlocks, blocks...)
@@ -946,30 +973,39 @@ func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje
// Error out if parts except last part sizing < 5MiB.
for i, size := range partSizes[:len(partSizes)-1] {
if size < azureS3MinPartSize {
return objInfo, errors.Trace(minio.PartTooSmall{
logger.LogIf(ctx, minio.PartTooSmall{
PartNumber: uploadedParts[i].PartNumber,
PartSize: size,
PartETag: uploadedParts[i].ETag,
})
return objInfo, minio.PartTooSmall{
PartNumber: uploadedParts[i].PartNumber,
PartSize: size,
PartETag: uploadedParts[i].ETag,
}
}
}
err = objBlob.PutBlockList(allBlocks, nil)
if err != nil {
return objInfo, azureToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return objInfo, azureToObjectError(err, bucket, object)
}
if len(metadata.Metadata) > 0 {
objBlob.Metadata, objBlob.Properties, err = s3MetaToAzureProperties(metadata.Metadata)
objBlob.Metadata, objBlob.Properties, err = s3MetaToAzureProperties(ctx, metadata.Metadata)
if err != nil {
logger.LogIf(ctx, err)
return objInfo, azureToObjectError(err, bucket, object)
}
err = objBlob.SetProperties(nil)
if err != nil {
return objInfo, azureToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return objInfo, azureToObjectError(err, bucket, object)
}
err = objBlob.SetMetadata(nil)
if err != nil {
return objInfo, azureToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return objInfo, azureToObjectError(err, bucket, object)
}
}
return a.GetObjectInfo(ctx, bucket, object)
@@ -992,13 +1028,16 @@ func (a *azureObjects) SetBucketPolicy(ctx context.Context, bucket string, polic
}
prefix := bucket + "/*" // For all objects inside the bucket.
if len(policies) != 1 {
return errors.Trace(minio.NotImplemented{})
logger.LogIf(ctx, minio.NotImplemented{})
return minio.NotImplemented{}
}
if policies[0].Prefix != prefix {
return errors.Trace(minio.NotImplemented{})
logger.LogIf(ctx, minio.NotImplemented{})
return minio.NotImplemented{}
}
if policies[0].Policy != policy.BucketPolicyReadOnly {
return errors.Trace(minio.NotImplemented{})
logger.LogIf(ctx, minio.NotImplemented{})
return minio.NotImplemented{}
}
perm := storage.ContainerPermissions{
AccessType: storage.ContainerAccessTypeContainer,
@@ -1006,7 +1045,8 @@ func (a *azureObjects) SetBucketPolicy(ctx context.Context, bucket string, polic
}
container := a.client.GetContainerReference(bucket)
err := container.SetPermissions(perm, nil)
return azureToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return azureToObjectError(err, bucket)
}
// GetBucketPolicy - Get the container ACL and convert it to canonical []bucketAccessPolicy
@@ -1015,15 +1055,18 @@ func (a *azureObjects) GetBucketPolicy(ctx context.Context, bucket string) (poli
container := a.client.GetContainerReference(bucket)
perm, err := container.GetPermissions(nil)
if err != nil {
return policy.BucketAccessPolicy{}, azureToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return policy.BucketAccessPolicy{}, azureToObjectError(err, bucket)
}
switch perm.AccessType {
case storage.ContainerAccessTypePrivate:
return policy.BucketAccessPolicy{}, errors.Trace(minio.PolicyNotFound{Bucket: bucket})
logger.LogIf(ctx, minio.PolicyNotFound{Bucket: bucket})
return policy.BucketAccessPolicy{}, minio.PolicyNotFound{Bucket: bucket}
case storage.ContainerAccessTypeContainer:
policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyReadOnly, bucket, "")
default:
return policy.BucketAccessPolicy{}, azureToObjectError(errors.Trace(minio.NotImplemented{}))
logger.LogIf(ctx, minio.NotImplemented{})
return policy.BucketAccessPolicy{}, azureToObjectError(minio.NotImplemented{})
}
return policyInfo, nil
}
@@ -1036,5 +1079,6 @@ func (a *azureObjects) DeleteBucketPolicy(ctx context.Context, bucket string) er
}
container := a.client.GetContainerReference(bucket)
err := container.SetPermissions(perm, nil)
return azureToObjectError(errors.Trace(err))
logger.LogIf(ctx, err)
return azureToObjectError(err)
}

46
cmd/gateway/azure/gateway-azure_test.go
View File

@@ -17,6 +17,7 @@
package azure
import (
"context"
"fmt"
"net/http"
"reflect"
@@ -55,7 +56,7 @@ func TestS3MetaToAzureProperties(t *testing.T) {
"X_Amz_Matdesc": "{}",
"X_Amz_Iv": "eWmyryl8kq+EVnnsE7jpOg==",
}
meta, _, err := s3MetaToAzureProperties(headers)
meta, _, err := s3MetaToAzureProperties(context.Background(), headers)
if err != nil {
t.Fatalf("Test failed, with %s", err)
}
@@ -65,7 +66,7 @@ func TestS3MetaToAzureProperties(t *testing.T) {
headers = map[string]string{
"invalid--meta": "value",
}
_, _, err = s3MetaToAzureProperties(headers)
_, _, err = s3MetaToAzureProperties(context.Background(), headers)
if err = errors.Cause(err); err != nil {
if _, ok := err.(minio.UnsupportedMetadata); !ok {
t.Fatalf("Test failed with unexpected error %s, expected UnsupportedMetadata", err)
@@ -75,7 +76,7 @@ func TestS3MetaToAzureProperties(t *testing.T) {
headers = map[string]string{
"content-md5": "Dce7bmCX61zvxzP5QmfelQ==",
}
_, props, err := s3MetaToAzureProperties(headers)
_, props, err := s3MetaToAzureProperties(context.Background(), headers)
if err != nil {
t.Fatalf("Test failed, with %s", err)
}
@@ -137,53 +138,46 @@ func TestAzureToObjectError(t *testing.T) {
nil, nil, "", "",
},
{
errors.Trace(fmt.Errorf("Non azure error")),
fmt.Errorf("Non azure error"),
fmt.Errorf("Non azure error"), "", "",
},
{
storage.AzureStorageServiceError{
Code: "ContainerAlreadyExists",
}, storage.AzureStorageServiceError{
Code: "ContainerAlreadyExists",
}, "bucket", "",
}, minio.BucketExists{Bucket: "bucket"}, "bucket", "",
},
{
errors.Trace(storage.AzureStorageServiceError{
Code: "ContainerAlreadyExists",
}), minio.BucketExists{Bucket: "bucket"}, "bucket", "",
},
{
errors.Trace(storage.AzureStorageServiceError{
storage.AzureStorageServiceError{
Code: "InvalidResourceName",
}), minio.BucketNameInvalid{Bucket: "bucket."}, "bucket.", "",
}, minio.BucketNameInvalid{Bucket: "bucket."}, "bucket.", "",
},
{
errors.Trace(storage.AzureStorageServiceError{
storage.AzureStorageServiceError{
Code: "RequestBodyTooLarge",
}), minio.PartTooBig{}, "", "",
}, minio.PartTooBig{}, "", "",
},
{
errors.Trace(storage.AzureStorageServiceError{
storage.AzureStorageServiceError{
Code: "InvalidMetadata",
}), minio.UnsupportedMetadata{}, "", "",
}, minio.UnsupportedMetadata{}, "", "",
},
{
errors.Trace(storage.AzureStorageServiceError{
storage.AzureStorageServiceError{
StatusCode: http.StatusNotFound,
}), minio.ObjectNotFound{
}, minio.ObjectNotFound{
Bucket: "bucket",
Object: "object",
}, "bucket", "object",
},
{
errors.Trace(storage.AzureStorageServiceError{
storage.AzureStorageServiceError{
StatusCode: http.StatusNotFound,
}), minio.BucketNotFound{Bucket: "bucket"}, "bucket", "",
}, minio.BucketNotFound{Bucket: "bucket"}, "bucket", "",
},
{
errors.Trace(storage.AzureStorageServiceError{
storage.AzureStorageServiceError{
StatusCode: http.StatusBadRequest,
}), minio.BucketNameInvalid{Bucket: "bucket."}, "bucket.", "",
}, minio.BucketNameInvalid{Bucket: "bucket."}, "bucket.", "",
},
}
for i, testCase := range testCases {
@@ -307,7 +301,7 @@ func TestCheckAzureUploadID(t *testing.T) {
}
for _, uploadID := range invalidUploadIDs {
if err := checkAzureUploadID(uploadID); err == nil {
if err := checkAzureUploadID(context.Background(), uploadID); err == nil {
t.Fatalf("%s: expected: <error>, got: <nil>", uploadID)
}
}
@@ -318,7 +312,7 @@ func TestCheckAzureUploadID(t *testing.T) {
}
for _, uploadID := range validUploadIDs {
if err := checkAzureUploadID(uploadID); err != nil {
if err := checkAzureUploadID(context.Background(), uploadID); err != nil {
t.Fatalf("%s: expected: <nil>, got: %s", uploadID, err)
}
}

104
cmd/gateway/b2/gateway-b2.go
View File

@@ -31,8 +31,8 @@ import (
b2 "github.com/minio/blazer/base"
"github.com/minio/cli"
"github.com/minio/minio-go/pkg/policy"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/errors"
h2 "github.com/minio/minio/pkg/hash"
minio "github.com/minio/minio/cmd"
@@ -146,16 +146,6 @@ func b2ToObjectError(err error, params ...string) error {
if err == nil {
return nil
}
e, ok := err.(*errors.Error)
if !ok {
// Code should be fixed if this function is called without doing errors.Trace()
// Else handling different situations in this function makes this function complicated.
minio.ErrorIf(err, "Expected type *Error")
return err
}
err = e.Cause
bucket := ""
object := ""
uploadID := ""
@@ -177,7 +167,7 @@ func b2ToObjectError(err error, params ...string) error {
if statusCode == 0 {
// We don't interpret non B2 errors. B2 errors have statusCode
// to help us convert them to S3 object errors.
return e
return err
}
switch code {
@@ -208,8 +198,7 @@ func b2ToObjectError(err error, params ...string) error {
err = minio.InvalidUploadID{UploadID: uploadID}
}
e.Cause = err
return e
return err
}
// Shutdown saves any gateway metadata to disk
@@ -230,7 +219,8 @@ func (l *b2Objects) MakeBucketWithLocation(ctx context.Context, bucket, location
// All buckets are set to private by default.
_, err := l.b2Client.CreateBucket(l.ctx, bucket, bucketTypePrivate, nil, nil)
return b2ToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return b2ToObjectError(err, bucket)
}
func (l *b2Objects) reAuthorizeAccount(ctx context.Context) error {
@@ -271,14 +261,16 @@ func (l *b2Objects) listBuckets(ctx context.Context, err error) ([]*b2.Bucket, e
func (l *b2Objects) Bucket(ctx context.Context, bucket string) (*b2.Bucket, error) {
bktList, err := l.listBuckets(ctx, nil)
if err != nil {
return nil, b2ToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return nil, b2ToObjectError(err, bucket)
}
for _, bkt := range bktList {
if bkt.Name == bucket {
return bkt, nil
}
}
return nil, errors.Trace(minio.BucketNotFound{Bucket: bucket})
logger.LogIf(ctx, minio.BucketNotFound{Bucket: bucket})
return nil, minio.BucketNotFound{Bucket: bucket}
}
// GetBucketInfo gets bucket metadata..
@@ -315,7 +307,8 @@ func (l *b2Objects) DeleteBucket(ctx context.Context, bucket string) error {
return err
}
err = bkt.DeleteBucket(l.ctx)
return b2ToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return b2ToObjectError(err, bucket)
}
// ListObjects lists all objects in B2 bucket filtered by prefix, returns upto at max 1000 entries at a time.
@@ -326,7 +319,8 @@ func (l *b2Objects) ListObjects(ctx context.Context, bucket string, prefix strin
}
files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter)
if lerr != nil {
return loi, b2ToObjectError(errors.Trace(lerr), bucket)
logger.LogIf(ctx, lerr)
return loi, b2ToObjectError(lerr, bucket)
}
loi.IsTruncated = next != ""
loi.NextMarker = next
@@ -359,7 +353,8 @@ func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat
}
files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, continuationToken, prefix, delimiter)
if lerr != nil {
return loi, b2ToObjectError(errors.Trace(lerr), bucket)
logger.LogIf(ctx, lerr)
return loi, b2ToObjectError(lerr, bucket)
}
loi.IsTruncated = next != ""
loi.ContinuationToken = continuationToken
@@ -396,11 +391,13 @@ func (l *b2Objects) GetObject(ctx context.Context, bucket string, object string,
}
reader, err := bkt.DownloadFileByName(l.ctx, object, startOffset, length)
if err != nil {
return b2ToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return b2ToObjectError(err, bucket, object)
}
defer reader.Close()
_, err = io.Copy(writer, reader)
return b2ToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return b2ToObjectError(err, bucket, object)
}
// GetObjectInfo reads object info and replies back ObjectInfo
@@ -411,12 +408,14 @@ func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object str
}
f, err := bkt.DownloadFileByName(l.ctx, object, 0, 1)
if err != nil {
return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return objInfo, b2ToObjectError(err, bucket, object)
}
f.Close()
fi, err := bkt.File(f.ID, object).GetFileInfo(l.ctx)
if err != nil {
return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return objInfo, b2ToObjectError(err, bucket, object)
}
return minio.ObjectInfo{
Bucket: bucket,
@@ -504,20 +503,23 @@ func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string,
var u *b2.URL
u, err = bkt.GetUploadURL(l.ctx)
if err != nil {
return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return objInfo, b2ToObjectError(err, bucket, object)
}
hr := newB2Reader(data, data.Size())
var f *b2.File
f, err = u.UploadFile(l.ctx, hr, int(hr.Size()), object, contentType, sha1AtEOF, metadata)
if err != nil {
return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return objInfo, b2ToObjectError(err, bucket, object)
}
var fi *b2.FileInfo
fi, err = f.GetFileInfo(l.ctx)
if err != nil {
return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return objInfo, b2ToObjectError(err, bucket, object)
}
return minio.ObjectInfo{
@@ -539,12 +541,14 @@ func (l *b2Objects) DeleteObject(ctx context.Context, bucket string, object stri
}
reader, err := bkt.DownloadFileByName(l.ctx, object, 0, 1)
if err != nil {
return b2ToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return b2ToObjectError(err, bucket, object)
}
io.Copy(ioutil.Discard, reader)
reader.Close()
err = bkt.File(reader.ID, object).DeleteFileVersion(l.ctx)
return b2ToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return b2ToObjectError(err, bucket, object)
}
// ListMultipartUploads lists all multipart uploads.
@@ -563,7 +567,8 @@ func (l *b2Objects) ListMultipartUploads(ctx context.Context, bucket string, pre
}
largeFiles, nextMarker, err := bkt.ListUnfinishedLargeFiles(l.ctx, uploadIDMarker, maxUploads)
if err != nil {
return lmi, b2ToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return lmi, b2ToObjectError(err, bucket)
}
lmi = minio.ListMultipartsInfo{
MaxUploads: maxUploads,
@@ -598,7 +603,8 @@ func (l *b2Objects) NewMultipartUpload(ctx context.Context, bucket string, objec
delete(metadata, "content-type")
lf, err := bkt.StartLargeFile(l.ctx, object, contentType, metadata)
if err != nil {
return uploadID, b2ToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return uploadID, b2ToObjectError(err, bucket, object)
}
return lf.ID, nil
@@ -613,13 +619,15 @@ func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object str
fc, err := bkt.File(uploadID, object).CompileParts(0, nil).GetUploadPartURL(l.ctx)
if err != nil {
return pi, b2ToObjectError(errors.Trace(err), bucket, object, uploadID)
logger.LogIf(ctx, err)
return pi, b2ToObjectError(err, bucket, object, uploadID)
}
hr := newB2Reader(data, data.Size())
sha1, err := fc.UploadPart(l.ctx, hr, sha1AtEOF, int(hr.Size()), partID)
if err != nil {
return pi, b2ToObjectError(errors.Trace(err), bucket, object, uploadID)
logger.LogIf(ctx, err)
return pi, b2ToObjectError(err, bucket, object, uploadID)
}
return minio.PartInfo{
@@ -647,7 +655,8 @@ func (l *b2Objects) ListObjectParts(ctx context.Context, bucket string, object s
partNumberMarker++
partsList, next, err := bkt.File(uploadID, object).ListParts(l.ctx, partNumberMarker, maxParts)
if err != nil {
return lpi, b2ToObjectError(errors.Trace(err), bucket, object, uploadID)
logger.LogIf(ctx, err)
return lpi, b2ToObjectError(err, bucket, object, uploadID)
}
if next != 0 {
lpi.IsTruncated = true
@@ -670,7 +679,8 @@ func (l *b2Objects) AbortMultipartUpload(ctx context.Context, bucket string, obj
return err
}
err = bkt.File(uploadID, object).CompileParts(0, nil).CancelLargeFile(l.ctx)
return b2ToObjectError(errors.Trace(err), bucket, object, uploadID)
logger.LogIf(ctx, err)
return b2ToObjectError(err, bucket, object, uploadID)
}
// CompleteMultipartUpload completes ongoing multipart upload and finalizes object, uses B2's LargeFile upload API.
@@ -684,7 +694,8 @@ func (l *b2Objects) CompleteMultipartUpload(ctx context.Context, bucket string,
// B2 requires contigous part numbers starting with 1, they do not support
// hand picking part numbers, we return an S3 compatible error instead.
if i+1 != uploadedPart.PartNumber {
return oi, b2ToObjectError(errors.Trace(minio.InvalidPart{}), bucket, object, uploadID)
logger.LogIf(ctx, minio.InvalidPart{})
return oi, b2ToObjectError(minio.InvalidPart{}, bucket, object, uploadID)
}
// Trim "-1" suffix in ETag as PutObjectPart() treats B2 returned SHA1 as ETag.
@@ -692,7 +703,8 @@ func (l *b2Objects) CompleteMultipartUpload(ctx context.Context, bucket string,
}
if _, err = bkt.File(uploadID, object).CompileParts(0, hashes).FinishLargeFile(l.ctx); err != nil {
return oi, b2ToObjectError(errors.Trace(err), bucket, object, uploadID)
logger.LogIf(ctx, err)
return oi, b2ToObjectError(err, bucket, object, uploadID)
}
return l.GetObjectInfo(ctx, bucket, object)
@@ -713,13 +725,16 @@ func (l *b2Objects) SetBucketPolicy(ctx context.Context, bucket string, policyIn
}
prefix := bucket + "/*" // For all objects inside the bucket.
if len(policies) != 1 {
return errors.Trace(minio.NotImplemented{})
logger.LogIf(ctx, minio.NotImplemented{})
return minio.NotImplemented{}
}
if policies[0].Prefix != prefix {
return errors.Trace(minio.NotImplemented{})
logger.LogIf(ctx, minio.NotImplemented{})
return minio.NotImplemented{}
}
if policies[0].Policy != policy.BucketPolicyReadOnly {
return errors.Trace(minio.NotImplemented{})
logger.LogIf(ctx, minio.NotImplemented{})
return minio.NotImplemented{}
}
bkt, err := l.Bucket(ctx, bucket)
if err != nil {
@@ -727,7 +742,8 @@ func (l *b2Objects) SetBucketPolicy(ctx context.Context, bucket string, policyIn
}
bkt.Type = bucketTypeReadOnly
_, err = bkt.Update(l.ctx)
return b2ToObjectError(errors.Trace(err))
logger.LogIf(ctx, err)
return b2ToObjectError(err)
}
// GetBucketPolicy, returns the current bucketType from B2 backend and convert
@@ -745,7 +761,8 @@ func (l *b2Objects) GetBucketPolicy(ctx context.Context, bucket string) (policy.
// bkt.Type can also be snapshot, but it is only allowed through B2 browser console,
// just return back as policy not found for all cases.
// CreateBucket always sets the value to allPrivate by default.
return policy.BucketAccessPolicy{}, errors.Trace(minio.PolicyNotFound{Bucket: bucket})
logger.LogIf(ctx, minio.PolicyNotFound{Bucket: bucket})
return policy.BucketAccessPolicy{}, minio.PolicyNotFound{Bucket: bucket}
}
// DeleteBucketPolicy - resets the bucketType of bucket on B2 to 'allPrivate'.
@@ -756,5 +773,6 @@ func (l *b2Objects) DeleteBucketPolicy(ctx context.Context, bucket string) error
}
bkt.Type = bucketTypePrivate
_, err = bkt.Update(l.ctx)
return b2ToObjectError(errors.Trace(err))
logger.LogIf(ctx, err)
return b2ToObjectError(err)
}

35
cmd/gateway/b2/gateway-b2_test.go
View File

@@ -21,7 +21,6 @@ import (
"testing"
b2 "github.com/minio/blazer/base"
"github.com/minio/minio/pkg/errors"
minio "github.com/minio/minio/cmd"
)
@@ -40,70 +39,70 @@ func TestB2ObjectError(t *testing.T) {
[]string{}, fmt.Errorf("Not *Error"), fmt.Errorf("Not *Error"),
},
{
[]string{}, errors.Trace(fmt.Errorf("Non B2 Error")), fmt.Errorf("Non B2 Error"),
[]string{}, fmt.Errorf("Non B2 Error"), fmt.Errorf("Non B2 Error"),
},
{
[]string{"bucket"}, errors.Trace(b2.Error{
[]string{"bucket"}, b2.Error{
StatusCode: 1,
Code: "duplicate_bucket_name",
}), minio.BucketAlreadyOwnedByYou{
}, minio.BucketAlreadyOwnedByYou{
Bucket: "bucket",
},
},
{
[]string{"bucket"}, errors.Trace(b2.Error{
[]string{"bucket"}, b2.Error{
StatusCode: 1,
Code: "bad_request",
}), minio.BucketNotFound{
}, minio.BucketNotFound{
Bucket: "bucket",
},
},
{
[]string{"bucket", "object"}, errors.Trace(b2.Error{
[]string{"bucket", "object"}, b2.Error{
StatusCode: 1,
Code: "bad_request",
}), minio.ObjectNameInvalid{
}, minio.ObjectNameInvalid{
Bucket: "bucket",
Object: "object",
},
},
{
[]string{"bucket"}, errors.Trace(b2.Error{
[]string{"bucket"}, b2.Error{
StatusCode: 1,
Code: "bad_bucket_id",
}), minio.BucketNotFound{Bucket: "bucket"},
}, minio.BucketNotFound{Bucket: "bucket"},
},
{
[]string{"bucket", "object"}, errors.Trace(b2.Error{
[]string{"bucket", "object"}, b2.Error{
StatusCode: 1,
Code: "file_not_present",
}), minio.ObjectNotFound{
}, minio.ObjectNotFound{
Bucket: "bucket",
Object: "object",
},
},
{
[]string{"bucket", "object"}, errors.Trace(b2.Error{
[]string{"bucket", "object"}, b2.Error{
StatusCode: 1,
Code: "not_found",
}), minio.ObjectNotFound{
}, minio.ObjectNotFound{
Bucket: "bucket",
Object: "object",
},
},
{
[]string{"bucket"}, errors.Trace(b2.Error{
[]string{"bucket"}, b2.Error{
StatusCode: 1,
Code: "cannot_delete_non_empty_bucket",
}), minio.BucketNotEmpty{
}, minio.BucketNotEmpty{
Bucket: "bucket",
},
},
{
[]string{"bucket", "object", "uploadID"}, errors.Trace(b2.Error{
[]string{"bucket", "object", "uploadID"}, b2.Error{
StatusCode: 1,
Message: "No active upload for",
}), minio.InvalidUploadID{
}, minio.InvalidUploadID{
UploadID: "uploadID",
},
},

177
cmd/gateway/gcs/gateway-gcs.go
View File

@@ -34,8 +34,8 @@ import (
humanize "github.com/dustin/go-humanize"
"github.com/minio/cli"
"github.com/minio/minio-go/pkg/policy"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/errors"
"github.com/minio/minio/pkg/hash"
"google.golang.org/api/googleapi"
@@ -154,11 +154,13 @@ EXAMPLES:
func gcsGatewayMain(ctx *cli.Context) {
projectID := ctx.Args().First()
if projectID == "" && os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" {
minio.ErrorIf(errGCSProjectIDNotFound, "project-id should be provided as argument or GOOGLE_APPLICATION_CREDENTIALS should be set with path to credentials.json")
logger.LogIf(context.Background(), errGCSProjectIDNotFound)
cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
}
if projectID != "" && !isValidGCSProjectIDFormat(projectID) {
minio.ErrorIf(errGCSInvalidProjectID, "Unable to start GCS gateway with %s", ctx.Args().First())
reqInfo := (&logger.ReqInfo{}).AppendTags("projectID", ctx.Args().First())
contxt := logger.SetReqInfo(context.Background(), reqInfo)
logger.LogIf(contxt, errGCSInvalidProjectID)
cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
}
@@ -237,16 +239,6 @@ func gcsToObjectError(err error, params ...string) error {
return nil
}
e, ok := err.(*errors.Error)
if !ok {
// Code should be fixed if this function is called without doing errors.Trace()
// Else handling different situations in this function makes this function complicated.
minio.ErrorIf(err, "Expected type *Error")
return err
}
err = e.Cause
bucket := ""
object := ""
uploadID := ""
@@ -266,8 +258,7 @@ func gcsToObjectError(err error, params ...string) error {
err = minio.BucketNotFound{
Bucket: bucket,
}
e.Cause = err
return e
return err
case "storage: object doesn't exist":
if uploadID != "" {
err = minio.InvalidUploadID{
@@ -279,21 +270,18 @@ func gcsToObjectError(err error, params ...string) error {
Object: object,
}
}
e.Cause = err
return e
return err
}
googleAPIErr, ok := err.(*googleapi.Error)
if !ok {
// We don't interpret non Minio errors. As minio errors will
// have StatusCode to help to convert to object errors.
e.Cause = err
return e
return err
}
if len(googleAPIErr.Errors) == 0 {
e.Cause = err
return e
return err
}
reason := googleAPIErr.Errors[0].Reason
@@ -337,8 +325,7 @@ func gcsToObjectError(err error, params ...string) error {
err = fmt.Errorf("Unsupported error reason: %s", reason)
}
e.Cause = err
return e
return err
}
// gcsProjectIDRegex defines a valid gcs project id format
@@ -381,7 +368,9 @@ func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(bucket string) {
attrs, err := it.Next()
if err != nil {
if err != iterator.Done {
minio.ErrorIf(err, "Object listing error on bucket %s during purging of old files in minio.sys.tmp", bucket)
reqInfo := &logger.ReqInfo{BucketName: bucket}
ctx := logger.SetReqInfo(context.Background(), reqInfo)
logger.LogIf(ctx, err)
}
return
}
@@ -389,7 +378,9 @@ func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(bucket string) {
// Delete files older than 2 weeks.
err := l.client.Bucket(bucket).Object(attrs.Name).Delete(l.ctx)
if err != nil {
minio.ErrorIf(err, "Unable to delete %s/%s during purging of old files in minio.sys.tmp", bucket, attrs.Name)
reqInfo := &logger.ReqInfo{BucketName: bucket, ObjectName: attrs.Name}
ctx := logger.SetReqInfo(context.Background(), reqInfo)
logger.LogIf(ctx, err)
return
}
}
@@ -404,7 +395,8 @@ func (l *gcsGateway) CleanupGCSMinioSysTmp() {
attrs, err := it.Next()
if err != nil {
if err != iterator.Done {
minio.ErrorIf(err, "Bucket listing error during purging of old files in minio.sys.tmp")
ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{})
logger.LogIf(ctx, err)
}
break
}
@@ -438,15 +430,16 @@ func (l *gcsGateway) MakeBucketWithLocation(ctx context.Context, bucket, locatio
err := bkt.Create(l.ctx, l.projectID, &storage.BucketAttrs{
Location: location,
})
return gcsToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return gcsToObjectError(err, bucket)
}
// GetBucketInfo - Get bucket metadata..
func (l *gcsGateway) GetBucketInfo(ctx context.Context, bucket string) (minio.BucketInfo, error) {
attrs, err := l.client.Bucket(bucket).Attrs(l.ctx)
if err != nil {
return minio.BucketInfo{}, gcsToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return minio.BucketInfo{}, gcsToObjectError(err, bucket)
}
return minio.BucketInfo{
@@ -467,7 +460,8 @@ func (l *gcsGateway) ListBuckets(ctx context.Context) (buckets []minio.BucketInf
}
if ierr != nil {
return buckets, gcsToObjectError(errors.Trace(ierr))
logger.LogIf(ctx, ierr)
return buckets, gcsToObjectError(ierr)
}
buckets = append(buckets, minio.BucketInfo{
@@ -495,7 +489,8 @@ func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string) error {
break
}
if err != nil {
return gcsToObjectError(errors.Trace(err))
logger.LogIf(ctx, err)
return gcsToObjectError(err)
}
if objAttrs.Prefix == minio.GatewayMinioSysTmp {
gcsMinioPathFound = true
@@ -505,7 +500,8 @@ func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string) error {
break
}
if nonGCSMinioPathFound {
return gcsToObjectError(errors.Trace(minio.BucketNotEmpty{}))
logger.LogIf(ctx, minio.BucketNotEmpty{})
return gcsToObjectError(minio.BucketNotEmpty{})
}
if gcsMinioPathFound {
// Remove minio.sys.tmp before deleting the bucket.
@@ -516,16 +512,19 @@ func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string) error {
break
}
if err != nil {
return gcsToObjectError(errors.Trace(err))
logger.LogIf(ctx, err)
return gcsToObjectError(err)
}
err = l.client.Bucket(bucket).Object(objAttrs.Name).Delete(l.ctx)
if err != nil {
return gcsToObjectError(errors.Trace(err))
logger.LogIf(ctx, err)
return gcsToObjectError(err)
}
}
}
err := l.client.Bucket(bucket).Delete(l.ctx)
return gcsToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return gcsToObjectError(err, bucket)
}
func toGCSPageToken(name string) string {
@@ -607,7 +606,8 @@ func (l *gcsGateway) ListObjects(ctx context.Context, bucket string, prefix stri
break
}
if err != nil {
return minio.ListObjectsInfo{}, gcsToObjectError(errors.Trace(err), bucket, prefix)
logger.LogIf(ctx, err)
return minio.ListObjectsInfo{}, gcsToObjectError(err, bucket, prefix)
}
nextMarker = toGCSPageToken(attrs.Name)
@@ -689,7 +689,8 @@ func (l *gcsGateway) ListObjectsV2(ctx context.Context, bucket, prefix, continua
}
if err != nil {
return minio.ListObjectsV2Info{}, gcsToObjectError(errors.Trace(err), bucket, prefix)
logger.LogIf(ctx, err)
return minio.ListObjectsV2Info{}, gcsToObjectError(err, bucket, prefix)
}
if attrs.Prefix == minio.GatewayMinioSysTmp {
@@ -733,18 +734,21 @@ func (l *gcsGateway) GetObject(ctx context.Context, bucket string, key string, s
// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
// otherwise gcs will just return object not exist in case of non-existing bucket
if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
return gcsToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return gcsToObjectError(err, bucket)
}
object := l.client.Bucket(bucket).Object(key)
r, err := object.NewRangeReader(l.ctx, startOffset, length)
if err != nil {
return gcsToObjectError(errors.Trace(err), bucket, key)
logger.LogIf(ctx, err)
return gcsToObjectError(err, bucket, key)
}
defer r.Close()
if _, err := io.Copy(writer, r); err != nil {
return gcsToObjectError(errors.Trace(err), bucket, key)
logger.LogIf(ctx, err)
return gcsToObjectError(err, bucket, key)
}
return nil
@@ -771,12 +775,14 @@ func (l *gcsGateway) GetObjectInfo(ctx context.Context, bucket string, object st
// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
// otherwise gcs will just return object not exist in case of non-existing bucket
if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return minio.ObjectInfo{}, gcsToObjectError(err, bucket)
}
attrs, err := l.client.Bucket(bucket).Object(object).Attrs(l.ctx)
if err != nil {
return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, object)
}
return fromGCSAttrsToObjectInfo(attrs), nil
@@ -787,7 +793,8 @@ func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, d
// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
// otherwise gcs will just return object not exist in case of non-existing bucket
if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return minio.ObjectInfo{}, gcsToObjectError(err, bucket)
}
object := l.client.Bucket(bucket).Object(key)
@@ -801,7 +808,8 @@ func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, d
if _, err := io.Copy(w, data); err != nil {
// Close the object writer upon error.
w.CloseWithError(err)
return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), bucket, key)
logger.LogIf(ctx, err)
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
}
// Close the object writer upon success.
@@ -809,7 +817,8 @@ func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, d
attrs, err := object.Attrs(l.ctx)
if err != nil {
return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), bucket, key)
logger.LogIf(ctx, err)
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
}
return fromGCSAttrsToObjectInfo(attrs), nil
@@ -827,7 +836,8 @@ func (l *gcsGateway) CopyObject(ctx context.Context, srcBucket string, srcObject
attrs, err := copier.Run(l.ctx)
if err != nil {
return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), destBucket, destObject)
logger.LogIf(ctx, err)
return minio.ObjectInfo{}, gcsToObjectError(err, destBucket, destObject)
}
return fromGCSAttrsToObjectInfo(attrs), nil
@@ -837,7 +847,8 @@ func (l *gcsGateway) CopyObject(ctx context.Context, srcBucket string, srcObject
func (l *gcsGateway) DeleteObject(ctx context.Context, bucket string, object string) error {
err := l.client.Bucket(bucket).Object(object).Delete(l.ctx)
if err != nil {
return gcsToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return gcsToObjectError(err, bucket, object)
}
return nil
@@ -863,7 +874,8 @@ func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key
bucket,
key,
}); err != nil {
return "", gcsToObjectError(errors.Trace(err), bucket, key)
logger.LogIf(ctx, err)
return "", gcsToObjectError(err, bucket, key)
}
return uploadID, nil
}
@@ -883,7 +895,8 @@ func (l *gcsGateway) ListMultipartUploads(ctx context.Context, bucket string, pr
// an object layer compatible error upon any error.
func (l *gcsGateway) checkUploadIDExists(ctx context.Context, bucket string, key string, uploadID string) error {
_, err := l.client.Bucket(bucket).Object(gcsMultipartMetaName(uploadID)).Attrs(l.ctx)
return gcsToObjectError(errors.Trace(err), bucket, key, uploadID)
logger.LogIf(ctx, err)
return gcsToObjectError(err, bucket, key, uploadID)
}
// PutObjectPart puts a part of object in bucket
@@ -904,7 +917,8 @@ func (l *gcsGateway) PutObjectPart(ctx context.Context, bucket string, key strin
if _, err := io.Copy(w, data); err != nil {
// Make sure to close object writer upon error.
w.Close()
return minio.PartInfo{}, gcsToObjectError(errors.Trace(err), bucket, key)
logger.LogIf(ctx, err)
return minio.PartInfo{}, gcsToObjectError(err, bucket, key)
}
// Make sure to close the object writer upon success.
w.Close()
@@ -923,7 +937,7 @@ func (l *gcsGateway) ListObjectParts(ctx context.Context, bucket string, key str
}
// Called by AbortMultipartUpload and CompleteMultipartUpload for cleaning up.
func (l *gcsGateway) cleanupMultipartUpload(bucket, key, uploadID string) error {
func (l *gcsGateway) cleanupMultipartUpload(ctx context.Context, bucket, key, uploadID string) error {
prefix := fmt.Sprintf("%s/%s/", gcsMinioMultipartPathV1, uploadID)
// iterate through all parts and delete them
@@ -935,7 +949,8 @@ func (l *gcsGateway) cleanupMultipartUpload(bucket, key, uploadID string) error
break
}
if err != nil {
return gcsToObjectError(errors.Trace(err), bucket, key)
logger.LogIf(ctx, err)
return gcsToObjectError(err, bucket, key)
}
object := l.client.Bucket(bucket).Object(attrs.Name)
@@ -951,7 +966,7 @@ func (l *gcsGateway) AbortMultipartUpload(ctx context.Context, bucket string, ke
if err := l.checkUploadIDExists(ctx, bucket, key, uploadID); err != nil {
return err
}
return l.cleanupMultipartUpload(bucket, key, uploadID)
return l.cleanupMultipartUpload(ctx, bucket, key, uploadID)
}
// CompleteMultipartUpload completes ongoing multipart upload and finalizes object
@@ -968,23 +983,27 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
partZeroAttrs, err := object.Attrs(l.ctx)
if err != nil {
return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), bucket, key, uploadID)
logger.LogIf(ctx, err)
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key, uploadID)
}
r, err := object.NewReader(l.ctx)
if err != nil {
return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), bucket, key)
logger.LogIf(ctx, err)
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
}
defer r.Close()
// Check version compatibility of the meta file before compose()
multipartMeta := gcsMultipartMetaV1{}
if err = json.NewDecoder(r).Decode(&multipartMeta); err != nil {
return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), bucket, key)
logger.LogIf(ctx, err)
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
}
if multipartMeta.Version != gcsMinioMultipartMetaCurrentVersion {
return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(errGCSFormat), bucket, key)
logger.LogIf(ctx, errGCSFormat)
return minio.ObjectInfo{}, gcsToObjectError(errGCSFormat, bucket, key)
}
// Validate if the gcs.json stores valid entries for the bucket and key.
@@ -1001,7 +1020,8 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
uploadedPart.PartNumber, uploadedPart.ETag)))
partAttr, pErr := l.client.Bucket(bucket).Object(gcsMultipartDataName(uploadID, uploadedPart.PartNumber, uploadedPart.ETag)).Attrs(l.ctx)
if pErr != nil {
return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(pErr), bucket, key, uploadID)
logger.LogIf(ctx, pErr)
return minio.ObjectInfo{}, gcsToObjectError(pErr, bucket, key, uploadID)
}
partSizes[i] = partAttr.Size
}
@@ -1009,11 +1029,16 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
// Error out if parts except last part sizing < 5MiB.
for i, size := range partSizes[:len(partSizes)-1] {
if size < 5*humanize.MiByte {
return minio.ObjectInfo{}, errors.Trace(minio.PartTooSmall{
logger.LogIf(ctx, minio.PartTooSmall{
PartNumber: uploadedParts[i].PartNumber,
PartSize: size,
PartETag: uploadedParts[i].ETag,
})
return minio.ObjectInfo{}, minio.PartTooSmall{
PartNumber: uploadedParts[i].PartNumber,
PartSize: size,
PartETag: uploadedParts[i].ETag,
}
}
}
@@ -1040,7 +1065,8 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
composer.Metadata = partZeroAttrs.Metadata
if _, err = composer.Run(l.ctx); err != nil {
return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), bucket, key)
logger.LogIf(ctx, err)
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
}
}
@@ -1053,10 +1079,11 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
composer.Metadata = partZeroAttrs.Metadata
attrs, err := composer.Run(l.ctx)
if err != nil {
return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), bucket, key)
logger.LogIf(ctx, err)
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
}
if err = l.cleanupMultipartUpload(bucket, key, uploadID); err != nil {
return minio.ObjectInfo{}, gcsToObjectError(errors.Trace(err), bucket, key)
if err = l.cleanupMultipartUpload(ctx, bucket, key, uploadID); err != nil {
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
}
return fromGCSAttrsToObjectInfo(attrs), nil
}
@@ -1075,16 +1102,19 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, policyI
prefix := bucket + "/*" // For all objects inside the bucket.
if len(policies) != 1 {
return errors.Trace(minio.NotImplemented{})
logger.LogIf(ctx, minio.NotImplemented{})
return minio.NotImplemented{}
}
if policies[0].Prefix != prefix {
return errors.Trace(minio.NotImplemented{})
logger.LogIf(ctx, minio.NotImplemented{})
return minio.NotImplemented{}
}
acl := l.client.Bucket(bucket).ACL()
if policies[0].Policy == policy.BucketPolicyNone {
if err := acl.Delete(l.ctx, storage.AllUsers); err != nil {
return gcsToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return gcsToObjectError(err, bucket)
}
return nil
}
@@ -1096,11 +1126,13 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, policyI
case policy.BucketPolicyWriteOnly:
role = storage.RoleWriter
default:
return errors.Trace(minio.NotImplemented{})
logger.LogIf(ctx, minio.NotImplemented{})
return minio.NotImplemented{}
}
if err := acl.Set(l.ctx, storage.AllUsers, role); err != nil {
return gcsToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return gcsToObjectError(err, bucket)
}
return nil
@@ -1110,7 +1142,8 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, policyI
func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) {
rules, err := l.client.Bucket(bucket).ACL().List(l.ctx)
if err != nil {
return policy.BucketAccessPolicy{}, gcsToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return policy.BucketAccessPolicy{}, gcsToObjectError(err, bucket)
}
policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"}
for _, r := range rules {
@@ -1126,7 +1159,8 @@ func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (policy
}
// Return NoSuchBucketPolicy error, when policy is not set
if len(policyInfo.Statements) == 0 {
return policy.BucketAccessPolicy{}, gcsToObjectError(errors.Trace(minio.PolicyNotFound{}), bucket)
logger.LogIf(ctx, minio.PolicyNotFound{})
return policy.BucketAccessPolicy{}, gcsToObjectError(minio.PolicyNotFound{}, bucket)
}
return policyInfo, nil
}
@@ -1135,7 +1169,8 @@ func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (policy
func (l *gcsGateway) DeleteBucketPolicy(ctx context.Context, bucket string) error {
// This only removes the storage.AllUsers policies
if err := l.client.Bucket(bucket).ACL().Delete(l.ctx, storage.AllUsers); err != nil {
return gcsToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return gcsToObjectError(err, bucket)
}
return nil

53
cmd/gateway/gcs/gateway-gcs_test.go
View File

@@ -24,7 +24,6 @@ import (
"reflect"
"testing"
"github.com/minio/minio/pkg/errors"
"google.golang.org/api/googleapi"
miniogo "github.com/minio/minio-go"
@@ -237,14 +236,14 @@ func TestGCSToObjectError(t *testing.T) {
},
{
[]string{"bucket"},
errors.Trace(fmt.Errorf("storage: bucket doesn't exist")),
fmt.Errorf("storage: bucket doesn't exist"),
minio.BucketNotFound{
Bucket: "bucket",
},
},
{
[]string{"bucket", "object"},
errors.Trace(fmt.Errorf("storage: object doesn't exist")),
fmt.Errorf("storage: object doesn't exist"),
minio.ObjectNotFound{
Bucket: "bucket",
Object: "object",
@@ -252,76 +251,76 @@ func TestGCSToObjectError(t *testing.T) {
},
{
[]string{"bucket", "object", "uploadID"},
errors.Trace(fmt.Errorf("storage: object doesn't exist")),
fmt.Errorf("storage: object doesn't exist"),
minio.InvalidUploadID{
UploadID: "uploadID",
},
},
{
[]string{},
errors.Trace(fmt.Errorf("Unknown error")),
fmt.Errorf("Unknown error"),
fmt.Errorf("Unknown error"),
},
{
[]string{"bucket", "object"},
errors.Trace(&googleapi.Error{
&googleapi.Error{
Message: "No list of errors",
}),
},
&googleapi.Error{
Message: "No list of errors",
},
},
{
[]string{"bucket", "object"},
errors.Trace(&googleapi.Error{
&googleapi.Error{
Errors: []googleapi.ErrorItem{{
Reason: "conflict",
Message: "You already own this bucket. Please select another name.",
}},
}),
},
minio.BucketAlreadyOwnedByYou{
Bucket: "bucket",
},
},
{
[]string{"bucket", "object"},
errors.Trace(&googleapi.Error{
&googleapi.Error{
Errors: []googleapi.ErrorItem{{
Reason: "conflict",
Message: "Sorry, that name is not available. Please try a different one.",
}},
}),
},
minio.BucketAlreadyExists{
Bucket: "bucket",
},
},
{
[]string{"bucket", "object"},
errors.Trace(&googleapi.Error{
&googleapi.Error{
Errors: []googleapi.ErrorItem{{
Reason: "conflict",
}},
}),
},
minio.BucketNotEmpty{Bucket: "bucket"},
},
{
[]string{"bucket"},
errors.Trace(&googleapi.Error{
&googleapi.Error{
Errors: []googleapi.ErrorItem{{
Reason: "notFound",
}},
}),
},
minio.BucketNotFound{
Bucket: "bucket",
},
},
{
[]string{"bucket", "object"},
errors.Trace(&googleapi.Error{
&googleapi.Error{
Errors: []googleapi.ErrorItem{{
Reason: "notFound",
}},
}),
},
minio.ObjectNotFound{
Bucket: "bucket",
Object: "object",
@@ -329,22 +328,22 @@ func TestGCSToObjectError(t *testing.T) {
},
{
[]string{"bucket"},
errors.Trace(&googleapi.Error{
&googleapi.Error{
Errors: []googleapi.ErrorItem{{
Reason: "invalid",
}},
}),
},
minio.BucketNameInvalid{
Bucket: "bucket",
},
},
{
[]string{"bucket", "object"},
errors.Trace(&googleapi.Error{
&googleapi.Error{
Errors: []googleapi.ErrorItem{{
Reason: "forbidden",
}},
}),
},
minio.PrefixAccessDenied{
Bucket: "bucket",
Object: "object",
@@ -352,11 +351,11 @@ func TestGCSToObjectError(t *testing.T) {
},
{
[]string{"bucket", "object"},
errors.Trace(&googleapi.Error{
&googleapi.Error{
Errors: []googleapi.ErrorItem{{
Reason: "keyInvalid",
}},
}),
},
minio.PrefixAccessDenied{
Bucket: "bucket",
Object: "object",
@@ -364,11 +363,11 @@ func TestGCSToObjectError(t *testing.T) {
},
{
[]string{"bucket", "object"},
errors.Trace(&googleapi.Error{
&googleapi.Error{
Errors: []googleapi.ErrorItem{{
Reason: "required",
}},
}),
},
minio.PrefixAccessDenied{
Bucket: "bucket",
Object: "object",
@@ -376,11 +375,11 @@ func TestGCSToObjectError(t *testing.T) {
},
{
[]string{"bucket", "object"},
errors.Trace(&googleapi.Error{
&googleapi.Error{
Errors: []googleapi.ErrorItem{{
Reason: "unknown",
}},
}),
},
fmt.Errorf("Unsupported error reason: unknown"),
},
}

38
cmd/gateway/manta/gateway-manta.go
View File

@@ -33,8 +33,8 @@ import (
"github.com/joyent/triton-go/storage"
"github.com/minio/cli"
minio "github.com/minio/minio/cmd"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/errors"
"github.com/minio/minio/pkg/hash"
)
@@ -118,7 +118,7 @@ func mantaGatewayMain(ctx *cli.Context) {
// Validate gateway arguments.
host := ctx.Args().First()
// Validate gateway arguments.
minio.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
minio.StartGateway(ctx, &Manta{host})
}
@@ -139,6 +139,7 @@ func (g *Manta) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, erro
var err error
var signer authentication.Signer
var endpoint = defaultMantaURL
ctx := context.Background()
if g.host != "" {
endpoint, _, err = minio.ParseGatewayEndpoint(g.host)
@@ -163,7 +164,8 @@ func (g *Manta) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, erro
}
signer, err = authentication.NewSSHAgentSigner(input)
if err != nil {
return nil, errors.Trace(err)
logger.LogIf(ctx, err)
return nil, err
}
} else {
var keyBytes []byte
@@ -200,7 +202,8 @@ func (g *Manta) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, erro
signer, err = authentication.NewPrivateKeySigner(input)
if err != nil {
return nil, errors.Trace(err)
logger.LogIf(ctx, err)
return nil, err
}
}
@@ -352,7 +355,8 @@ func (t *tritonObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
if terrors.IsResourceNotFoundError(err) {
return result, nil
}
return result, errors.Trace(err)
logger.LogIf(ctx, err)
return result, err
}
for _, obj := range objs.Entries {
@@ -362,7 +366,8 @@ func (t *tritonObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
input.DirectoryName = path.Join(mantaRoot, bucket, prefix)
objs, err = t.client.Dir().List(ctx, input)
if err != nil {
return result, errors.Trace(err)
logger.LogIf(ctx, err)
return result, err
}
break
}
@@ -428,7 +433,8 @@ func (t *tritonObjects) ListObjectsV2(ctx context.Context, bucket, prefix, conti
if terrors.IsResourceNotFoundError(err) {
return result, nil
}
return result, errors.Trace(err)
logger.LogIf(ctx, err)
return result, err
}
for _, obj := range objs.Entries {
@@ -436,7 +442,8 @@ func (t *tritonObjects) ListObjectsV2(ctx context.Context, bucket, prefix, conti
input.DirectoryName = path.Join(mantaRoot, bucket, prefix)
objs, err = t.client.Dir().List(ctx, input)
if err != nil {
return result, errors.Trace(err)
logger.LogIf(ctx, err)
return result, err
}
break
}
@@ -479,7 +486,8 @@ func (t *tritonObjects) ListObjectsV2(ctx context.Context, bucket, prefix, conti
func (t *tritonObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
// Start offset cannot be negative.
if startOffset < 0 {
return errors.Trace(fmt.Errorf("Unexpected error"))
logger.LogIf(ctx, fmt.Errorf("Unexpected error"))
return fmt.Errorf("Unexpected error")
}
output, err := t.client.Objects().Get(ctx, &storage.GetObjectInput{
@@ -555,11 +563,13 @@ func (t *tritonObjects) PutObject(ctx context.Context, bucket, object string, da
ObjectReader: dummySeeker{data},
ForceInsert: true,
}); err != nil {
return objInfo, errors.Trace(err)
logger.LogIf(ctx, err)
return objInfo, err
}
if err = data.Verify(); err != nil {
t.DeleteObject(ctx, bucket, object)
return objInfo, errors.Trace(err)
logger.LogIf(ctx, err)
return objInfo, err
}
return t.GetObjectInfo(ctx, bucket, object)
@@ -574,7 +584,8 @@ func (t *tritonObjects) CopyObject(ctx context.Context, srcBucket, srcObject, de
SourcePath: path.Join(mantaRoot, srcBucket, srcObject),
LinkPath: path.Join(mantaRoot, destBucket, destObject),
}); err != nil {
return objInfo, errors.Trace(err)
logger.LogIf(ctx, err)
return objInfo, err
}
return t.GetObjectInfo(ctx, destBucket, destObject)
@@ -587,7 +598,8 @@ func (t *tritonObjects) DeleteObject(ctx context.Context, bucket, object string)
if err := t.client.Objects().Delete(ctx, &storage.DeleteObjectInput{
ObjectPath: path.Join(mantaRoot, bucket, object),
}); err != nil {
return errors.Trace(err)
logger.LogIf(ctx, err)
return err
}
return nil

192
cmd/gateway/oss/gateway-oss.go
View File

@@ -32,8 +32,8 @@ import (
"github.com/minio/cli"
"github.com/minio/minio-go/pkg/policy"
minio "github.com/minio/minio/cmd"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/errors"
"github.com/minio/minio/pkg/hash"
)
@@ -113,7 +113,7 @@ func ossGatewayMain(ctx *cli.Context) {
// Validate gateway arguments.
host := ctx.Args().First()
minio.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
minio.StartGateway(ctx, &OSS{host})
}
@@ -161,7 +161,7 @@ func (g *OSS) Production() bool {
// `X-Amz-Meta-` prefix and converted into `X-Oss-Meta-`.
//
// Header names are canonicalized as in http.Header.
func appendS3MetaToOSSOptions(opts []oss.Option, s3Metadata map[string]string) ([]oss.Option, error) {
func appendS3MetaToOSSOptions(ctx context.Context, opts []oss.Option, s3Metadata map[string]string) ([]oss.Option, error) {
if opts == nil {
opts = make([]oss.Option, 0, len(s3Metadata))
}
@@ -173,7 +173,8 @@ func appendS3MetaToOSSOptions(opts []oss.Option, s3Metadata map[string]string) (
metaKey := k[len("X-Amz-Meta-"):]
// NOTE(timonwong): OSS won't allow headers with underscore(_).
if strings.Contains(metaKey, "_") {
return nil, errors.Trace(minio.UnsupportedMetadata{})
logger.LogIf(ctx, minio.UnsupportedMetadata{})
return nil, minio.UnsupportedMetadata{}
}
opts = append(opts, oss.Meta(metaKey, v))
case k == "X-Amz-Acl":
@@ -271,15 +272,6 @@ func ossToObjectError(err error, params ...string) error {
return nil
}
e, ok := err.(*errors.Error)
if !ok {
// Code should be fixed if this function is called without doing errors.Trace()
// Else handling different situations in this function makes this function complicated.
minio.ErrorIf(err, "Expected type *Error")
return err
}
err = e.Cause
bucket := ""
object := ""
uploadID := ""
@@ -298,7 +290,7 @@ func ossToObjectError(err error, params ...string) error {
if !ok {
// We don't interpret non OSS errors. As oss errors will
// have StatusCode to help to convert to object errors.
return e
return err
}
switch ossErr.Code {
@@ -330,8 +322,7 @@ func ossToObjectError(err error, params ...string) error {
err = minio.InvalidPart{}
}
e.Cause = err
return e
return err
}
// ossObjects implements gateway for Aliyun Object Storage Service.
@@ -366,22 +357,26 @@ func ossIsValidBucketName(bucket string) bool {
// MakeBucketWithLocation creates a new container on OSS backend.
func (l *ossObjects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error {
if !ossIsValidBucketName(bucket) {
return errors.Trace(minio.BucketNameInvalid{Bucket: bucket})
logger.LogIf(ctx, minio.BucketNameInvalid{Bucket: bucket})
return minio.BucketNameInvalid{Bucket: bucket}
}
err := l.Client.CreateBucket(bucket)
return ossToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return ossToObjectError(err, bucket)
}
// ossGeBucketInfo gets bucket metadata.
func ossGeBucketInfo(client *oss.Client, bucket string) (bi minio.BucketInfo, err error) {
func ossGeBucketInfo(ctx context.Context, client *oss.Client, bucket string) (bi minio.BucketInfo, err error) {
if !ossIsValidBucketName(bucket) {
return bi, errors.Trace(minio.BucketNameInvalid{Bucket: bucket})
logger.LogIf(ctx, minio.BucketNameInvalid{Bucket: bucket})
return bi, minio.BucketNameInvalid{Bucket: bucket}
}
bgir, err := client.GetBucketInfo(bucket)
if err != nil {
return bi, ossToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return bi, ossToObjectError(err, bucket)
}
return minio.BucketInfo{
@@ -392,7 +387,7 @@ func ossGeBucketInfo(client *oss.Client, bucket string) (bi minio.BucketInfo, er
// GetBucketInfo gets bucket metadata.
func (l *ossObjects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.BucketInfo, err error) {
return ossGeBucketInfo(l.Client, bucket)
return ossGeBucketInfo(ctx, l.Client, bucket)
}
// ListBuckets lists all OSS buckets.
@@ -401,7 +396,8 @@ func (l *ossObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketInf
for {
lbr, err := l.Client.ListBuckets(marker)
if err != nil {
return nil, ossToObjectError(errors.Trace(err))
logger.LogIf(ctx, err)
return nil, ossToObjectError(err)
}
for _, bi := range lbr.Buckets {
@@ -424,7 +420,8 @@ func (l *ossObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketInf
func (l *ossObjects) DeleteBucket(ctx context.Context, bucket string) error {
err := l.Client.DeleteBucket(bucket)
if err != nil {
return ossToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return ossToObjectError(err, bucket)
}
return nil
}
@@ -462,10 +459,11 @@ func fromOSSClientListObjectsResult(bucket string, lor oss.ListObjectsResult) mi
}
// ossListObjects lists all blobs in OSS bucket filtered by prefix.
func ossListObjects(client *oss.Client, bucket, prefix, marker, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) {
func ossListObjects(ctx context.Context, client *oss.Client, bucket, prefix, marker, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) {
buck, err := client.Bucket(bucket)
if err != nil {
return loi, ossToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return loi, ossToObjectError(err, bucket)
}
// maxKeys should default to 1000 or less.
@@ -475,19 +473,20 @@ func ossListObjects(client *oss.Client, bucket, prefix, marker, delimiter string
lor, err := buck.ListObjects(oss.Prefix(prefix), oss.Marker(marker), oss.Delimiter(delimiter), oss.MaxKeys(maxKeys))
if err != nil {
return loi, ossToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return loi, ossToObjectError(err, bucket)
}
return fromOSSClientListObjectsResult(bucket, lor), nil
}
// ossListObjectsV2 lists all blobs in OSS bucket filtered by prefix.
func ossListObjectsV2(client *oss.Client, bucket, prefix, continuationToken, delimiter string, maxKeys int,
func ossListObjectsV2(ctx context.Context, client *oss.Client, bucket, prefix, continuationToken, delimiter string, maxKeys int,
fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) {
// fetchOwner and startAfter are not supported and unused.
marker := continuationToken
resultV1, err := ossListObjects(client, bucket, prefix, marker, delimiter, maxKeys)
resultV1, err := ossListObjects(ctx, client, bucket, prefix, marker, delimiter, maxKeys)
if err != nil {
return loi, err
}
@@ -503,13 +502,13 @@ func ossListObjectsV2(client *oss.Client, bucket, prefix, continuationToken, del
// ListObjects lists all blobs in OSS bucket filtered by prefix.
func (l *ossObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) {
return ossListObjects(l.Client, bucket, prefix, marker, delimiter, maxKeys)
return ossListObjects(ctx, l.Client, bucket, prefix, marker, delimiter, maxKeys)
}
// ListObjectsV2 lists all blobs in OSS bucket filtered by prefix
func (l *ossObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int,
fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) {
return ossListObjectsV2(l.Client, bucket, prefix, continuationToken, delimiter, maxKeys, fetchOwner, startAfter)
return ossListObjectsV2(ctx, l.Client, bucket, prefix, continuationToken, delimiter, maxKeys, fetchOwner, startAfter)
}
// ossGetObject reads an object on OSS. Supports additional
@@ -518,14 +517,16 @@ func (l *ossObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continua
//
// startOffset indicates the starting read location of the object.
// length indicates the total length of the object.
func ossGetObject(client *oss.Client, bucket, key string, startOffset, length int64, writer io.Writer, etag string) error {
func ossGetObject(ctx context.Context, client *oss.Client, bucket, key string, startOffset, length int64, writer io.Writer, etag string) error {
if length < 0 && length != -1 {
return ossToObjectError(errors.Trace(fmt.Errorf("Invalid argument")), bucket, key)
logger.LogIf(ctx, fmt.Errorf("Invalid argument"))
return ossToObjectError(fmt.Errorf("Invalid argument"), bucket, key)
}
bkt, err := client.Bucket(bucket)
if err != nil {
return ossToObjectError(errors.Trace(err), bucket, key)
logger.LogIf(ctx, err)
return ossToObjectError(err, bucket, key)
}
var opts []oss.Option
@@ -535,12 +536,14 @@ func ossGetObject(client *oss.Client, bucket, key string, startOffset, length in
object, err := bkt.GetObject(key, opts...)
if err != nil {
return ossToObjectError(errors.Trace(err), bucket, key)
logger.LogIf(ctx, err)
return ossToObjectError(err, bucket, key)
}
defer object.Close()
if _, err := io.Copy(writer, object); err != nil {
return ossToObjectError(errors.Trace(err), bucket, key)
logger.LogIf(ctx, err)
return ossToObjectError(err, bucket, key)
}
return nil
}
@@ -552,7 +555,7 @@ func ossGetObject(client *oss.Client, bucket, key string, startOffset, length in
// startOffset indicates the starting read location of the object.
// length indicates the total length of the object.
func (l *ossObjects) GetObject(ctx context.Context, bucket, key string, startOffset, length int64, writer io.Writer, etag string) error {
return ossGetObject(l.Client, bucket, key, startOffset, length, writer, etag)
return ossGetObject(ctx, l.Client, bucket, key, startOffset, length, writer, etag)
}
func translatePlainError(err error) error {
@@ -569,15 +572,17 @@ func translatePlainError(err error) error {
}
// ossGetObjectInfo reads object info and replies back ObjectInfo.
func ossGetObjectInfo(client *oss.Client, bucket, object string) (objInfo minio.ObjectInfo, err error) {
func ossGetObjectInfo(ctx context.Context, client *oss.Client, bucket, object string) (objInfo minio.ObjectInfo, err error) {
bkt, err := client.Bucket(bucket)
if err != nil {
return objInfo, ossToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return objInfo, ossToObjectError(err, bucket, object)
}
header, err := bkt.GetObjectDetailedMeta(object)
if err != nil {
return objInfo, ossToObjectError(errors.Trace(translatePlainError(err)), bucket, object)
logger.LogIf(ctx, translatePlainError(err))
return objInfo, ossToObjectError(translatePlainError(err), bucket, object)
}
// Build S3 metadata from OSS metadata
@@ -600,40 +605,43 @@ func ossGetObjectInfo(client *oss.Client, bucket, object string) (objInfo minio.
// GetObjectInfo reads object info and replies back ObjectInfo.
func (l *ossObjects) GetObjectInfo(ctx context.Context, bucket, object string) (objInfo minio.ObjectInfo, err error) {
return ossGetObjectInfo(l.Client, bucket, object)
return ossGetObjectInfo(ctx, l.Client, bucket, object)
}
// ossPutObject creates a new object with the incoming data.
func ossPutObject(client *oss.Client, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
func ossPutObject(ctx context.Context, client *oss.Client, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
bkt, err := client.Bucket(bucket)
if err != nil {
return objInfo, ossToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return objInfo, ossToObjectError(err, bucket, object)
}
// Build OSS metadata
opts, err := appendS3MetaToOSSOptions(nil, metadata)
opts, err := appendS3MetaToOSSOptions(ctx, nil, metadata)
if err != nil {
return objInfo, ossToObjectError(err, bucket, object)
}
err = bkt.PutObject(object, data, opts...)
if err != nil {
return objInfo, ossToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return objInfo, ossToObjectError(err, bucket, object)
}
return ossGetObjectInfo(client, bucket, object)
return ossGetObjectInfo(ctx, client, bucket, object)
}
// PutObject creates a new object with the incoming data.
func (l *ossObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
return ossPutObject(l.Client, bucket, object, data, metadata)
return ossPutObject(ctx, l.Client, bucket, object, data, metadata)
}
// CopyObject copies an object from source bucket to a destination bucket.
func (l *ossObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) {
bkt, err := l.Client.Bucket(srcBucket)
if err != nil {
return objInfo, ossToObjectError(errors.Trace(err), srcBucket, srcObject)
logger.LogIf(ctx, err)
return objInfo, ossToObjectError(err, srcBucket, srcObject)
}
opts := make([]oss.Option, 0, len(srcInfo.UserDefined)+1)
@@ -644,13 +652,14 @@ func (l *ossObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
opts = append(opts, oss.MetadataDirective(oss.MetaReplace))
// Build OSS metadata
opts, err = appendS3MetaToOSSOptions(opts, srcInfo.UserDefined)
opts, err = appendS3MetaToOSSOptions(ctx, opts, srcInfo.UserDefined)
if err != nil {
return objInfo, ossToObjectError(err, srcBucket, srcObject)
}
if _, err = bkt.CopyObjectTo(dstBucket, dstObject, srcObject, opts...); err != nil {
return objInfo, ossToObjectError(errors.Trace(err), srcBucket, srcObject)
logger.LogIf(ctx, err)
return objInfo, ossToObjectError(err, srcBucket, srcObject)
}
return l.GetObjectInfo(ctx, dstBucket, dstObject)
}
@@ -659,12 +668,14 @@ func (l *ossObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
func (l *ossObjects) DeleteObject(ctx context.Context, bucket, object string) error {
bkt, err := l.Client.Bucket(bucket)
if err != nil {
return ossToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return ossToObjectError(err, bucket, object)
}
err = bkt.DeleteObject(object)
if err != nil {
return ossToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return ossToObjectError(err, bucket, object)
}
return nil
}
@@ -701,13 +712,15 @@ func fromOSSClientListMultipartsInfo(lmur oss.ListMultipartUploadResult) minio.L
func (l *ossObjects) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, err error) {
bkt, err := l.Client.Bucket(bucket)
if err != nil {
return lmi, ossToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return lmi, ossToObjectError(err, bucket)
}
lmur, err := bkt.ListMultipartUploads(oss.Prefix(prefix), oss.KeyMarker(keyMarker), oss.UploadIDMarker(uploadIDMarker),
oss.Delimiter(delimiter), oss.MaxUploads(maxUploads))
if err != nil {
return lmi, ossToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return lmi, ossToObjectError(err, bucket)
}
return fromOSSClientListMultipartsInfo(lmur), nil
@@ -717,18 +730,20 @@ func (l *ossObjects) ListMultipartUploads(ctx context.Context, bucket, prefix, k
func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) {
bkt, err := l.Client.Bucket(bucket)
if err != nil {
return uploadID, ossToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return uploadID, ossToObjectError(err, bucket, object)
}
// Build OSS metadata
opts, err := appendS3MetaToOSSOptions(nil, metadata)
opts, err := appendS3MetaToOSSOptions(ctx, nil, metadata)
if err != nil {
return uploadID, ossToObjectError(err, bucket, object)
}
lmur, err := bkt.InitiateMultipartUpload(object, opts...)
if err != nil {
return uploadID, ossToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return uploadID, ossToObjectError(err, bucket, object)
}
return lmur.UploadID, nil
@@ -738,7 +753,8 @@ func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object stri
func (l *ossObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (pi minio.PartInfo, err error) {
bkt, err := l.Client.Bucket(bucket)
if err != nil {
return pi, ossToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return pi, ossToObjectError(err, bucket, object)
}
imur := oss.InitiateMultipartUploadResult{
@@ -749,7 +765,8 @@ func (l *ossObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
size := data.Size()
up, err := bkt.UploadPart(imur, data, size, partID)
if err != nil {
return pi, ossToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return pi, ossToObjectError(err, bucket, object)
}
return minio.PartInfo{
@@ -820,11 +837,12 @@ func (l *ossObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d
bkt, err := l.Client.Bucket(destBucket)
if err != nil {
return p, ossToObjectError(errors.Trace(err), destBucket)
logger.LogIf(ctx, err)
return p, ossToObjectError(err, destBucket)
}
// Build OSS metadata
opts, err := appendS3MetaToOSSOptions(nil, srcInfo.UserDefined)
opts, err := appendS3MetaToOSSOptions(ctx, nil, srcInfo.UserDefined)
if err != nil {
return p, ossToObjectError(err, srcBucket, srcObject)
}
@@ -835,7 +853,8 @@ func (l *ossObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d
}, srcBucket, srcObject, startOffset, length, partID, opts...)
if err != nil {
return p, ossToObjectError(errors.Trace(err), srcBucket, srcObject)
logger.LogIf(ctx, err)
return p, ossToObjectError(err, srcBucket, srcObject)
}
p.PartNumber = completePart.PartNumber
@@ -847,7 +866,8 @@ func (l *ossObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d
func (l *ossObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int) (lpi minio.ListPartsInfo, err error) {
lupr, err := ossListObjectParts(l.Client, bucket, object, uploadID, partNumberMarker, maxParts)
if err != nil {
return lpi, ossToObjectError(errors.Trace(err), bucket, object, uploadID)
logger.LogIf(ctx, err)
return lpi, ossToObjectError(err, bucket, object, uploadID)
}
return fromOSSClientListPartsInfo(lupr, partNumberMarker), nil
@@ -857,7 +877,8 @@ func (l *ossObjects) ListObjectParts(ctx context.Context, bucket, object, upload
func (l *ossObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error {
bkt, err := l.Client.Bucket(bucket)
if err != nil {
return ossToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return ossToObjectError(err, bucket, object)
}
err = bkt.AbortMultipartUpload(oss.InitiateMultipartUploadResult{
@@ -866,7 +887,8 @@ func (l *ossObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
UploadID: uploadID,
})
if err != nil {
return ossToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return ossToObjectError(err, bucket, object)
}
return nil
}
@@ -876,7 +898,8 @@ func (l *ossObjects) CompleteMultipartUpload(ctx context.Context, bucket, object
client := l.Client
bkt, err := client.Bucket(bucket)
if err != nil {
return oi, ossToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return oi, ossToObjectError(err, bucket, object)
}
// Error out if uploadedParts except last part sizing < 5MiB.
@@ -886,7 +909,8 @@ func (l *ossObjects) CompleteMultipartUpload(ctx context.Context, bucket, object
for lupr.IsTruncated {
lupr, err = ossListObjectParts(client, bucket, object, uploadID, partNumberMarker, ossMaxParts)
if err != nil {
return oi, ossToObjectError(errors.Trace(err), bucket, object, uploadID)
logger.LogIf(ctx, err)
return oi, ossToObjectError(err, bucket, object, uploadID)
}
uploadedParts := lupr.UploadedParts
@@ -900,11 +924,16 @@ func (l *ossObjects) CompleteMultipartUpload(ctx context.Context, bucket, object
for _, part := range uploadedParts {
if part.Size < ossS3MinPartSize {
return oi, errors.Trace(minio.PartTooSmall{
logger.LogIf(ctx, minio.PartTooSmall{
PartNumber: part.PartNumber,
PartSize: int64(part.Size),
PartETag: minio.ToS3ETag(part.ETag),
})
return oi, minio.PartTooSmall{
PartNumber: part.PartNumber,
PartSize: int64(part.Size),
PartETag: minio.ToS3ETag(part.ETag),
}
}
}
@@ -926,7 +955,8 @@ func (l *ossObjects) CompleteMultipartUpload(ctx context.Context, bucket, object
_, err = bkt.CompleteMultipartUpload(imur, parts)
if err != nil {
return oi, ossToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return oi, ossToObjectError(err, bucket, object)
}
return l.GetObjectInfo(ctx, bucket, object)
@@ -940,13 +970,15 @@ func (l *ossObjects) CompleteMultipartUpload(ctx context.Context, bucket, object
func (l *ossObjects) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error {
bucketPolicies := policy.GetPolicies(policyInfo.Statements, bucket, "")
if len(bucketPolicies) != 1 {
return errors.Trace(minio.NotImplemented{})
logger.LogIf(ctx, minio.NotImplemented{})
return minio.NotImplemented{}
}
prefix := bucket + "/*" // For all objects inside the bucket.
for policyPrefix, bucketPolicy := range bucketPolicies {
if policyPrefix != prefix {
return errors.Trace(minio.NotImplemented{})
logger.LogIf(ctx, minio.NotImplemented{})
return minio.NotImplemented{}
}
var acl oss.ACLType
@@ -958,12 +990,14 @@ func (l *ossObjects) SetBucketPolicy(ctx context.Context, bucket string, policyI
case policy.BucketPolicyReadWrite:
acl = oss.ACLPublicReadWrite
default:
return errors.Trace(minio.NotImplemented{})
logger.LogIf(ctx, minio.NotImplemented{})
return minio.NotImplemented{}
}
err := l.Client.SetBucketACL(bucket, acl)
if err != nil {
return ossToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return ossToObjectError(err, bucket)
}
}
@@ -974,20 +1008,23 @@ func (l *ossObjects) SetBucketPolicy(ctx context.Context, bucket string, policyI
func (l *ossObjects) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) {
result, err := l.Client.GetBucketACL(bucket)
if err != nil {
return policy.BucketAccessPolicy{}, ossToObjectError(errors.Trace(err))
logger.LogIf(ctx, err)
return policy.BucketAccessPolicy{}, ossToObjectError(err)
}
policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"}
switch result.ACL {
case string(oss.ACLPrivate):
// By default, all buckets starts with a "private" policy.
return policy.BucketAccessPolicy{}, ossToObjectError(errors.Trace(minio.PolicyNotFound{}), bucket)
logger.LogIf(ctx, minio.PolicyNotFound{})
return policy.BucketAccessPolicy{}, ossToObjectError(minio.PolicyNotFound{}, bucket)
case string(oss.ACLPublicRead):
policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyReadOnly, bucket, "")
case string(oss.ACLPublicReadWrite):
policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyReadWrite, bucket, "")
default:
return policy.BucketAccessPolicy{}, errors.Trace(minio.NotImplemented{})
logger.LogIf(ctx, minio.NotImplemented{})
return policy.BucketAccessPolicy{}, minio.NotImplemented{}
}
return policyInfo, nil
@@ -997,7 +1034,8 @@ func (l *ossObjects) GetBucketPolicy(ctx context.Context, bucket string) (policy
func (l *ossObjects) DeleteBucketPolicy(ctx context.Context, bucket string) error {
err := l.Client.SetBucketACL(bucket, oss.ACLPrivate)
if err != nil {
return ossToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return ossToObjectError(err, bucket)
}
return nil
}

9
cmd/gateway/oss/gateway-oss_test.go
View File

@@ -17,6 +17,7 @@
package oss
import (
"context"
"fmt"
"net/http"
"reflect"
@@ -29,9 +30,9 @@ import (
)
func ossErrResponse(code string) error {
return errors.Trace(oss.ServiceError{
return oss.ServiceError{
Code: code,
})
}
}
func TestOSSToObjectError(t *testing.T) {
@@ -116,7 +117,7 @@ func TestS3MetaToOSSOptions(t *testing.T) {
headers = map[string]string{
"x-amz-meta-invalid_meta": "value",
}
_, err = appendS3MetaToOSSOptions(nil, headers)
_, err = appendS3MetaToOSSOptions(context.Background(), nil, headers)
if err = errors.Cause(err); err != nil {
if _, ok := err.(minio.UnsupportedMetadata); !ok {
t.Fatalf("Test failed with unexpected error %s, expected UnsupportedMetadata", err)
@@ -133,7 +134,7 @@ func TestS3MetaToOSSOptions(t *testing.T) {
"X-Amz-Meta-X-Amz-Matdesc": "{}",
"X-Amz-Meta-X-Amz-Iv": "eWmyryl8kq+EVnnsE7jpOg==",
}
opts, err := appendS3MetaToOSSOptions(nil, headers)
opts, err := appendS3MetaToOSSOptions(context.Background(), nil, headers)
if err != nil {
t.Fatalf("Test failed, with %s", err)
}

76
cmd/gateway/s3/gateway-s3.go
View File

@@ -24,8 +24,8 @@ import (
miniogo "github.com/minio/minio-go"
"github.com/minio/minio-go/pkg/policy"
"github.com/minio/minio-go/pkg/s3utils"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/errors"
"github.com/minio/minio/pkg/hash"
minio "github.com/minio/minio/cmd"
@@ -101,7 +101,7 @@ func s3GatewayMain(ctx *cli.Context) {
// Validate gateway arguments.
host := ctx.Args().First()
// Validate gateway arguments.
minio.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
minio.StartGateway(ctx, &S3{host})
}
@@ -173,7 +173,8 @@ func (l *s3Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
func (l *s3Objects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error {
err := l.Client.MakeBucket(bucket, location)
if err != nil {
return minio.ErrorRespToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return minio.ErrorRespToObjectError(err, bucket)
}
return err
}
@@ -188,12 +189,14 @@ func (l *s3Objects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.
// access to these buckets.
// Ref - http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
if s3utils.CheckValidBucketName(bucket) != nil {
return bi, errors.Trace(minio.BucketNameInvalid{Bucket: bucket})
logger.LogIf(ctx, minio.BucketNameInvalid{Bucket: bucket})
return bi, minio.BucketNameInvalid{Bucket: bucket}
}
buckets, err := l.Client.ListBuckets()
if err != nil {
return bi, minio.ErrorRespToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return bi, minio.ErrorRespToObjectError(err, bucket)
}
for _, bi := range buckets {
@@ -207,14 +210,16 @@ func (l *s3Objects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.
}, nil
}
return bi, errors.Trace(minio.BucketNotFound{Bucket: bucket})
logger.LogIf(ctx, minio.BucketNotFound{Bucket: bucket})
return bi, minio.BucketNotFound{Bucket: bucket}
}
// ListBuckets lists all S3 buckets
func (l *s3Objects) ListBuckets(ctx context.Context) ([]minio.BucketInfo, error) {
buckets, err := l.Client.ListBuckets()
if err != nil {
return nil, minio.ErrorRespToObjectError(errors.Trace(err))
logger.LogIf(ctx, err)
return nil, minio.ErrorRespToObjectError(err)
}
b := make([]minio.BucketInfo, len(buckets))
@@ -232,7 +237,8 @@ func (l *s3Objects) ListBuckets(ctx context.Context) ([]minio.BucketInfo, error)
func (l *s3Objects) DeleteBucket(ctx context.Context, bucket string) error {
err := l.Client.RemoveBucket(bucket)
if err != nil {
return minio.ErrorRespToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return minio.ErrorRespToObjectError(err, bucket)
}
return nil
}
@@ -241,7 +247,8 @@ func (l *s3Objects) DeleteBucket(ctx context.Context, bucket string) error {
func (l *s3Objects) ListObjects(ctx context.Context, bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, e error) {
result, err := l.Client.ListObjects(bucket, prefix, marker, delimiter, maxKeys)
if err != nil {
return loi, minio.ErrorRespToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return loi, minio.ErrorRespToObjectError(err, bucket)
}
return minio.FromMinioClientListBucketResult(bucket, result), nil
@@ -251,7 +258,8 @@ func (l *s3Objects) ListObjects(ctx context.Context, bucket string, prefix strin
func (l *s3Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, e error) {
result, err := l.Client.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys)
if err != nil {
return loi, minio.ErrorRespToObjectError(errors.Trace(err), bucket)
logger.LogIf(ctx, err)
return loi, minio.ErrorRespToObjectError(err, bucket)
}
return minio.FromMinioClientListBucketV2Result(bucket, result), nil
@@ -265,23 +273,27 @@ func (l *s3Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat
// length indicates the total length of the object.
func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string) error {
if length < 0 && length != -1 {
return minio.ErrorRespToObjectError(errors.Trace(minio.InvalidRange{}), bucket, key)
logger.LogIf(ctx, minio.InvalidRange{})
return minio.ErrorRespToObjectError(minio.InvalidRange{}, bucket, key)
}
opts := miniogo.GetObjectOptions{}
if startOffset >= 0 && length >= 0 {
if err := opts.SetRange(startOffset, startOffset+length-1); err != nil {
return minio.ErrorRespToObjectError(errors.Trace(err), bucket, key)
logger.LogIf(ctx, err)
return minio.ErrorRespToObjectError(err, bucket, key)
}
}
object, _, err := l.Client.GetObject(bucket, key, opts)
if err != nil {
return minio.ErrorRespToObjectError(errors.Trace(err), bucket, key)
logger.LogIf(ctx, err)
return minio.ErrorRespToObjectError(err, bucket, key)
}
defer object.Close()
if _, err := io.Copy(writer, object); err != nil {
return minio.ErrorRespToObjectError(errors.Trace(err), bucket, key)
logger.LogIf(ctx, err)
return minio.ErrorRespToObjectError(err, bucket, key)
}
return nil
}
@@ -290,7 +302,8 @@ func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, st
func (l *s3Objects) GetObjectInfo(ctx context.Context, bucket string, object string) (objInfo minio.ObjectInfo, err error) {
oi, err := l.Client.StatObject(bucket, object, miniogo.StatObjectOptions{})
if err != nil {
return minio.ObjectInfo{}, minio.ErrorRespToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return minio.ObjectInfo{}, minio.ErrorRespToObjectError(err, bucket, object)
}
return minio.FromMinioClientObjectInfo(bucket, oi), nil
@@ -300,7 +313,8 @@ func (l *s3Objects) GetObjectInfo(ctx context.Context, bucket string, object str
func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
oi, err := l.Client.PutObject(bucket, object, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), minio.ToMinioClientMetadata(metadata))
if err != nil {
return objInfo, minio.ErrorRespToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return objInfo, minio.ErrorRespToObjectError(err, bucket, object)
}
return minio.FromMinioClientObjectInfo(bucket, oi), nil
@@ -315,7 +329,8 @@ func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject
srcInfo.UserDefined["x-amz-metadata-directive"] = "REPLACE"
srcInfo.UserDefined["x-amz-copy-source-if-match"] = srcInfo.ETag
if _, err = l.Client.CopyObject(srcBucket, srcObject, dstBucket, dstObject, srcInfo.UserDefined); err != nil {
return objInfo, minio.ErrorRespToObjectError(errors.Trace(err), srcBucket, srcObject)
logger.LogIf(ctx, err)
return objInfo, minio.ErrorRespToObjectError(err, srcBucket, srcObject)
}
return l.GetObjectInfo(ctx, dstBucket, dstObject)
}
@@ -324,7 +339,8 @@ func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject
func (l *s3Objects) DeleteObject(ctx context.Context, bucket string, object string) error {
err := l.Client.RemoveObject(bucket, object)
if err != nil {
return minio.ErrorRespToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return minio.ErrorRespToObjectError(err, bucket, object)
}
return nil
@@ -346,7 +362,8 @@ func (l *s3Objects) NewMultipartUpload(ctx context.Context, bucket string, objec
opts := miniogo.PutObjectOptions{UserMetadata: metadata}
uploadID, err = l.Client.NewMultipartUpload(bucket, object, opts)
if err != nil {
return uploadID, minio.ErrorRespToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return uploadID, minio.ErrorRespToObjectError(err, bucket, object)
}
return uploadID, nil
}
@@ -355,7 +372,8 @@ func (l *s3Objects) NewMultipartUpload(ctx context.Context, bucket string, objec
func (l *s3Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *hash.Reader) (pi minio.PartInfo, e error) {
info, err := l.Client.PutObjectPart(bucket, object, uploadID, partID, data, data.Size(), data.MD5Base64String(), data.SHA256HexString())
if err != nil {
return pi, minio.ErrorRespToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return pi, minio.ErrorRespToObjectError(err, bucket, object)
}
return minio.FromMinioClientObjectPart(info), nil
@@ -372,7 +390,8 @@ func (l *s3Objects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, de
completePart, err := l.Client.CopyObjectPart(srcBucket, srcObject, destBucket, destObject,
uploadID, partID, startOffset, length, srcInfo.UserDefined)
if err != nil {
return p, minio.ErrorRespToObjectError(errors.Trace(err), srcBucket, srcObject)
logger.LogIf(ctx, err)
return p, minio.ErrorRespToObjectError(err, srcBucket, srcObject)
}
p.PartNumber = completePart.PartNumber
p.ETag = completePart.ETag
@@ -392,14 +411,16 @@ func (l *s3Objects) ListObjectParts(ctx context.Context, bucket string, object s
// AbortMultipartUpload aborts a ongoing multipart upload
func (l *s3Objects) AbortMultipartUpload(ctx context.Context, bucket string, object string, uploadID string) error {
err := l.Client.AbortMultipartUpload(bucket, object, uploadID)
return minio.ErrorRespToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return minio.ErrorRespToObjectError(err, bucket, object)
}
// CompleteMultipartUpload completes ongoing multipart upload and finalizes object
func (l *s3Objects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []minio.CompletePart) (oi minio.ObjectInfo, e error) {
err := l.Client.CompleteMultipartUpload(bucket, object, uploadID, minio.ToMinioClientCompleteParts(uploadedParts))
if err != nil {
return oi, minio.ErrorRespToObjectError(errors.Trace(err), bucket, object)
logger.LogIf(ctx, err)
return oi, minio.ErrorRespToObjectError(err, bucket, object)
}
return l.GetObjectInfo(ctx, bucket, object)
@@ -408,7 +429,8 @@ func (l *s3Objects) CompleteMultipartUpload(ctx context.Context, bucket string,
// SetBucketPolicy sets policy on bucket
func (l *s3Objects) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error {
if err := l.Client.PutBucketPolicy(bucket, policyInfo); err != nil {
return minio.ErrorRespToObjectError(errors.Trace(err), bucket, "")
logger.LogIf(ctx, err)
return minio.ErrorRespToObjectError(err, bucket, "")
}
return nil
@@ -418,7 +440,8 @@ func (l *s3Objects) SetBucketPolicy(ctx context.Context, bucket string, policyIn
func (l *s3Objects) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) {
policyInfo, err := l.Client.GetBucketPolicy(bucket)
if err != nil {
return policy.BucketAccessPolicy{}, minio.ErrorRespToObjectError(errors.Trace(err), bucket, "")
logger.LogIf(ctx, err)
return policy.BucketAccessPolicy{}, minio.ErrorRespToObjectError(err, bucket, "")
}
return policyInfo, nil
}
@@ -426,7 +449,8 @@ func (l *s3Objects) GetBucketPolicy(ctx context.Context, bucket string) (policy.
// DeleteBucketPolicy deletes all policies on bucket
func (l *s3Objects) DeleteBucketPolicy(ctx context.Context, bucket string) error {
if err := l.Client.PutBucketPolicy(bucket, policy.BucketAccessPolicy{}); err != nil {
return minio.ErrorRespToObjectError(errors.Trace(err), bucket, "")
logger.LogIf(ctx, err)
return minio.ErrorRespToObjectError(err, bucket, "")
}
return nil
}

4
cmd/gateway/s3/gateway-s3_test.go
View File

@@ -119,9 +119,9 @@ func TestS3ToObjectError(t *testing.T) {
}
for i, tc := range testCases {
actualErr := minio.ErrorRespToObjectError(errors.Trace(tc.inputErr), tc.bucket, tc.object)
actualErr := minio.ErrorRespToObjectError(tc.inputErr, tc.bucket, tc.object)
if e, ok := actualErr.(*errors.Error); ok && e.Cause.Error() != tc.expectedErr.Error() {
t.Errorf("Test case %d: Expected error %v but received error %v", i+1, tc.expectedErr, e)
t.Errorf("Test case %d: Expected error %v but received error %v", i+1, tc.expectedErr, actualErr)
}
}
}

91
cmd/gateway/sia/gateway-sia.go
View File

@@ -24,7 +24,6 @@ import (
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
@@ -38,8 +37,8 @@ import (
"github.com/minio/cli"
"github.com/minio/minio-go/pkg/set"
minio "github.com/minio/minio/cmd"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/errors"
"github.com/minio/minio/pkg/hash"
)
@@ -112,7 +111,7 @@ func siaGatewayMain(ctx *cli.Context) {
// Validate gateway arguments.
host := ctx.Args().First()
// Validate gateway arguments.
minio.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
minio.StartGateway(ctx, &Sia{host})
}
@@ -164,9 +163,9 @@ func (g *Sia) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
colorBlue := color.New(color.FgBlue).SprintfFunc()
colorBold := color.New(color.Bold).SprintFunc()
log.Println(colorBlue("\nSia Gateway Configuration:"))
log.Println(colorBlue(" Sia Daemon API Address:") + colorBold(fmt.Sprintf(" %s\n", sia.Address)))
log.Println(colorBlue(" Sia Temp Directory:") + colorBold(fmt.Sprintf(" %s\n", sia.TempDir)))
logger.Println(colorBlue("\nSia Gateway Configuration:"))
logger.Println(colorBlue(" Sia Daemon API Address:") + colorBold(fmt.Sprintf(" %s\n", sia.Address)))
logger.Println(colorBlue(" Sia Temp Directory:") + colorBold(fmt.Sprintf(" %s\n", sia.TempDir)))
return sia, nil
}
@@ -217,10 +216,11 @@ func (s MethodNotSupported) Error() string {
// apiGet wraps a GET request with a status code check, such that if the GET does
// not return 2xx, the error will be read and returned. The response body is
// not closed.
func apiGet(addr, call, apiPassword string) (*http.Response, error) {
func apiGet(ctx context.Context, addr, call, apiPassword string) (*http.Response, error) {
req, err := http.NewRequest("GET", "http://"+addr+call, nil)
if err != nil {
return nil, errors.Trace(err)
logger.LogIf(ctx, err)
return nil, err
}
req.Header.Set("User-Agent", "Sia-Agent")
if apiPassword != "" {
@@ -228,15 +228,18 @@ func apiGet(addr, call, apiPassword string) (*http.Response, error) {
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, errors.Trace(err)
logger.LogIf(ctx, err)
return nil, err
}
if resp.StatusCode == http.StatusNotFound {
resp.Body.Close()
logger.LogIf(ctx, MethodNotSupported{call})
return nil, MethodNotSupported{call}
}
if non2xx(resp.StatusCode) {
err := decodeError(resp)
resp.Body.Close()
logger.LogIf(ctx, err)
return nil, err
}
return resp, nil
@@ -245,7 +248,7 @@ func apiGet(addr, call, apiPassword string) (*http.Response, error) {
// apiPost wraps a POST request with a status code check, such that if the POST
// does not return 2xx, the error will be read and returned. The response body
// is not closed.
func apiPost(addr, call, vals, apiPassword string) (*http.Response, error) {
func apiPost(ctx context.Context, addr, call, vals, apiPassword string) (*http.Response, error) {
req, err := http.NewRequest("POST", "http://"+addr+call, strings.NewReader(vals))
if err != nil {
return nil, err
@@ -257,7 +260,8 @@ func apiPost(addr, call, vals, apiPassword string) (*http.Response, error) {
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, errors.Trace(err)
logger.LogIf(ctx, err)
return nil, err
}
if resp.StatusCode == http.StatusNotFound {
@@ -275,8 +279,8 @@ func apiPost(addr, call, vals, apiPassword string) (*http.Response, error) {
// post makes an API call and discards the response. An error is returned if
// the response status is not 2xx.
func post(addr, call, vals, apiPassword string) error {
resp, err := apiPost(addr, call, vals, apiPassword)
func post(ctx context.Context, addr, call, vals, apiPassword string) error {
resp, err := apiPost(ctx, addr, call, vals, apiPassword)
if err != nil {
return err
}
@@ -285,24 +289,26 @@ func post(addr, call, vals, apiPassword string) error {
}
// list makes a lists all the uploaded files, decodes the json response.
func list(addr string, apiPassword string, obj *renterFiles) error {
resp, err := apiGet(addr, "/renter/files", apiPassword)
func list(ctx context.Context, addr string, apiPassword string, obj *renterFiles) error {
resp, err := apiGet(ctx, addr, "/renter/files", apiPassword)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNoContent {
logger.LogIf(ctx, fmt.Errorf("Expecting a response, but API returned %s", resp.Status))
return fmt.Errorf("Expecting a response, but API returned %s", resp.Status)
}
return json.NewDecoder(resp.Body).Decode(obj)
err = json.NewDecoder(resp.Body).Decode(obj)
logger.LogIf(ctx, err)
return err
}
// get makes an API call and discards the response. An error is returned if the
// responsee status is not 2xx.
func get(addr, call, apiPassword string) error {
resp, err := apiGet(addr, call, apiPassword)
func get(ctx context.Context, addr, call, apiPassword string) error {
resp, err := apiGet(ctx, addr, call, apiPassword)
if err != nil {
return err
}
@@ -336,7 +342,7 @@ func (s *siaObjects) MakeBucketWithLocation(ctx context.Context, bucket, locatio
sha256sum := sha256.Sum256([]byte(bucket))
var siaObj = path.Join(s.RootDir, bucket, hex.EncodeToString(sha256sum[:]))
return post(s.Address, "/renter/upload/"+siaObj, "source="+srcFile, s.password)
return post(ctx, s.Address, "/renter/upload/"+siaObj, "source="+srcFile, s.password)
}
// GetBucketInfo gets bucket metadata.
@@ -347,7 +353,7 @@ func (s *siaObjects) GetBucketInfo(ctx context.Context, bucket string) (bi minio
dstFile := path.Join(s.TempDir, minio.MustGetUUID())
defer os.Remove(dstFile)
if err := get(s.Address, "/renter/download/"+siaObj+"?destination="+url.QueryEscape(dstFile), s.password); err != nil {
if err := get(ctx, s.Address, "/renter/download/"+siaObj+"?destination="+url.QueryEscape(dstFile), s.password); err != nil {
return bi, err
}
return minio.BucketInfo{Name: bucket}, nil
@@ -355,7 +361,7 @@ func (s *siaObjects) GetBucketInfo(ctx context.Context, bucket string) (bi minio
// ListBuckets will detect and return existing buckets on Sia.
func (s *siaObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketInfo, err error) {
sObjs, serr := s.listRenterFiles("")
sObjs, serr := s.listRenterFiles(ctx, "")
if serr != nil {
return buckets, serr
}
@@ -388,11 +394,11 @@ func (s *siaObjects) DeleteBucket(ctx context.Context, bucket string) error {
sha256sum := sha256.Sum256([]byte(bucket))
var siaObj = path.Join(s.RootDir, bucket, hex.EncodeToString(sha256sum[:]))
return post(s.Address, "/renter/delete/"+siaObj, "", s.password)
return post(ctx, s.Address, "/renter/delete/"+siaObj, "", s.password)
}
func (s *siaObjects) ListObjects(ctx context.Context, bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) {
siaObjs, siaErr := s.listRenterFiles(bucket)
siaObjs, siaErr := s.listRenterFiles(ctx, bucket)
if siaErr != nil {
return loi, siaErr
}
@@ -429,7 +435,7 @@ func (s *siaObjects) GetObject(ctx context.Context, bucket string, object string
defer os.Remove(dstFile)
var siaObj = path.Join(s.RootDir, bucket, object)
if err := get(s.Address, "/renter/download/"+siaObj+"?destination="+url.QueryEscape(dstFile), s.password); err != nil {
if err := get(ctx, s.Address, "/renter/download/"+siaObj+"?destination="+url.QueryEscape(dstFile), s.password); err != nil {
return err
}
@@ -459,11 +465,16 @@ func (s *siaObjects) GetObject(ctx context.Context, bucket string, object string
// Reply back invalid range if the input offset and length fall out of range.
if startOffset > size || startOffset+length > size {
return errors.Trace(minio.InvalidRange{
logger.LogIf(ctx, minio.InvalidRange{
OffsetBegin: startOffset,
OffsetEnd: length,
ResourceSize: size,
})
return minio.InvalidRange{
OffsetBegin: startOffset,
OffsetEnd: length,
ResourceSize: size,
}
}
// Allocate a staging buffer.
@@ -476,10 +487,10 @@ func (s *siaObjects) GetObject(ctx context.Context, bucket string, object string
// findSiaObject retrieves the siaObjectInfo for the Sia object with the given
// Sia path name.
func (s *siaObjects) findSiaObject(bucket, object string) (siaObjectInfo, error) {
func (s *siaObjects) findSiaObject(ctx context.Context, bucket, object string) (siaObjectInfo, error) {
siaPath := path.Join(s.RootDir, bucket, object)
sObjs, err := s.listRenterFiles("")
sObjs, err := s.listRenterFiles(ctx, "")
if err != nil {
return siaObjectInfo{}, err
}
@@ -489,16 +500,19 @@ func (s *siaObjects) findSiaObject(bucket, object string) (siaObjectInfo, error)
return sObj, nil
}
}
return siaObjectInfo{}, errors.Trace(minio.ObjectNotFound{
logger.LogIf(ctx, minio.ObjectNotFound{
Bucket: bucket,
Object: object,
})
return siaObjectInfo{}, minio.ObjectNotFound{
Bucket: bucket,
Object: object,
}
}
// GetObjectInfo reads object info and replies back ObjectInfo
func (s *siaObjects) GetObjectInfo(ctx context.Context, bucket string, object string) (minio.ObjectInfo, error) {
so, err := s.findSiaObject(bucket, object)
so, err := s.findSiaObject(ctx, bucket, object)
if err != nil {
return minio.ObjectInfo{}, err
}
@@ -527,11 +541,11 @@ func (s *siaObjects) PutObject(ctx context.Context, bucket string, object string
return objInfo, err
}
if err = post(s.Address, "/renter/upload/"+path.Join(s.RootDir, bucket, object), "source="+srcFile, s.password); err != nil {
if err = post(ctx, s.Address, "/renter/upload/"+path.Join(s.RootDir, bucket, object), "source="+srcFile, s.password); err != nil {
os.Remove(srcFile)
return objInfo, err
}
defer s.deleteTempFileWhenUploadCompletes(srcFile, bucket, object)
defer s.deleteTempFileWhenUploadCompletes(ctx, srcFile, bucket, object)
return minio.ObjectInfo{
Name: object,
@@ -546,7 +560,7 @@ func (s *siaObjects) PutObject(ctx context.Context, bucket string, object string
func (s *siaObjects) DeleteObject(ctx context.Context, bucket string, object string) error {
// Tell Sia daemon to delete the object
var siaObj = path.Join(s.RootDir, bucket, object)
return post(s.Address, "/renter/delete/"+siaObj, "", s.password)
return post(ctx, s.Address, "/renter/delete/"+siaObj, "", s.password)
}
// siaObjectInfo represents object info stored on Sia
@@ -565,10 +579,10 @@ type renterFiles struct {
}
// listRenterFiles will return a list of existing objects in the bucket provided
func (s *siaObjects) listRenterFiles(bucket string) (siaObjs []siaObjectInfo, err error) {
func (s *siaObjects) listRenterFiles(ctx context.Context, bucket string) (siaObjs []siaObjectInfo, err error) {
// Get list of all renter files
var rf renterFiles
if err = list(s.Address, s.password, &rf); err != nil {
if err = list(ctx, s.Address, s.password, &rf); err != nil {
return siaObjs, err
}
@@ -592,16 +606,15 @@ func (s *siaObjects) listRenterFiles(bucket string) (siaObjs []siaObjectInfo, er
// deleteTempFileWhenUploadCompletes checks the status of a Sia file upload
// until it reaches 100% upload progress, then deletes the local temp copy from
// the filesystem.
func (s *siaObjects) deleteTempFileWhenUploadCompletes(tempFile string, bucket, object string) {
func (s *siaObjects) deleteTempFileWhenUploadCompletes(ctx context.Context, tempFile string, bucket, object string) {
var soi siaObjectInfo
// Wait until 100% upload instead of 1x redundancy because if we delete
// after 1x redundancy, the user has to pay the cost of other hosts
// redistributing the file.
for soi.UploadProgress < 100.0 {
var err error
soi, err = s.findSiaObject(bucket, object)
soi, err = s.findSiaObject(ctx, bucket, object)
if err != nil {
minio.ErrorIf(err, "Unable to find file uploaded to Sia path %s/%s", bucket, object)
break
}