Switch to kurin/blazer from minio/blazer fork for b2 gateway (#7879)

cmd/gateway/b2/gateway-b2.go

@@ -29,7 +29,7 @@ import (
 	"sync"
 	"time"
 
-	b2 "github.com/minio/blazer/base"
+	b2 "github.com/kurin/blazer/base"
 	"github.com/minio/cli"
 	miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
 	"github.com/minio/minio/cmd/logger"
@@ -147,6 +147,29 @@ func b2ToObjectError(err error, params ...string) error {
 	if err == nil {
 		return nil
 	}
+
+	code, msgCode, msg := b2.MsgCode(err)
+	if code == 0 {
+		// We don't interpret non B2 errors. B2 errors have statusCode
+		// to help us convert them to S3 object errors.
+		return err
+	}
+
+	objErr := b2MsgCodeToObjectError(code, msgCode, msg, params...)
+	if objErr == nil {
+		return err
+	}
+	return objErr
+}
+
+func b2MsgCodeToObjectError(code int, msgCode string, msg string, params ...string) error {
+	// Following code is a non-exhaustive check to convert
+	// B2 errors into S3 compatible errors.
+	//
+	// For a more complete information - https://www.backblaze.com/b2/docs/
+
+	var err error
+
 	bucket := ""
 	object := ""
 	uploadID := ""
@@ -160,18 +183,7 @@ func b2ToObjectError(err error, params ...string) error {
 		uploadID = params[2]
 	}
 
-	// Following code is a non-exhaustive check to convert
+	switch msgCode {
-	// B2 errors into S3 compatible errors.
-	//
-	// For a more complete information - https://www.backblaze.com/b2/docs/
-	statusCode, code, msg := b2.Code(err)
-	if statusCode == 0 {
-		// We don't interpret non B2 errors. B2 errors have statusCode
-		// to help us convert them to S3 object errors.
-		return err
-	}
-
-	switch code {
 	case "duplicate_bucket_name":
 		err = minio.BucketAlreadyOwnedByYou{Bucket: bucket}
 	case "bad_request":
@@ -343,7 +355,7 @@ func (l *b2Objects) ListObjects(ctx context.Context, bucket string, prefix strin
 				Name:        file.Name,
 				ModTime:     file.Timestamp,
 				Size:        file.Size,
-				ETag:        minio.ToS3ETag(file.Info.ID),
+				ETag:        minio.ToS3ETag(file.ID),
 				ContentType: file.Info.ContentType,
 				UserDefined: file.Info.Info,
 			})
@@ -386,7 +398,7 @@ func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat
 				Name:        file.Name,
 				ModTime:     file.Timestamp,
 				Size:        file.Size,
-				ETag:        minio.ToS3ETag(file.Info.ID),
+				ETag:        minio.ToS3ETag(file.ID),
 				ContentType: file.Info.ContentType,
 				UserDefined: file.Info.Info,
 			})
@@ -462,7 +474,7 @@ func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object str
 	return minio.ObjectInfo{
 		Bucket:      bucket,
 		Name:        object,
-		ETag:        minio.ToS3ETag(fi.ID),
+		ETag:        minio.ToS3ETag(f.ID),
 		Size:        fi.Size,
 		ModTime:     fi.Timestamp,
 		ContentType: fi.ContentType,
@@ -569,7 +581,7 @@ func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string,
 	return minio.ObjectInfo{
 		Bucket:      bucket,
 		Name:        object,
-		ETag:        minio.ToS3ETag(fi.ID),
+		ETag:        minio.ToS3ETag(f.ID),
 		Size:        fi.Size,
 		ModTime:     fi.Timestamp,
 		ContentType: fi.ContentType,
@@ -617,7 +629,7 @@ func (l *b2Objects) ListMultipartUploads(ctx context.Context, bucket string, pre
 	if maxUploads > 100 {
 		maxUploads = 100
 	}
-	largeFiles, nextMarker, err := bkt.ListUnfinishedLargeFiles(l.ctx, uploadIDMarker, maxUploads)
+	largeFiles, nextMarker, err := bkt.ListUnfinishedLargeFiles(l.ctx, maxUploads, uploadIDMarker)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return lmi, b2ToObjectError(err, bucket)
@@ -677,7 +689,7 @@ func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object str
 	}
 
 	hr := newB2Reader(data, data.Size())
-	sha1, err := fc.UploadPart(l.ctx, hr, sha1AtEOF, int(hr.Size()), partID)
+	_, err = fc.UploadPart(l.ctx, hr, sha1AtEOF, int(hr.Size()), partID)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return pi, b2ToObjectError(err, bucket, object, uploadID)
@@ -686,7 +698,7 @@ func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object str
 	return minio.PartInfo{
 		PartNumber:   partID,
 		LastModified: minio.UTCNow(),
-		ETag:         minio.ToS3ETag(sha1),
+		ETag:         minio.ToS3ETag(fmt.Sprintf("%x", hr.sha1Hash.Sum(nil))),
 		Size:         data.Size(),
 	}, nil
 }

cmd/gateway/b2/gateway-b2_test.go

@@ -20,8 +20,6 @@ import (
 	"fmt"
 	"testing"
 
-	b2 "github.com/minio/blazer/base"
-
 	minio "github.com/minio/minio/cmd"
 )
 
@@ -41,71 +39,6 @@ func TestB2ObjectError(t *testing.T) {
 		{
 			[]string{}, fmt.Errorf("Non B2 Error"), fmt.Errorf("Non B2 Error"),
 		},
-		{
-			[]string{"bucket"}, b2.Error{
-				StatusCode: 1,
-				Code:       "duplicate_bucket_name",
-			}, minio.BucketAlreadyOwnedByYou{
-				Bucket: "bucket",
-			},
-		},
-		{
-			[]string{"bucket"}, b2.Error{
-				StatusCode: 1,
-				Code:       "bad_request",
-			}, minio.BucketNotFound{
-				Bucket: "bucket",
-			},
-		},
-		{
-			[]string{"bucket", "object"}, b2.Error{
-				StatusCode: 1,
-				Code:       "bad_request",
-			}, minio.ObjectNameInvalid{
-				Bucket: "bucket",
-				Object: "object",
-			},
-		},
-		{
-			[]string{"bucket"}, b2.Error{
-				StatusCode: 1,
-				Code:       "bad_bucket_id",
-			}, minio.BucketNotFound{Bucket: "bucket"},
-		},
-		{
-			[]string{"bucket", "object"}, b2.Error{
-				StatusCode: 1,
-				Code:       "file_not_present",
-			}, minio.ObjectNotFound{
-				Bucket: "bucket",
-				Object: "object",
-			},
-		},
-		{
-			[]string{"bucket", "object"}, b2.Error{
-				StatusCode: 1,
-				Code:       "not_found",
-			}, minio.ObjectNotFound{
-				Bucket: "bucket",
-				Object: "object",
-			},
-		},
-		{
-			[]string{"bucket"}, b2.Error{
-				StatusCode: 1,
-				Code:       "cannot_delete_non_empty_bucket",
-			}, minio.BucketNotEmpty{
-				Bucket: "bucket",
-			},
-		},
-		{
-			[]string{"bucket", "object", "uploadID"}, b2.Error{
-				StatusCode: 1,
-				Message:    "No active upload for",
-			}, minio.InvalidUploadID{
-				UploadID: "uploadID",
-			},
-		},
 	}
 
 	for i, testCase := range testCases {
@@ -117,3 +50,97 @@ func TestB2ObjectError(t *testing.T) {
 		}
 	}
 }
+
+// Test b2 msg code to object error.
+func TestB2MsgCodeToObjectError(t *testing.T) {
+	testCases := []struct {
+		params      []string
+		code        int
+		msgCode     string
+		msg         string
+		expectedErr error
+	}{
+		{
+			[]string{"bucket"},
+			1,
+			"duplicate_bucket_name",
+			"",
+			minio.BucketAlreadyOwnedByYou{
+				Bucket: "bucket",
+			},
+		},
+		{
+			[]string{"bucket"},
+			1,
+			"bad_request",
+			"",
+			minio.BucketNotFound{
+				Bucket: "bucket",
+			},
+		},
+		{
+			[]string{"bucket", "object"},
+			1,
+			"bad_request",
+			"",
+			minio.ObjectNameInvalid{
+				Bucket: "bucket",
+				Object: "object",
+			},
+		},
+		{
+			[]string{"bucket"},
+			1,
+			"bad_bucket_id",
+			"",
+			minio.BucketNotFound{Bucket: "bucket"},
+		},
+		{
+			[]string{"bucket", "object"},
+			1,
+			"file_not_present",
+			"",
+			minio.ObjectNotFound{
+				Bucket: "bucket",
+				Object: "object",
+			},
+		},
+		{
+			[]string{"bucket", "object"},
+			1,
+			"not_found",
+			"",
+			minio.ObjectNotFound{
+				Bucket: "bucket",
+				Object: "object",
+			},
+		},
+		{
+			[]string{"bucket"},
+			1,
+			"cannot_delete_non_empty_bucket",
+			"",
+			minio.BucketNotEmpty{
+				Bucket: "bucket",
+			},
+		},
+		{
+			[]string{"bucket", "object", "uploadID"},
+			1,
+			"",
+			"No active upload for",
+			minio.InvalidUploadID{
+				UploadID: "uploadID",
+			},
+		},
+	}
+
+	for i, testCase := range testCases {
+		actualErr := b2MsgCodeToObjectError(testCase.code, testCase.msgCode, testCase.msg, testCase.params...)
+		if actualErr != nil {
+			if actualErr.Error() != testCase.expectedErr.Error() {
+				t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.expectedErr, actualErr)
+			}
+		}
+	}
+}

go.mod

@@ -47,11 +47,11 @@ require (
 	github.com/klauspost/pgzip v1.2.1
 	github.com/klauspost/readahead v1.3.0
 	github.com/klauspost/reedsolomon v1.9.1
+	github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3
 	github.com/lib/pq v1.0.0
 	github.com/marstr/guid v1.1.0 // indirect
 	github.com/mattn/go-isatty v0.0.7
 	github.com/miekg/dns v1.1.8
-	github.com/minio/blazer v0.0.0-20171126203752-2081f5bf0465
 	github.com/minio/cli v1.20.0
 	github.com/minio/dsync/v2 v2.0.0
 	github.com/minio/hdfs/v3 v3.0.0

go.sum

@@ -376,6 +376,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kurin/blazer v0.5.3 h1:SAgYv0TKU0kN/ETfO5ExjNAPyMt2FocO2s/UlCHfjAk=
+github.com/kurin/blazer v0.5.3/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU=
+github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3 h1:1sl2HmNtqGnDuydLgCJwZIpDLGqZOdwOkcY8WtUl8Cw=
+github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU=
 github.com/lib/pq v0.0.0-20181016162627-9eb73efc1fcc/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=