B2 gateway S3 compat: return MD5 hash as ETag from PutObject (#9183)

- B2 does actually return an MD5 hash for newly uploaded objects
  so we can use it to provide better compatibility with S3 client
  libraries that assume the ETag is the MD5 hash such as boto.
- depends on change in blazer library.
- new behaviour is only enabled if MinIO's --compat mode is active.
- behaviour for multipart uploads is unchanged (works fine as is).

cmd/gateway/b2/gateway-b2.go

@@ -87,11 +87,15 @@ EXAMPLES:
 
 // Handler for 'minio gateway b2' command line.
 func b2GatewayMain(ctx *cli.Context) {
-	minio.StartGateway(ctx, &B2{})
+	minio.StartGateway(ctx, &B2{
+		strictS3Compat: ctx.IsSet("compat") || ctx.GlobalIsSet("compat"),
+	})
 }
 
 // B2 implements MinIO Gateway
-type B2 struct{}
+type B2 struct {
+	strictS3Compat bool
+}
 
 // Name implements Gateway interface.
 func (g *B2) Name() string {
@@ -113,7 +117,8 @@ func (g *B2) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
 		httpClient: &http.Client{
 			Transport: minio.NewGatewayHTTPTransport(),
 		},
-		ctx: ctx,
+		ctx:            ctx,
+		strictS3Compat: g.strictS3Compat,
 	}, nil
 }
 
@@ -125,12 +130,13 @@ func (g *B2) Production() bool {
 // b2Object implements gateway for MinIO and BackBlaze B2 compatible object storage servers.
 type b2Objects struct {
 	minio.GatewayUnsupported
-	mu         sync.Mutex
+	mu             sync.Mutex
-	creds      auth.Credentials
+	creds          auth.Credentials
-	b2Client   *b2.B2
+	b2Client       *b2.B2
-	httpClient *http.Client
+	httpClient     *http.Client
-	ctx        context.Context
+	ctx            context.Context
-	buckets    []*b2.Bucket
+	buckets        []*b2.Bucket
+	strictS3Compat bool
 }
 
 // Convert B2 errors to minio object layer errors.
@@ -248,6 +254,15 @@ func (l *b2Objects) reAuthorizeAccount(ctx context.Context) error {
 	return nil
 }
 
+// getETag returns an S3-compatible md5sum-ETag if requested and possible.
+func (l *b2Objects) getETag(f *b2.File, fi *b2.FileInfo) string {
+	if l.strictS3Compat && fi.MD5 != "" {
+		return fi.MD5
+	}
+
+	return minio.ToS3ETag(f.ID)
+}
+
 // listBuckets is a wrapper similar to ListBuckets, which re-authorizes
 // the account and updates the B2 client safely. Once successfully
 // authorized performs the call again and returns list of buckets.
@@ -374,7 +389,7 @@ func (l *b2Objects) ListObjects(ctx context.Context, bucket string, prefix strin
 				Name:        file.Name,
 				ModTime:     file.Timestamp,
 				Size:        file.Size,
-				ETag:        minio.ToS3ETag(file.ID),
+				ETag:        l.getETag(file, file.Info),
 				ContentType: file.Info.ContentType,
 				UserDefined: file.Info.Info,
 			})
@@ -427,7 +442,7 @@ func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat
 				Name:        file.Name,
 				ModTime:     file.Timestamp,
 				Size:        file.Size,
-				ETag:        minio.ToS3ETag(file.ID),
+				ETag:        l.getETag(file, file.Info),
 				ContentType: file.Info.ContentType,
 				UserDefined: file.Info.Info,
 			})
@@ -452,7 +467,7 @@ func (l *b2Objects) GetObjectNInfo(ctx context.Context, bucket, object string, r
 
 	pr, pw := io.Pipe()
 	go func() {
-		err := l.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts)
+		err := l.GetObject(ctx, bucket, object, startOffset, length, pw, "", opts)
 		pw.CloseWithError(err)
 	}()
 	// Setup cleanup function to cause the above go-routine to
@@ -472,7 +487,7 @@ func (l *b2Objects) GetObject(ctx context.Context, bucket string, object string,
 	if err != nil {
 		return err
 	}
-	reader, err := bkt.DownloadFileByName(l.ctx, object, startOffset, length)
+	reader, err := bkt.DownloadFileByName(l.ctx, object, startOffset, length, false)
 	if err != nil {
 		l.buckets = []*b2.Bucket{}
 
@@ -481,7 +496,7 @@ func (l *b2Objects) GetObject(ctx context.Context, bucket string, object string,
 			return err
 		}
 
-		reader, err = bkt.DownloadFileByName(l.ctx, object, startOffset, length)
+		reader, err = bkt.DownloadFileByName(l.ctx, object, startOffset, length, false)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			return b2ToObjectError(err, bucket, object)
@@ -543,7 +558,7 @@ func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object str
 	return minio.ObjectInfo{
 		Bucket:      bucket,
 		Name:        object,
-		ETag:        minio.ToS3ETag(f[0].ID),
+		ETag:        l.getETag(f[0], fi),
 		Size:        fi.Size,
 		ModTime:     fi.Timestamp,
 		ContentType: fi.ContentType,
@@ -660,7 +675,7 @@ func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string,
 	return minio.ObjectInfo{
 		Bucket:      bucket,
 		Name:        object,
-		ETag:        minio.ToS3ETag(f.ID),
+		ETag:        l.getETag(f, fi),
 		Size:        fi.Size,
 		ModTime:     fi.Timestamp,
 		ContentType: fi.ContentType,

docs/gateway/b2.md

@@ -43,15 +43,16 @@ mc ls myb2
 ### Known limitations
 Gateway inherits the following B2 limitations:
 
-- PutObject S3 API does not return the md5sum of an uploaded file as it's etag response. Applications that check the validity of the uploaded object by comparing these two values will fail.
+- No support for CopyObject S3 API (yet).
-- No support for CopyObject S3 API (There are no equivalent APIs available on Backblaze B2).
+- No support for CopyObjectPart S3 API (yet).
-- No support for CopyObjectPart S3 API (There are no equivalent APIs available on Backblaze B2).
 - Only read-only bucket policy supported at bucket level, all other variations will return API Notimplemented error.
 - DeleteObject() might not delete the object right away on Backblaze B2, so you might see the object immediately after a Delete request.
+- Multipart uploads don't return an MD5-hash ETag, not even with the `--compat` flag.
 
 Other limitations:
 
 - Bucket notification APIs are not supported.
+- The PutObject S3 API only returns the md5sum of an uploaded file as it's ETag response when minio is started in `--compat` mode. Don't use this unless you are stuck with a client library where this check cannot be turned off (e.g. boto).
 
 ## Explore Further
 - [`mc` command-line interface](https://docs.min.io/docs/minio-client-quickstart-guide)

go.mod

@@ -56,7 +56,7 @@ require (
 	github.com/klauspost/pgzip v1.2.1
 	github.com/klauspost/readahead v1.3.1
 	github.com/klauspost/reedsolomon v1.9.3
-	github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3
+	github.com/kurin/blazer v0.5.4-0.20200327014341-8f90a40f8af7
 	github.com/lib/pq v1.1.1
 	github.com/mattn/go-colorable v0.1.1
 	github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect

go.sum

@@ -246,6 +246,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3 h1:1sl2HmNtqGnDuydLgCJwZIpDLGqZOdwOkcY8WtUl8Cw=
 github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU=
+github.com/kurin/blazer v0.5.4-0.20200327014341-8f90a40f8af7 h1:smZXPopqRVVywwzou4WYWvUbJvSAzIDFizfWElpmAqY=
+github.com/kurin/blazer v0.5.4-0.20200327014341-8f90a40f8af7/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU=
 github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/mailru/easyjson v0.0.0-20180730094502-03f2033d19d5 h1:0x4qcEHDpruK6ML/m/YSlFUUu0UpRD3I2PHsNCuGnyA=