mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
gcs: Translate S3 user-defined metadata prefix to/from GCS custom metadata prefix (#6270)
This commit is contained in:
parent
64f2c61813
commit
ff29aed05d
@ -23,8 +23,8 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
@ -768,22 +768,27 @@ func fromGCSAttrsToObjectInfo(attrs *storage.ObjectAttrs) minio.ObjectInfo {
|
||||
// Refer https://cloud.google.com/storage/docs/hashes-etags. Use CRC32C for ETag
|
||||
metadata := make(map[string]string)
|
||||
for k, v := range attrs.Metadata {
|
||||
k = http.CanonicalHeaderKey(k)
|
||||
// Translate the GCS custom metadata prefix
|
||||
if strings.HasPrefix(k, "X-Goog-Meta-") {
|
||||
k = strings.Replace(k, "X-Goog-Meta-", "X-Amz-Meta-", 1)
|
||||
}
|
||||
metadata[k] = v
|
||||
}
|
||||
if attrs.ContentType != "" {
|
||||
metadata["content-type"] = attrs.ContentType
|
||||
metadata["Content-Type"] = attrs.ContentType
|
||||
}
|
||||
if attrs.ContentEncoding != "" {
|
||||
metadata["content-encoding"] = attrs.ContentEncoding
|
||||
metadata["Content-Encoding"] = attrs.ContentEncoding
|
||||
}
|
||||
if attrs.CacheControl != "" {
|
||||
metadata["cache-control"] = attrs.CacheControl
|
||||
metadata["Cache-Control"] = attrs.CacheControl
|
||||
}
|
||||
if attrs.ContentDisposition != "" {
|
||||
metadata["content-disposition"] = attrs.ContentDisposition
|
||||
metadata["Content-Disposition"] = attrs.ContentDisposition
|
||||
}
|
||||
if attrs.ContentLanguage != "" {
|
||||
metadata["content-language"] = attrs.ContentLanguage
|
||||
metadata["Content-Language"] = attrs.ContentLanguage
|
||||
}
|
||||
return minio.ObjectInfo{
|
||||
Name: attrs.Name,
|
||||
@ -799,21 +804,25 @@ func fromGCSAttrsToObjectInfo(attrs *storage.ObjectAttrs) minio.ObjectInfo {
|
||||
|
||||
// applyMetadataToGCSAttrs applies metadata to a GCS ObjectAttrs instance
|
||||
func applyMetadataToGCSAttrs(metadata map[string]string, attrs *storage.ObjectAttrs) {
|
||||
attrs.ContentType = metadata["content-type"]
|
||||
attrs.ContentEncoding = metadata["content-encoding"]
|
||||
attrs.CacheControl = metadata["cache-control"]
|
||||
attrs.ContentDisposition = metadata["content-disposition"]
|
||||
attrs.ContentLanguage = metadata["content-language"]
|
||||
|
||||
attrs.Metadata = make(map[string]string)
|
||||
for k, v := range metadata {
|
||||
k = http.CanonicalHeaderKey(k)
|
||||
switch {
|
||||
case strings.HasPrefix(k, "X-Amz-Meta-"):
|
||||
// Translate the S3 user-defined metadata prefix
|
||||
k = strings.Replace(k, "X-Amz-Meta-", "x-goog-meta-", 1)
|
||||
attrs.Metadata[k] = v
|
||||
case k == "Content-Type":
|
||||
attrs.ContentType = v
|
||||
case k == "Content-Encoding":
|
||||
attrs.ContentEncoding = v
|
||||
case k == "Cache-Control":
|
||||
attrs.CacheControl = v
|
||||
case k == "Content-Disposition":
|
||||
attrs.ContentDisposition = v
|
||||
case k == "Content-Language":
|
||||
attrs.ContentLanguage = v
|
||||
}
|
||||
// Filter metadata which is stored as a unique attribute
|
||||
for _, key := range []string{
|
||||
"content-type", "content-encoding", "cache-control", "content-disposition", "content-language",
|
||||
} {
|
||||
delete(attrs.Metadata, key)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -23,7 +23,9 @@ import (
|
||||
"path"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"cloud.google.com/go/storage"
|
||||
"google.golang.org/api/googleapi"
|
||||
|
||||
miniogo "github.com/minio/minio-go"
|
||||
@ -393,3 +395,105 @@ func TestGCSToObjectError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3MetaToGCSAttributes(t *testing.T) {
|
||||
headers := map[string]string{
|
||||
"accept-encoding": "gzip",
|
||||
"content-encoding": "gzip",
|
||||
"cache-control": "age: 3600",
|
||||
"content-disposition": "dummy",
|
||||
"content-type": "application/javascript",
|
||||
"Content-Language": "en",
|
||||
"X-Amz-Meta-Hdr": "value",
|
||||
"X-Amz-Meta-X-Amz-Key": "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
|
||||
"X-Amz-Meta-X-Amz-Matdesc": "{}",
|
||||
"X-Amz-Meta-X-Amz-Iv": "eWmyryl8kq+EVnnsE7jpOg==",
|
||||
}
|
||||
// Only X-Amz-Meta- prefixed entries will be returned in
|
||||
// Metadata (without the prefix!)
|
||||
expectedHeaders := map[string]string{
|
||||
"x-goog-meta-Hdr": "value",
|
||||
"x-goog-meta-X-Amz-Key": "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
|
||||
"x-goog-meta-X-Amz-Matdesc": "{}",
|
||||
"x-goog-meta-X-Amz-Iv": "eWmyryl8kq+EVnnsE7jpOg==",
|
||||
}
|
||||
|
||||
attrs := storage.ObjectAttrs{}
|
||||
applyMetadataToGCSAttrs(headers, &attrs)
|
||||
|
||||
if !reflect.DeepEqual(attrs.Metadata, expectedHeaders) {
|
||||
t.Fatalf("Test failed, expected %#v, got %#v", expectedHeaders, attrs.Metadata)
|
||||
}
|
||||
|
||||
if attrs.CacheControl != headers["cache-control"] {
|
||||
t.Fatalf("Test failed with Cache-Control mistmatch, expected %s, got %s", headers["cache-control"], attrs.CacheControl)
|
||||
}
|
||||
if attrs.ContentDisposition != headers["content-disposition"] {
|
||||
t.Fatalf("Test failed with Content-Disposition mistmatch, expected %s, got %s", headers["content-disposition"], attrs.ContentDisposition)
|
||||
}
|
||||
if attrs.ContentEncoding != headers["content-encoding"] {
|
||||
t.Fatalf("Test failed with Content-Encoding mistmatch, expected %s, got %s", headers["content-encoding"], attrs.ContentEncoding)
|
||||
}
|
||||
if attrs.ContentLanguage != headers["Content-Language"] {
|
||||
t.Fatalf("Test failed with Content-Language mistmatch, expected %s, got %s", headers["Content-Language"], attrs.ContentLanguage)
|
||||
}
|
||||
if attrs.ContentType != headers["content-type"] {
|
||||
t.Fatalf("Test failed with Content-Type mistmatch, expected %s, got %s", headers["content-type"], attrs.ContentType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGCSAttrsToObjectInfo(t *testing.T) {
|
||||
metadata := map[string]string{
|
||||
"x-goog-meta-Hdr": "value",
|
||||
"x-goog-meta-x_amz_key": "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
|
||||
"x-goog-meta-x-amz-matdesc": "{}",
|
||||
"x-goog-meta-X-Amz-Iv": "eWmyryl8kq+EVnnsE7jpOg==",
|
||||
}
|
||||
expectedMeta := map[string]string{
|
||||
"X-Amz-Meta-Hdr": "value",
|
||||
"X-Amz-Meta-X_amz_key": "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
|
||||
"X-Amz-Meta-X-Amz-Matdesc": "{}",
|
||||
"X-Amz-Meta-X-Amz-Iv": "eWmyryl8kq+EVnnsE7jpOg==",
|
||||
"Cache-Control": "max-age: 3600",
|
||||
"Content-Disposition": "dummy",
|
||||
"Content-Encoding": "gzip",
|
||||
"Content-Language": "en",
|
||||
"Content-Type": "application/javascript",
|
||||
}
|
||||
|
||||
attrs := storage.ObjectAttrs{
|
||||
Name: "test-obj",
|
||||
Bucket: "test-bucket",
|
||||
Updated: time.Now(),
|
||||
Size: 123,
|
||||
CRC32C: 45312398,
|
||||
CacheControl: "max-age: 3600",
|
||||
ContentDisposition: "dummy",
|
||||
ContentEncoding: "gzip",
|
||||
ContentLanguage: "en",
|
||||
ContentType: "application/javascript",
|
||||
Metadata: metadata,
|
||||
}
|
||||
expectedETag := minio.ToS3ETag(fmt.Sprintf("%d", attrs.CRC32C))
|
||||
|
||||
objInfo := fromGCSAttrsToObjectInfo(&attrs)
|
||||
if !reflect.DeepEqual(objInfo.UserDefined, expectedMeta) {
|
||||
t.Fatalf("Test failed, expected %#v, got %#v", expectedMeta, objInfo.UserDefined)
|
||||
}
|
||||
|
||||
if objInfo.Name != attrs.Name {
|
||||
t.Fatalf("Test failed with Name mistmatch, expected %s, got %s", attrs.Name, objInfo.Name)
|
||||
}
|
||||
if objInfo.Bucket != attrs.Bucket {
|
||||
t.Fatalf("Test failed with Bucket mistmatch, expected %s, got %s", attrs.Bucket, objInfo.Bucket)
|
||||
}
|
||||
if objInfo.ModTime != attrs.Updated {
|
||||
t.Fatalf("Test failed with ModTime mistmatch, expected %s, got %s", attrs.Updated, objInfo.ModTime)
|
||||
}
|
||||
if objInfo.Size != attrs.Size {
|
||||
t.Fatalf("Test failed with Size mistmatch, expected %d, got %d", attrs.Size, objInfo.Size)
|
||||
}
|
||||
if objInfo.ETag != expectedETag {
|
||||
t.Fatalf("Test failed with ETag mistmatch, expected %s, got %s", expectedETag, objInfo.ETag)
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user