mirror of
https://github.com/minio/minio.git
synced 2024-12-26 23:25:54 -05:00
c829e3a13b
With this change, MinIO's ILM supports transitioning objects to a remote tier. This change includes support for Azure Blob Storage, AWS S3 compatible object storage incl. MinIO and Google Cloud Storage as remote tier storage backends. Some new additions include: - Admin APIs remote tier configuration management - Simple journal to track remote objects to be 'collected' This is used by object API handlers which 'mutate' object versions by overwriting/replacing content (Put/CopyObject) or removing the version itself (e.g DeleteObjectVersion). - Rework of previous ILM transition to fit the new model In the new model, a storage class (a.k.a remote tier) is defined by the 'remote' object storage type (one of s3, azure, GCS), bucket name and a prefix. * Fixed bugs, review comments, and more unit-tests - Leverage inline small object feature - Migrate legacy objects to the latest object format before transitioning - Fix restore to particular version if specified - Extend SharedDataDirCount to handle transitioned and restored objects - Restore-object should accept version-id for version-suspended bucket (#12091) - Check if remote tier creds have sufficient permissions - Bonus minor fixes to existing error messages Co-authored-by: Poorna Krishnamoorthy <poorna@minio.io> Co-authored-by: Krishna Srinivas <krishna@minio.io> Signed-off-by: Harshavardhana <harsha@minio.io>
463 lines
9.0 KiB
Go
463 lines
9.0 KiB
Go
package cmd
|
|
|
|
// Code generated by github.com/tinylib/msgp DO NOT EDIT.
|
|
|
|
import (
|
|
"bytes"
|
|
"testing"
|
|
|
|
"github.com/tinylib/msgp/msgp"
|
|
)
|
|
|
|
func TestMarshalUnmarshalxlMetaV2(t *testing.T) {
|
|
v := xlMetaV2{}
|
|
bts, err := v.MarshalMsg(nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
left, err := v.UnmarshalMsg(bts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(left) > 0 {
|
|
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
|
}
|
|
|
|
left, err = msgp.Skip(bts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(left) > 0 {
|
|
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
|
}
|
|
}
|
|
|
|
func BenchmarkMarshalMsgxlMetaV2(b *testing.B) {
|
|
v := xlMetaV2{}
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
v.MarshalMsg(nil)
|
|
}
|
|
}
|
|
|
|
func BenchmarkAppendMsgxlMetaV2(b *testing.B) {
|
|
v := xlMetaV2{}
|
|
bts := make([]byte, 0, v.Msgsize())
|
|
bts, _ = v.MarshalMsg(bts[0:0])
|
|
b.SetBytes(int64(len(bts)))
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
bts, _ = v.MarshalMsg(bts[0:0])
|
|
}
|
|
}
|
|
|
|
func BenchmarkUnmarshalxlMetaV2(b *testing.B) {
|
|
v := xlMetaV2{}
|
|
bts, _ := v.MarshalMsg(nil)
|
|
b.ReportAllocs()
|
|
b.SetBytes(int64(len(bts)))
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
_, err := v.UnmarshalMsg(bts)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestEncodeDecodexlMetaV2(t *testing.T) {
|
|
v := xlMetaV2{}
|
|
var buf bytes.Buffer
|
|
msgp.Encode(&buf, &v)
|
|
|
|
m := v.Msgsize()
|
|
if buf.Len() > m {
|
|
t.Log("WARNING: TestEncodeDecodexlMetaV2 Msgsize() is inaccurate")
|
|
}
|
|
|
|
vn := xlMetaV2{}
|
|
err := msgp.Decode(&buf, &vn)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
|
|
buf.Reset()
|
|
msgp.Encode(&buf, &v)
|
|
err = msgp.NewReader(&buf).Skip()
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
}
|
|
|
|
func BenchmarkEncodexlMetaV2(b *testing.B) {
|
|
v := xlMetaV2{}
|
|
var buf bytes.Buffer
|
|
msgp.Encode(&buf, &v)
|
|
b.SetBytes(int64(buf.Len()))
|
|
en := msgp.NewWriter(msgp.Nowhere)
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
v.EncodeMsg(en)
|
|
}
|
|
en.Flush()
|
|
}
|
|
|
|
func BenchmarkDecodexlMetaV2(b *testing.B) {
|
|
v := xlMetaV2{}
|
|
var buf bytes.Buffer
|
|
msgp.Encode(&buf, &v)
|
|
b.SetBytes(int64(buf.Len()))
|
|
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
|
dc := msgp.NewReader(rd)
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
err := v.DecodeMsg(dc)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestMarshalUnmarshalxlMetaV2DeleteMarker(t *testing.T) {
|
|
v := xlMetaV2DeleteMarker{}
|
|
bts, err := v.MarshalMsg(nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
left, err := v.UnmarshalMsg(bts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(left) > 0 {
|
|
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
|
}
|
|
|
|
left, err = msgp.Skip(bts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(left) > 0 {
|
|
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
|
}
|
|
}
|
|
|
|
func BenchmarkMarshalMsgxlMetaV2DeleteMarker(b *testing.B) {
|
|
v := xlMetaV2DeleteMarker{}
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
v.MarshalMsg(nil)
|
|
}
|
|
}
|
|
|
|
func BenchmarkAppendMsgxlMetaV2DeleteMarker(b *testing.B) {
|
|
v := xlMetaV2DeleteMarker{}
|
|
bts := make([]byte, 0, v.Msgsize())
|
|
bts, _ = v.MarshalMsg(bts[0:0])
|
|
b.SetBytes(int64(len(bts)))
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
bts, _ = v.MarshalMsg(bts[0:0])
|
|
}
|
|
}
|
|
|
|
func BenchmarkUnmarshalxlMetaV2DeleteMarker(b *testing.B) {
|
|
v := xlMetaV2DeleteMarker{}
|
|
bts, _ := v.MarshalMsg(nil)
|
|
b.ReportAllocs()
|
|
b.SetBytes(int64(len(bts)))
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
_, err := v.UnmarshalMsg(bts)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestEncodeDecodexlMetaV2DeleteMarker(t *testing.T) {
|
|
v := xlMetaV2DeleteMarker{}
|
|
var buf bytes.Buffer
|
|
msgp.Encode(&buf, &v)
|
|
|
|
m := v.Msgsize()
|
|
if buf.Len() > m {
|
|
t.Log("WARNING: TestEncodeDecodexlMetaV2DeleteMarker Msgsize() is inaccurate")
|
|
}
|
|
|
|
vn := xlMetaV2DeleteMarker{}
|
|
err := msgp.Decode(&buf, &vn)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
|
|
buf.Reset()
|
|
msgp.Encode(&buf, &v)
|
|
err = msgp.NewReader(&buf).Skip()
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
}
|
|
|
|
func BenchmarkEncodexlMetaV2DeleteMarker(b *testing.B) {
|
|
v := xlMetaV2DeleteMarker{}
|
|
var buf bytes.Buffer
|
|
msgp.Encode(&buf, &v)
|
|
b.SetBytes(int64(buf.Len()))
|
|
en := msgp.NewWriter(msgp.Nowhere)
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
v.EncodeMsg(en)
|
|
}
|
|
en.Flush()
|
|
}
|
|
|
|
func BenchmarkDecodexlMetaV2DeleteMarker(b *testing.B) {
|
|
v := xlMetaV2DeleteMarker{}
|
|
var buf bytes.Buffer
|
|
msgp.Encode(&buf, &v)
|
|
b.SetBytes(int64(buf.Len()))
|
|
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
|
dc := msgp.NewReader(rd)
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
err := v.DecodeMsg(dc)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestMarshalUnmarshalxlMetaV2Object(t *testing.T) {
|
|
v := xlMetaV2Object{}
|
|
bts, err := v.MarshalMsg(nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
left, err := v.UnmarshalMsg(bts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(left) > 0 {
|
|
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
|
}
|
|
|
|
left, err = msgp.Skip(bts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(left) > 0 {
|
|
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
|
}
|
|
}
|
|
|
|
func BenchmarkMarshalMsgxlMetaV2Object(b *testing.B) {
|
|
v := xlMetaV2Object{}
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
v.MarshalMsg(nil)
|
|
}
|
|
}
|
|
|
|
func BenchmarkAppendMsgxlMetaV2Object(b *testing.B) {
|
|
v := xlMetaV2Object{}
|
|
bts := make([]byte, 0, v.Msgsize())
|
|
bts, _ = v.MarshalMsg(bts[0:0])
|
|
b.SetBytes(int64(len(bts)))
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
bts, _ = v.MarshalMsg(bts[0:0])
|
|
}
|
|
}
|
|
|
|
func BenchmarkUnmarshalxlMetaV2Object(b *testing.B) {
|
|
v := xlMetaV2Object{}
|
|
bts, _ := v.MarshalMsg(nil)
|
|
b.ReportAllocs()
|
|
b.SetBytes(int64(len(bts)))
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
_, err := v.UnmarshalMsg(bts)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestEncodeDecodexlMetaV2Object(t *testing.T) {
|
|
v := xlMetaV2Object{}
|
|
var buf bytes.Buffer
|
|
msgp.Encode(&buf, &v)
|
|
|
|
m := v.Msgsize()
|
|
if buf.Len() > m {
|
|
t.Log("WARNING: TestEncodeDecodexlMetaV2Object Msgsize() is inaccurate")
|
|
}
|
|
|
|
vn := xlMetaV2Object{}
|
|
err := msgp.Decode(&buf, &vn)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
|
|
buf.Reset()
|
|
msgp.Encode(&buf, &v)
|
|
err = msgp.NewReader(&buf).Skip()
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
}
|
|
|
|
func BenchmarkEncodexlMetaV2Object(b *testing.B) {
|
|
v := xlMetaV2Object{}
|
|
var buf bytes.Buffer
|
|
msgp.Encode(&buf, &v)
|
|
b.SetBytes(int64(buf.Len()))
|
|
en := msgp.NewWriter(msgp.Nowhere)
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
v.EncodeMsg(en)
|
|
}
|
|
en.Flush()
|
|
}
|
|
|
|
func BenchmarkDecodexlMetaV2Object(b *testing.B) {
|
|
v := xlMetaV2Object{}
|
|
var buf bytes.Buffer
|
|
msgp.Encode(&buf, &v)
|
|
b.SetBytes(int64(buf.Len()))
|
|
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
|
dc := msgp.NewReader(rd)
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
err := v.DecodeMsg(dc)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestMarshalUnmarshalxlMetaV2Version(t *testing.T) {
|
|
v := xlMetaV2Version{}
|
|
bts, err := v.MarshalMsg(nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
left, err := v.UnmarshalMsg(bts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(left) > 0 {
|
|
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
|
}
|
|
|
|
left, err = msgp.Skip(bts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(left) > 0 {
|
|
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
|
}
|
|
}
|
|
|
|
func BenchmarkMarshalMsgxlMetaV2Version(b *testing.B) {
|
|
v := xlMetaV2Version{}
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
v.MarshalMsg(nil)
|
|
}
|
|
}
|
|
|
|
func BenchmarkAppendMsgxlMetaV2Version(b *testing.B) {
|
|
v := xlMetaV2Version{}
|
|
bts := make([]byte, 0, v.Msgsize())
|
|
bts, _ = v.MarshalMsg(bts[0:0])
|
|
b.SetBytes(int64(len(bts)))
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
bts, _ = v.MarshalMsg(bts[0:0])
|
|
}
|
|
}
|
|
|
|
func BenchmarkUnmarshalxlMetaV2Version(b *testing.B) {
|
|
v := xlMetaV2Version{}
|
|
bts, _ := v.MarshalMsg(nil)
|
|
b.ReportAllocs()
|
|
b.SetBytes(int64(len(bts)))
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
_, err := v.UnmarshalMsg(bts)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestEncodeDecodexlMetaV2Version(t *testing.T) {
|
|
v := xlMetaV2Version{}
|
|
var buf bytes.Buffer
|
|
msgp.Encode(&buf, &v)
|
|
|
|
m := v.Msgsize()
|
|
if buf.Len() > m {
|
|
t.Log("WARNING: TestEncodeDecodexlMetaV2Version Msgsize() is inaccurate")
|
|
}
|
|
|
|
vn := xlMetaV2Version{}
|
|
err := msgp.Decode(&buf, &vn)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
|
|
buf.Reset()
|
|
msgp.Encode(&buf, &v)
|
|
err = msgp.NewReader(&buf).Skip()
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
}
|
|
|
|
func BenchmarkEncodexlMetaV2Version(b *testing.B) {
|
|
v := xlMetaV2Version{}
|
|
var buf bytes.Buffer
|
|
msgp.Encode(&buf, &v)
|
|
b.SetBytes(int64(buf.Len()))
|
|
en := msgp.NewWriter(msgp.Nowhere)
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
v.EncodeMsg(en)
|
|
}
|
|
en.Flush()
|
|
}
|
|
|
|
func BenchmarkDecodexlMetaV2Version(b *testing.B) {
|
|
v := xlMetaV2Version{}
|
|
var buf bytes.Buffer
|
|
msgp.Encode(&buf, &v)
|
|
b.SetBytes(int64(buf.Len()))
|
|
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
|
dc := msgp.NewReader(rd)
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
err := v.DecodeMsg(dc)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
}
|