MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 21:02:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

Support for remote tier management (#12090)

Browse Source 
With this change, MinIO's ILM supports transitioning objects to a remote tier.
This change includes support for Azure Blob Storage, AWS S3 compatible object
storage incl. MinIO and Google Cloud Storage as remote tier storage backends.

Some new additions include:

 - Admin APIs remote tier configuration management

 - Simple journal to track remote objects to be 'collected'
   This is used by object API handlers which 'mutate' object versions by
   overwriting/replacing content (Put/CopyObject) or removing the version
   itself (e.g DeleteObjectVersion).

 - Rework of previous ILM transition to fit the new model
   In the new model, a storage class (a.k.a remote tier) is defined by the
   'remote' object storage type (one of s3, azure, GCS), bucket name and a
   prefix.

* Fixed bugs, review comments, and more unit-tests

- Leverage inline small object feature
- Migrate legacy objects to the latest object format before transitioning
- Fix restore to particular version if specified
- Extend SharedDataDirCount to handle transitioned and restored objects
- Restore-object should accept version-id for version-suspended bucket (#12091)
- Check if remote tier creds have sufficient permissions
- Bonus minor fixes to existing error messages

Co-authored-by: Poorna Krishnamoorthy <poorna@minio.io>
Co-authored-by: Krishna Srinivas <krishna@minio.io>
Signed-off-by: Harshavardhana <harsha@minio.io>
This commit is contained in:
Krishnan Parthasarathi
2021-04-19 10:30:42 -07:00
committed by Harshavardhana
parent 069432566f
commit c829e3a13b
302 changed files with 10260 additions and 3800 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
cmd/xl-storage-format-v2.go
View File

@@ -30,6 +30,7 @@ import (
"github.com/google/uuid"
xhttp "github.com/minio/minio/cmd/http"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/bucket/lifecycle"
"github.com/tinylib/msgp/msgp"
)
@@ -841,6 +842,16 @@ func (z *xlMetaV2) AddVersion(fi FileInfo) error {
if len(fi.Data) > 0 || fi.Size == 0 {
z.data.replace(fi.VersionID, fi.Data)
}
if fi.TransitionStatus != "" {
ventry.ObjectV2.MetaSys[ReservedMetadataPrefixLower+TransitionStatus] = []byte(fi.TransitionStatus)
}
if fi.TransitionedObjName != "" {
ventry.ObjectV2.MetaSys[ReservedMetadataPrefixLower+TransitionedObjectName] = []byte(fi.TransitionedObjName)
}
if fi.TransitionTier != "" {
ventry.ObjectV2.MetaSys[ReservedMetadataPrefixLower+TransitionTier] = []byte(fi.TransitionTier)
}
}
if !ventry.Valid() {
@@ -910,6 +921,18 @@ func (j xlMetaV2DeleteMarker) ToFileInfo(volume, path string) (FileInfo, error)
return fi, nil
}
// UsesDataDir returns true if this object version uses its data directory for
// its contents and false otherwise.
func (j *xlMetaV2Object) UsesDataDir() bool {
// Skip if this version is not transitioned, i.e it uses its data directory.
if !bytes.Equal(j.MetaSys[ReservedMetadataPrefixLower+TransitionStatus], []byte(lifecycle.TransitionComplete)) {
return true
}
// Check if this transitioned object has been restored on disk.
return isRestoredObjectOnDisk(j.MetaUser)
}
func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) {
versionID := ""
var uv uuid.UUID
@@ -953,8 +976,6 @@ func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) {
}
for k, v := range j.MetaSys {
switch {
case equals(k, ReservedMetadataPrefixLower+"transition-status"):
fi.TransitionStatus = string(v)
case equals(k, VersionPurgeStatusKey):
fi.VersionPurgeStatus = VersionPurgeStatusType(string(v))
case strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower):
@@ -972,6 +993,15 @@ func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) {
}
fi.DataDir = uuid.UUID(j.DataDir).String()
if st, ok := j.MetaSys[ReservedMetadataPrefixLower+TransitionStatus]; ok {
fi.TransitionStatus = string(st)
}
if o, ok := j.MetaSys[ReservedMetadataPrefixLower+TransitionedObjectName]; ok {
fi.TransitionedObjName = string(o)
}
if sc, ok := j.MetaSys[ReservedMetadataPrefixLower+TransitionTier]; ok {
fi.TransitionTier = string(sc)
}
return fi, nil
}
@@ -1009,7 +1039,10 @@ func (z *xlMetaV2) SharedDataDirCount(versionID [16]byte, dataDir [16]byte) int
if version.ObjectV2.VersionID == versionID {
continue
}
if version.ObjectV2.DataDir == dataDir {
if version.ObjectV2.DataDir != dataDir {
continue
}
if version.ObjectV2.UsesDataDir() {
sameDataDirCount++
}
}
@@ -1083,11 +1116,6 @@ func (z *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
switch version.Type {
case LegacyType:
if version.ObjectV1.VersionID == fi.VersionID {
if fi.TransitionStatus != "" {
z.Versions[i].ObjectV1.Meta[ReservedMetadataPrefixLower+"transition-status"] = fi.TransitionStatus
return uuid.UUID(version.ObjectV2.DataDir).String(), len(z.Versions) == 0, nil
}
z.Versions = append(z.Versions[:i], z.Versions[i+1:]...)
if fi.Deleted {
z.Versions = append(z.Versions, ventry)
@@ -1131,22 +1159,27 @@ func (z *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
switch version.Type {
case ObjectType:
if version.ObjectV2.VersionID == uv {
if fi.TransitionStatus != "" {
z.Versions[i].ObjectV2.MetaSys[ReservedMetadataPrefixLower+"transition-status"] = []byte(fi.TransitionStatus)
return uuid.UUID(version.ObjectV2.DataDir).String(), len(z.Versions) == 0, nil
switch {
case fi.ExpireRestored:
delete(z.Versions[i].ObjectV2.MetaUser, xhttp.AmzRestore)
delete(z.Versions[i].ObjectV2.MetaUser, xhttp.AmzRestoreExpiryDays)
delete(z.Versions[i].ObjectV2.MetaUser, xhttp.AmzRestoreRequestDate)
case fi.TransitionStatus == lifecycle.TransitionComplete:
z.Versions[i].ObjectV2.MetaSys[ReservedMetadataPrefixLower+TransitionStatus] = []byte(fi.TransitionStatus)
z.Versions[i].ObjectV2.MetaSys[ReservedMetadataPrefixLower+TransitionedObjectName] = []byte(fi.TransitionedObjName)
z.Versions[i].ObjectV2.MetaSys[ReservedMetadataPrefixLower+TransitionTier] = []byte(fi.TransitionTier)
default:
z.Versions = append(z.Versions[:i], z.Versions[i+1:]...)
}
if fi.Deleted {
z.Versions = append(z.Versions, ventry)
}
z.Versions = append(z.Versions[:i], z.Versions[i+1:]...)
if z.SharedDataDirCount(version.ObjectV2.VersionID, version.ObjectV2.DataDir) > 0 {
if fi.Deleted {
z.Versions = append(z.Versions, ventry)
}
// Found that another version references the same dataDir
// we shouldn't remove it, and only remove the version instead
return "", len(z.Versions) == 0, nil
}
if fi.Deleted {
z.Versions = append(z.Versions, ventry)
}
return uuid.UUID(version.ObjectV2.DataDir).String(), len(z.Versions) == 0, nil
}
}