MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 04:42:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

Support for remote tier management (#12090)

Browse Source 
With this change, MinIO's ILM supports transitioning objects to a remote tier.
This change includes support for Azure Blob Storage, AWS S3 compatible object
storage incl. MinIO and Google Cloud Storage as remote tier storage backends.

Some new additions include:

 - Admin APIs remote tier configuration management

 - Simple journal to track remote objects to be 'collected'
   This is used by object API handlers which 'mutate' object versions by
   overwriting/replacing content (Put/CopyObject) or removing the version
   itself (e.g DeleteObjectVersion).

 - Rework of previous ILM transition to fit the new model
   In the new model, a storage class (a.k.a remote tier) is defined by the
   'remote' object storage type (one of s3, azure, GCS), bucket name and a
   prefix.

* Fixed bugs, review comments, and more unit-tests

- Leverage inline small object feature
- Migrate legacy objects to the latest object format before transitioning
- Fix restore to particular version if specified
- Extend SharedDataDirCount to handle transitioned and restored objects
- Restore-object should accept version-id for version-suspended bucket (#12091)
- Check if remote tier creds have sufficient permissions
- Bonus minor fixes to existing error messages

Co-authored-by: Poorna Krishnamoorthy <poorna@minio.io>
Co-authored-by: Krishna Srinivas <krishna@minio.io>
Signed-off-by: Harshavardhana <harsha@minio.io>
This commit is contained in:
Krishnan Parthasarathi
2021-04-19 10:30:42 -07:00
committed by Harshavardhana
parent 069432566f
commit c829e3a13b
302 changed files with 10260 additions and 3800 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
pkg/bucket/lifecycle/action_string.go
View File

@@ -1,20 +1,3 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Code generated by "stringer -type Action lifecycle.go"; DO NOT EDIT.
package lifecycle

94
pkg/bucket/lifecycle/lifecycle.go
View File

@@ -244,17 +244,29 @@ func (lc Lifecycle) FilterActionableRules(obj ObjectOpts) []Rule {
// ObjectOpts provides information to deduce the lifecycle actions
// which can be triggered on the resultant object.
type ObjectOpts struct {
Name string
UserTags string
ModTime time.Time
VersionID string
IsLatest bool
DeleteMarker bool
NumVersions int
SuccessorModTime time.Time
TransitionStatus string
RestoreOngoing bool
RestoreExpires time.Time
Name string
UserTags string
ModTime time.Time
VersionID string
IsLatest bool
DeleteMarker bool
NumVersions int
SuccessorModTime time.Time
TransitionStatus string
RestoreOngoing bool
RestoreExpires time.Time
RemoteTiersImmediately []string // strictly for debug only
}
// doesMatchDebugTiers returns true if tier matches one of the debugTiers, false
// otherwise.
func doesMatchDebugTiers(tier string, debugTiers []string) bool {
for _, t := range debugTiers {
if strings.ToUpper(tier) == strings.ToUpper(t) {
return true
}
}
return false
}
// ExpiredObjectDeleteMarker returns true if an object version referred to by o
@@ -271,7 +283,6 @@ func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action {
if obj.ModTime.IsZero() {
return action
}
for _, rule := range lc.FilterActionableRules(obj) {
if obj.ExpiredObjectDeleteMarker() && rule.Expiration.DeleteMarker.val {
// Indicates whether MinIO will remove a delete marker with no noncurrent versions.
@@ -304,11 +315,17 @@ func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action {
if !rule.NoncurrentVersionTransition.IsDaysNull() {
if obj.VersionID != "" && !obj.IsLatest && !obj.SuccessorModTime.IsZero() && !obj.DeleteMarker && obj.TransitionStatus != TransitionComplete {
// Non current versions should be deleted if their age exceeds non current days configuration
// Non current versions should be transitioned if their age exceeds non current days configuration
// https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#intro-lifecycle-rules-actions
if time.Now().After(ExpectedExpiryTime(obj.SuccessorModTime, int(rule.NoncurrentVersionTransition.NoncurrentDays))) {
return TransitionVersionAction
}
// this if condition is strictly for debug purposes to force immediate
// transition to remote tier if _MINIO_DEBUG_REMOTE_TIERS_IMMEDIATELY is set
if doesMatchDebugTiers(rule.NoncurrentVersionTransition.StorageClass, obj.RemoteTiersImmediately) {
return TransitionVersionAction
}
}
}
@@ -335,6 +352,20 @@ func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action {
if time.Now().UTC().After(ExpectedExpiryTime(obj.ModTime, int(rule.Transition.Days))) {
action = TransitionAction
}
}
// this if condition is strictly for debug purposes to force immediate
// transition to remote tier if _MINIO_DEBUG_REMOTE_TIERS_IMMEDIATELY is set
if !rule.Transition.IsNull() && doesMatchDebugTiers(rule.Transition.StorageClass, obj.RemoteTiersImmediately) {
action = TransitionAction
}
if !obj.RestoreExpires.IsZero() && time.Now().After(obj.RestoreExpires) {
if obj.VersionID != "" {
action = DeleteRestoredVersionAction
} else {
action = DeleteRestoredAction
}
}
}
if !obj.RestoreExpires.IsZero() && time.Now().After(obj.RestoreExpires) {
@@ -361,7 +392,7 @@ func ExpectedExpiryTime(modTime time.Time, days int) time.Time {
}
// PredictExpiryTime returns the expiry date/time of a given object
// after evaluting the current lifecycle document.
// after evaluating the current lifecycle document.
func (lc Lifecycle) PredictExpiryTime(obj ObjectOpts) (string, time.Time) {
if obj.DeleteMarker {
// We don't need to send any x-amz-expiration for delete marker.
@@ -394,3 +425,38 @@ func (lc Lifecycle) PredictExpiryTime(obj ObjectOpts) (string, time.Time) {
}
return finalExpiryRuleID, finalExpiryDate
}
// PredictTransitionTime returns the transition date/time of a given object
// after evaluating the current lifecycle document.
func (lc Lifecycle) PredictTransitionTime(obj ObjectOpts) (string, time.Time) {
if obj.DeleteMarker {
// We don't need to send any x-minio-transition for delete marker.
return "", time.Time{}
}
if obj.TransitionStatus == TransitionComplete {
return "", time.Time{}
}
var finalTransitionDate time.Time
var finalTransitionRuleID string
// Iterate over all actionable rules and find the earliest
// transition date and its associated rule ID.
for _, rule := range lc.FilterActionableRules(obj) {
switch {
case !rule.Transition.IsDateNull():
if finalTransitionDate.IsZero() || finalTransitionDate.After(rule.Transition.Date.Time) {
finalTransitionRuleID = rule.ID
finalTransitionDate = rule.Transition.Date.Time
}
case !rule.Transition.IsDaysNull():
expectedTransition := ExpectedExpiryTime(obj.ModTime, int(rule.Expiration.Days))
if finalTransitionDate.IsZero() || finalTransitionDate.After(expectedTransition) {
finalTransitionRuleID = rule.ID
finalTransitionDate = expectedTransition
}
}
}
return finalTransitionRuleID, finalTransitionDate
}