Support bucket versioning (#9377)

- Implement a new xl.json 2.0.0 format to support,
  this moves the entire marshaling logic to POSIX
  layer, top layer always consumes a common FileInfo
  construct which simplifies the metadata reads.
- Implement list object versions
- Migrate to siphash from crchash for new deployments
  for object placements.

Fixes #2111

pkg/bucket/lifecycle/lifecycle.go

@@ -132,8 +132,8 @@ func (lc Lifecycle) Validate() error {
 
 // FilterActionableRules returns the rules actions that need to be executed
 // after evaluating prefix/tag filtering
-func (lc Lifecycle) FilterActionableRules(objName, objTags string) []Rule {
-	if objName == "" {
+func (lc Lifecycle) FilterActionableRules(obj ObjectOpts) []Rule {
+	if obj.Name == "" {
 		return nil
 	}
 	var rules []Rule
@@ -141,30 +141,86 @@ func (lc Lifecycle) FilterActionableRules(objName, objTags string) []Rule {
 		if rule.Status == Disabled {
 			continue
 		}
-		if !strings.HasPrefix(objName, rule.Prefix()) {
+		if !strings.HasPrefix(obj.Name, rule.Prefix()) {
 			continue
 		}
-		tags := strings.Split(objTags, "&")
-		if rule.Filter.TestTags(tags) {
+		// Indicates whether MinIO will remove a delete marker with no
+		// noncurrent versions. If set to true, the delete marker will
+		// be expired; if set to false the policy takes no action. This
+		// cannot be specified with Days or Date in a Lifecycle
+		// Expiration Policy.
+		if rule.Expiration.DeleteMarker {
+			rules = append(rules, rule)
+			continue
+		}
+		// The NoncurrentVersionExpiration action requests MinIO to expire
+		// noncurrent versions of objects 100 days after the objects become
+		// noncurrent.
+		if !rule.NoncurrentVersionExpiration.IsDaysNull() {
+			rules = append(rules, rule)
+			continue
+		}
+		if rule.Filter.TestTags(strings.Split(obj.UserTags, "&")) {
 			rules = append(rules, rule)
 		}
 	}
 	return rules
 }
 
+// ObjectOpts provides information to deduce the lifecycle actions
+// which can be triggered on the resultant object.
+type ObjectOpts struct {
+	Name         string
+	UserTags     string
+	ModTime      time.Time
+	VersionID    string
+	IsLatest     bool
+	DeleteMarker bool
+}
+
 // ComputeAction returns the action to perform by evaluating all lifecycle rules
 // against the object name and its modification time.
-func (lc Lifecycle) ComputeAction(objName, objTags string, modTime time.Time) (action Action) {
-	action = NoneAction
-	if modTime.IsZero() {
-		return
+func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action {
+	var action = NoneAction
+	if obj.ModTime.IsZero() {
+		return action
 	}
 
-	_, expiryTime := lc.PredictExpiryTime(objName, modTime, objTags)
-	if !expiryTime.IsZero() && time.Now().After(expiryTime) {
-		return DeleteAction
+	for _, rule := range lc.FilterActionableRules(obj) {
+		if obj.DeleteMarker && obj.IsLatest && bool(rule.Expiration.DeleteMarker) {
+			// Indicates whether MinIO will remove a delete marker with no noncurrent versions.
+			// Only latest marker is removed. If set to true, the delete marker will be expired;
+			// if set to false the policy takes no action. This cannot be specified with Days or
+			// Date in a Lifecycle Expiration Policy.
+			return DeleteAction
+		}
+
+		if !rule.NoncurrentVersionExpiration.IsDaysNull() {
+			if obj.VersionID != "" && !obj.IsLatest {
+				// Non current versions should be deleted.
+				if time.Now().After(expectedExpiryTime(obj.ModTime, rule.NoncurrentVersionExpiration.NoncurrentDays)) {
+					return DeleteAction
+				}
+				return NoneAction
+			}
+			return NoneAction
+		}
+
+		// All other expiration only applies to latest versions.
+		if obj.IsLatest {
+			switch {
+			case !rule.Expiration.IsDateNull():
+				if time.Now().UTC().After(rule.Expiration.Date.Time) {
+					action = DeleteAction
+				}
+			case !rule.Expiration.IsDaysNull():
+				if time.Now().UTC().After(expectedExpiryTime(obj.ModTime, rule.Expiration.Days)) {
+					action = DeleteAction
+				}
+			}
+		}
 	}
-	return
+	return action
 }
 
 // expectedExpiryTime calculates the expiry date/time based on a object modtime.
@@ -179,13 +235,22 @@ func expectedExpiryTime(modTime time.Time, days ExpirationDays) time.Time {
 
 // PredictExpiryTime returns the expiry date/time of a given object
 // after evaluting the current lifecycle document.
-func (lc Lifecycle) PredictExpiryTime(objName string, modTime time.Time, objTags string) (string, time.Time) {
+func (lc Lifecycle) PredictExpiryTime(obj ObjectOpts) (string, time.Time) {
+	if obj.DeleteMarker {
+		// We don't need to send any x-amz-expiration for delete marker.
+		return "", time.Time{}
+	}
+
 	var finalExpiryDate time.Time
 	var finalExpiryRuleID string
 
 	// Iterate over all actionable rules and find the earliest
 	// expiration date and its associated rule ID.
-	for _, rule := range lc.FilterActionableRules(objName, objTags) {
+	for _, rule := range lc.FilterActionableRules(obj) {
+		if !rule.NoncurrentVersionExpiration.IsDaysNull() && !obj.IsLatest && obj.VersionID != "" {
+			return rule.ID, expectedExpiryTime(time.Now(), ExpirationDays(rule.NoncurrentVersionExpiration.NoncurrentDays))
+		}
+
 		if !rule.Expiration.IsDateNull() {
 			if finalExpiryDate.IsZero() || finalExpiryDate.After(rule.Expiration.Date.Time) {
 				finalExpiryRuleID = rule.ID
@@ -193,7 +258,7 @@ func (lc Lifecycle) PredictExpiryTime(objName string, modTime time.Time, objTags
 			}
 		}
 		if !rule.Expiration.IsDaysNull() {
-			expectedExpiry := expectedExpiryTime(modTime, rule.Expiration.Days)
+			expectedExpiry := expectedExpiryTime(obj.ModTime, rule.Expiration.Days)
 			if finalExpiryDate.IsZero() || finalExpiryDate.After(expectedExpiry) {
 				finalExpiryRuleID = rule.ID
 				finalExpiryDate = expectedExpiry