MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-20 01:50:24 -05:00
Code Issues Packages Projects Releases Wiki Activity

[Tiering] Support remote tiers with object versioning (#12342)

Browse Source 
- Adds versioning support for S3 based remote tiers that have versioning
enabled. This ensures that when reading or deleting we specify the specific
version ID of the object. In case of deletion, this is important to ensure that
the object version is actually deleted instead of simply being marked for
deletion.

- Stores the remote object's version id in the tier-journal. Tier-journal file
version is not bumped up as serializing the new struct version is
compatible with old journals without the remote object version id.

- `storageRESTVersion` is bumped up as FileInfo struct now includes a
`TransitionRemoteVersionID` member.

- Azure and GCS support for this feature will be added subsequently.

Co-authored-by: Krishnan Parthasarathi <krisis@users.noreply.github.com>
This commit is contained in:
Aditya Manthramurthy
2021-06-03 14:26:51 -07:00
committed by GitHub
parent 41d4d650e4
commit 30a3921d3e
18 changed files with 536 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
cmd/tier-journal.go
View File

@@ -42,8 +42,9 @@ type tierJournal struct {
}
type jentry struct {
ObjName string `msg:"obj"`
TierName string `msg:"tier"`
ObjName string `msg:"obj"`
VersionID string `msg:"vid"`
TierName string `msg:"tier"`
}
const (
@@ -51,6 +52,10 @@ const (
tierJournalHdrLen = 2 // 2 bytes
)
var (
errUnsupportedJournalVersion = errors.New("unsupported pending deletes journal version")
)
func initTierDeletionJournal(done <-chan struct{}) (*tierJournal, error) {
diskPath := globalEndpoints.FirstLocalDiskPath()
j := &tierJournal{
@@ -84,7 +89,7 @@ func (j *tierJournal) rotate() error {
return j.Open()
}
type walkFn func(objName, tierName string) error
type walkFn func(objName, rvID, tierName string) error
func (j *tierJournal) ReadOnlyPath() string {
return filepath.Join(j.diskPath, minioMetaBucket, "ilm", "deletion-journal.ro.bin")
@@ -111,6 +116,7 @@ func (j *tierJournal) WalkEntries(fn walkFn) {
}
defer ro.Close()
mr := msgp.NewReader(ro)
done := false
for {
var entry jentry
@@ -123,9 +129,11 @@ func (j *tierJournal) WalkEntries(fn walkFn) {
logger.LogIf(context.Background(), fmt.Errorf("tier-journal: failed to decode journal entry %s", err))
break
}
err = fn(entry.ObjName, entry.TierName)
err = fn(entry.ObjName, entry.VersionID, entry.TierName)
if err != nil && !isErrObjectNotFound(err) {
logger.LogIf(context.Background(), fmt.Errorf("tier-journal: failed to delete transitioned object %s from %s due to %s", entry.ObjName, entry.TierName, err))
// We add the entry into the active journal to try again
// later.
j.AddEntry(entry)
}
}
@@ -134,12 +142,12 @@ func (j *tierJournal) WalkEntries(fn walkFn) {
}
}
func deleteObjectFromRemoteTier(objName, tierName string) error {
func deleteObjectFromRemoteTier(objName, rvID, tierName string) error {
w, err := globalTierConfigMgr.getDriver(tierName)
if err != nil {
return err
}
err = w.Remove(context.Background(), objName)
err = w.Remove(context.Background(), objName, remoteVersionID(rvID))
if err != nil {
return err
}
@@ -263,8 +271,15 @@ func (j *tierJournal) OpenRO() (io.ReadCloser, error) {
switch binary.LittleEndian.Uint16(data[:]) {
case tierJournalVersion:
return file, nil
default:
return nil, errors.New("unsupported pending deletes journal version")
return nil, errUnsupportedJournalVersion
}
return file, nil
}
// jentryV1 represents the entry in the journal before RemoteVersionID was
// added. It remains here for use in tests for the struct element addition.
type jentryV1 struct {
ObjName string `msg:"obj"`
TierName string `msg:"tier"`
}