MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 12:52:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

[Tiering] Support remote tiers with object versioning (#12342)

Browse Source 
- Adds versioning support for S3 based remote tiers that have versioning
enabled. This ensures that when reading or deleting we specify the specific
version ID of the object. In case of deletion, this is important to ensure that
the object version is actually deleted instead of simply being marked for
deletion.

- Stores the remote object's version id in the tier-journal. Tier-journal file
version is not bumped up as serializing the new struct version is
compatible with old journals without the remote object version id.

- `storageRESTVersion` is bumped up as FileInfo struct now includes a
`TransitionRemoteVersionID` member.

- Azure and GCS support for this feature will be added subsequently.

Co-authored-by: Krishnan Parthasarathi <krisis@users.noreply.github.com>
This commit is contained in:
Aditya Manthramurthy
2021-06-03 14:26:51 -07:00
committed by GitHub
parent 41d4d650e4
commit 30a3921d3e
18 changed files with 536 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
cmd/warm-backend-gcs.go
View File

@@ -43,7 +43,11 @@ func (gcs *warmBackendGCS) getDest(object string) string {
}
return destObj
}
func (gcs *warmBackendGCS) Put(ctx context.Context, key string, data io.Reader, length int64) error {
// FIXME: add support for remote version ID in GCS remote tier and remove this.
// Currently it's a no-op.
func (gcs *warmBackendGCS) Put(ctx context.Context, key string, data io.Reader, length int64) (remoteVersionID, error) {
object := gcs.client.Bucket(gcs.Bucket).Object(gcs.getDest(key))
//TODO: set storage class
w := object.NewWriter(ctx)
@@ -51,13 +55,13 @@ func (gcs *warmBackendGCS) Put(ctx context.Context, key string, data io.Reader,
w.ObjectAttrs.StorageClass = gcs.StorageClass
}
if _, err := io.Copy(w, data); err != nil {
return gcsToObjectError(err, gcs.Bucket, key)
return "", gcsToObjectError(err, gcs.Bucket, key)
}
return w.Close()
return "", w.Close()
}
func (gcs *warmBackendGCS) Get(ctx context.Context, key string, opts WarmBackendGetOpts) (r io.ReadCloser, err error) {
func (gcs *warmBackendGCS) Get(ctx context.Context, key string, rv remoteVersionID, opts WarmBackendGetOpts) (r io.ReadCloser, err error) {
// GCS storage decompresses a gzipped object by default and returns the data.
// Refer to https://cloud.google.com/storage/docs/transcoding#decompressive_transcoding
// Need to set `Accept-Encoding` header to `gzip` when issuing a GetObject call, to be able
@@ -73,7 +77,7 @@ func (gcs *warmBackendGCS) Get(ctx context.Context, key string, opts WarmBackend
return r, nil
}
func (gcs *warmBackendGCS) Remove(ctx context.Context, key string) error {
func (gcs *warmBackendGCS) Remove(ctx context.Context, key string, rv remoteVersionID) error {
err := gcs.client.Bucket(gcs.Bucket).Object(gcs.getDest(key)).Delete(ctx)
return gcsToObjectError(err, gcs.Bucket, key)
}