diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index f35b9f040..139e5556a 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -540,6 +540,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, oss[index].SetTransitionState(goi.TransitionedObject) } + // All deletes on directory objects needs to be for `nullVersionID` + if isDirObject(object.ObjectName) && object.VersionID == "" { + object.VersionID = nullVersionID + } + if replicateDeletes { dsc = checkReplicateDelete(ctx, bucket, ObjectToDelete{ ObjectV: ObjectV{ diff --git a/cmd/bucket-handlers_test.go b/cmd/bucket-handlers_test.go index a81c6cd6a..772d96d7c 100644 --- a/cmd/bucket-handlers_test.go +++ b/cmd/bucket-handlers_test.go @@ -747,8 +747,13 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa deletedObjects := make([]DeletedObject, len(requestList[0].Objects)) for i := range requestList[0].Objects { + var vid string + if isDirObject(requestList[0].Objects[i].ObjectName) { + vid = nullVersionID + } deletedObjects[i] = DeletedObject{ ObjectName: requestList[0].Objects[i].ObjectName, + VersionID: vid, } } @@ -758,9 +763,14 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa successRequest1 := encodeResponse(requestList[1]) deletedObjects = make([]DeletedObject, len(requestList[1].Objects)) - for i := range requestList[0].Objects { + for i := range requestList[1].Objects { + var vid string + if isDirObject(requestList[0].Objects[i].ObjectName) { + vid = nullVersionID + } deletedObjects[i] = DeletedObject{ ObjectName: requestList[1].Objects[i].ObjectName, + VersionID: vid, } } diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index 06c5940c9..d8f057882 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -937,15 +937,7 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec return ObjectInfo{}, err } - origObject := object object = encodeDirObject(object) - // Only directory objects skip creating new versions. - if object != origObject && isDirObject(object) && data.Size() == 0 { - // Treat all directory PUTs to behave as if they are performed - // on an unversioned bucket. - opts.Versioned = false - opts.VersionSuspended = false - } if z.SinglePool() { if !isMinioMetaBucketName(bucket) { diff --git a/cmd/object-api-options.go b/cmd/object-api-options.go index ec29ea1c9..a89a753b2 100644 --- a/cmd/object-api-options.go +++ b/cmd/object-api-options.go @@ -174,6 +174,11 @@ func delOpts(ctx context.Context, r *http.Request, bucket, object string) (opts // benefits of replication, make sure to apply version suspension // only at bucket level instead. opts.VersionSuspended = globalBucketVersioningSys.Suspended(bucket) + // For directory objects, delete `null` version permanently. + if isDirObject(object) && opts.VersionID == "" { + opts.VersionID = nullVersionID + } + delMarker := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceDeleteMarker)) if delMarker != "" { switch delMarker { @@ -321,9 +326,16 @@ func putOpts(ctx context.Context, r *http.Request, bucket, object string, metada if err != nil { return opts, err } + opts.VersionID = vid opts.Versioned = versioned opts.VersionSuspended = versionSuspended + + // For directory objects skip creating new versions. + if isDirObject(object) && vid == "" { + opts.VersionID = nullVersionID + } + opts.MTime = mtime opts.ReplicationSourceLegalholdTimestamp = lholdtimestmp opts.ReplicationSourceRetentionTimestamp = retaintimestmp diff --git a/cmd/utils.go b/cmd/utils.go index 6a5e61867..440eb2857 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -1038,6 +1038,9 @@ func decodeDirObject(object string) string { } func isDirObject(object string) bool { + if obj := encodeDirObject(object); obj != object { + object = obj + } return HasSuffix(object, globalDirSuffix) } diff --git a/docs/bucket/replication/setup_2site_existing_replication.sh b/docs/bucket/replication/setup_2site_existing_replication.sh index 4712d9607..211e158dd 100755 --- a/docs/bucket/replication/setup_2site_existing_replication.sh +++ b/docs/bucket/replication/setup_2site_existing_replication.sh @@ -189,4 +189,19 @@ if [ $ret -ne 0 ]; then exit 1 fi +sitea_count=$(cat /tmp/sitea_dirs.txt | wc -l) # need to do it this way to avoid filename in the output +siteb_count=$(cat /tmp/siteb_dirs.txt | wc -l) # need to do it this way to avoid filename in the output +sitea_out=$(cat /tmp/sitea_dirs.txt) +siteb_out=$(cat /tmp/siteb_dirs.txt) + +if [ $sitea_count -ne 0 ]; then + echo "BUG: expected no 'directory objects' left after deletion: ${sitea_out}" + exit 1 +fi + +if [ $siteb_count -ne 0 ]; then + echo "BUG: expected no 'directory objects' left after deletion: ${siteb_out}" + exit 1 +fi + catch diff --git a/docs/bucket/versioning/README.md b/docs/bucket/versioning/README.md index 1b7db4167..cdb226c87 100644 --- a/docs/bucket/versioning/README.md +++ b/docs/bucket/versioning/README.md @@ -65,6 +65,18 @@ Similarly to suspend versioning set the configuration with Status set to `Suspen ## MinIO extension to Bucket Versioning +### Idempotent versions on directory objects + +All directory objects such as objects that end with `/`, will only have one versionId (i.e `null`). A delete marker will never be created on these directory objects, instead a DELETE will delete the directory objects. This is done to ensure that directory objects even with multiple overwrites - do not ever need multiple versions in the first place. All overwrite calls on these directory objects are idempotent. + +> NOTE: Server side replication is supported for idempotent versions on directory objects. + +### Idempotent versions on delete markers + +Duplicate delete markers are not created on MinIO buckets with versioning, if an application performs a soft delete on an object repeatedly - that object will only ever have a single DELETE marker for all such successive attempts. This is done to ensure that repeated soft deletes do not ever need multiple versions in the first place. + +> NOTE: Server side replication is supported for idempotent versions on delete marked objects. + ### Motivation **PLEASE READ: This feature is meant for advanced usecases only where the setup is using bucket versioning or with replicated buckets, use this feature to optimize versioning behavior for some specific applications. MinIO experts will evaluate and guide on the benefits for your application, please reach out to us on .**