From b53376a3a4548f92878661410a1cebd1285611b3 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 2 May 2023 16:09:33 -0700 Subject: [PATCH] change directory objects to never create new versions (#17109) --- .github/markdown-lint-cfg.yaml | 5 -- .github/workflows/markdown-lint.yaml | 30 ---------- cmd/bucket-handlers_test.go | 4 +- cmd/data-usage.go | 2 +- cmd/erasure-server-pool.go | 8 +++ cmd/metacache-entries.go | 2 +- cmd/utils.go | 4 ++ .../setup_2site_existing_replication.sh | 57 +++++++++++++++++++ docs/bucket/versioning/README.md | 2 - 9 files changed, 74 insertions(+), 40 deletions(-) delete mode 100644 .github/markdown-lint-cfg.yaml delete mode 100644 .github/workflows/markdown-lint.yaml diff --git a/.github/markdown-lint-cfg.yaml b/.github/markdown-lint-cfg.yaml deleted file mode 100644 index 2a9c042ca..000000000 --- a/.github/markdown-lint-cfg.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# Config file for markdownlint-cli -MD033: - allowed_elements: - - details - - summary diff --git a/.github/workflows/markdown-lint.yaml b/.github/workflows/markdown-lint.yaml deleted file mode 100644 index 38b9fa2cc..000000000 --- a/.github/workflows/markdown-lint.yaml +++ /dev/null @@ -1,30 +0,0 @@ -name: Markdown Linter - -on: - pull_request: - branches: - - master - -# This ensures that previous jobs for the PR are canceled when the PR is -# updated. -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - lint: - name: Lint all docs - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v3 - - - name: Lint all docs - run: | - npm install -g markdownlint-cli - markdownlint --fix '**/*.md' \ - --config /home/runner/work/minio/minio/.github/markdown-lint-cfg.yaml \ - --disable MD013 MD040 MD051 diff --git a/cmd/bucket-handlers_test.go b/cmd/bucket-handlers_test.go index 25afdb129..a81c6cd6a 100644 --- a/cmd/bucket-handlers_test.go +++ b/cmd/bucket-handlers_test.go @@ -657,13 +657,14 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa ) { var err error - contentBytes := []byte("hello") sha256sum := "" var objectNames []string for i := 0; i < 10; i++ { + contentBytes := []byte("hello") objectName := "test-object-" + strconv.Itoa(i) if i == 0 { objectName += "/" + contentBytes = []byte{} } // uploading the object. _, err = obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{}) @@ -676,6 +677,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa objectNames = append(objectNames, objectName) } + contentBytes := []byte("hello") for _, name := range []string{"private/object", "public/object"} { // Uploading the object with retention enabled _, err = obj.PutObject(GlobalContext, bucketName, name, mustGetPutObjReader(t, bytes.NewReader(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{}) diff --git a/cmd/data-usage.go b/cmd/data-usage.go index eb6b5fee8..cecbb25a1 100644 --- a/cmd/data-usage.go +++ b/cmd/data-usage.go @@ -81,7 +81,7 @@ func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket for id, usageInfo := range cache.flattenChildrens(*root) { prefix := decodeDirObject(strings.TrimPrefix(id, bucket+slashSeparator)) - // decodeDirObject to avoid any __XL_DIR__ objects + // decodeDirObject to avoid any __XLDIR__ objects m[prefix] += uint64(usageInfo.Size) } } diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index d8f057882..06c5940c9 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -937,7 +937,15 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec return ObjectInfo{}, err } + origObject := object object = encodeDirObject(object) + // Only directory objects skip creating new versions. + if object != origObject && isDirObject(object) && data.Size() == 0 { + // Treat all directory PUTs to behave as if they are performed + // on an unversioned bucket. + opts.Versioned = false + opts.VersionSuspended = false + } if z.SinglePool() { if !isMinioMetaBucketName(bucket) { diff --git a/cmd/metacache-entries.go b/cmd/metacache-entries.go index f19703066..1277b0370 100644 --- a/cmd/metacache-entries.go +++ b/cmd/metacache-entries.go @@ -55,7 +55,7 @@ func (e metaCacheEntry) isObject() bool { return len(e.metadata) > 0 } -// isObjectDir returns if the entry is representing an object__XL_DIR__ +// isObjectDir returns if the entry is representing an object/ func (e metaCacheEntry) isObjectDir() bool { return len(e.metadata) > 0 && strings.HasSuffix(e.name, slashSeparator) } diff --git a/cmd/utils.go b/cmd/utils.go index 4c1b43b79..e5e9ffdea 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -1031,6 +1031,10 @@ func decodeDirObject(object string) string { return object } +func isDirObject(object string) bool { + return HasSuffix(object, globalDirSuffix) +} + // Helper method to return total number of nodes in cluster func totalNodeCount() uint64 { peers, _ := globalEndpoints.peers() diff --git a/docs/bucket/replication/setup_2site_existing_replication.sh b/docs/bucket/replication/setup_2site_existing_replication.sh index ce1e5cae6..4712d9607 100755 --- a/docs/bucket/replication/setup_2site_existing_replication.sh +++ b/docs/bucket/replication/setup_2site_existing_replication.sh @@ -132,4 +132,61 @@ if [ $ret -ne 0 ]; then exit 1 fi +./mc mb sitea/bucket-version/ +./mc mb siteb/bucket-version + +./mc version enable sitea/bucket-version/ +./mc version enable siteb/bucket-version/ + +echo "adding replication rule for site a -> site b" +./mc replicate add sitea/bucket-version/ \ + --remote-bucket http://minio:minio123@127.0.0.1:9004/bucket-version + +./mc mb sitea/bucket-version/directory/ + +sleep 2s + +./mc ls -r --versions sitea/bucket-version/ > /tmp/sitea_dirs.txt +./mc ls -r --versions siteb/bucket-version/ > /tmp/siteb_dirs.txt + +out=$(diff -qpruN /tmp/sitea_dirs.txt /tmp/siteb_dirs.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no 'diff' after replication: $out" + exit 1 +fi + +./mc rm -r --versions --force sitea/bucket-version/ + +sleep 2s + +./mc ls -r --versions sitea/bucket-version/ > /tmp/sitea_dirs.txt +./mc ls -r --versions siteb/bucket-version/ > /tmp/siteb_dirs.txt + +out=$(diff -qpruN /tmp/sitea_dirs.txt /tmp/siteb_dirs.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no 'diff' after replication: $out" + exit 1 +fi + +## check if we don't create delete markers on the directory objects, its always permanent delete. +./mc mb sitea/bucket-version/directory/ + +sleep 2s + +./mc rm -r --force sitea/bucket-version/ + +sleep 2s + +./mc ls -r --versions sitea/bucket-version/ > /tmp/sitea_dirs.txt +./mc ls -r --versions siteb/bucket-version/ > /tmp/siteb_dirs.txt + +out=$(diff -qpruN /tmp/sitea_dirs.txt /tmp/siteb_dirs.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no 'diff' after replication: $out" + exit 1 +fi + catch diff --git a/docs/bucket/versioning/README.md b/docs/bucket/versioning/README.md index 1e4ee7c7c..1b7db4167 100644 --- a/docs/bucket/versioning/README.md +++ b/docs/bucket/versioning/README.md @@ -78,8 +78,6 @@ To exclude objects under a list of prefix (glob) patterns from being versioned, ``` Enabled - true - */_temporary