Add decommission compression+encryption enabled tests (#15322)

update compression environment variables to follow
the expected sub-system style, however support fallback
mode.

.github/workflows/markdown-lint.yaml

@@ -27,4 +27,4 @@ jobs:
         npm install -g markdownlint-cli
         markdownlint --fix '**/*.md' \
           --config /home/runner/work/minio/minio/.github/markdown-lint-cfg.yaml \
-          --disable MD013 MD040
+          --disable MD013 MD040 MD051

Makefile

@@ -46,6 +46,7 @@ test-decom: install
 	@env bash $(PWD)/docs/distributed/decom.sh
 	@env bash $(PWD)/docs/distributed/decom-encrypted.sh
 	@env bash $(PWD)/docs/distributed/decom-encrypted-sse-s3.sh
+	@env bash $(PWD)/docs/distributed/decom-compressed-sse-s3.sh
 
 test-upgrade: build
 	@echo "Running minio upgrade tests"

README.fips.md

@@ -2,6 +2,6 @@
 
 MinIO creates FIPS builds using a patched version of the Go compiler (that uses BoringCrypto, from BoringSSL, which is [FIPS 140-2 validated](https://csrc.nist.gov/csrc/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp2964.pdf)) published by the Golang Team [here](https://github.com/golang/go/tree/dev.boringcrypto/misc/boring).
 
-MinIO FIPS executables are available at http://dl.min.io - they are only published for `linux-amd64` architecture as binary files with the suffix `.fips`. We also publish corresponding container images to our official image repositories.
+MinIO FIPS executables are available at <http://dl.min.io> - they are only published for `linux-amd64` architecture as binary files with the suffix `.fips`. We also publish corresponding container images to our official image repositories.
 
 We are not making any statements or representations about the suitability of this code or build in relation to the FIPS 140-2 standard. Interested users will have to evaluate for themselves whether this is useful for their own purposes.

cmd/object-api-utils.go

@@ -440,10 +440,7 @@ func isCompressible(header http.Header, object string) bool {
 	cfg := globalCompressConfig
 	globalCompressConfigMu.Unlock()
 
-	if !cfg.Enabled || (crypto.Requested(header) && !cfg.AllowEncrypted) || excludeForCompression(header, object, cfg) {
+	return !excludeForCompression(header, object, cfg)
-		return false
-	}
-	return true
 }
 
 // Eliminate the non-compressible objects.
@@ -454,6 +451,10 @@ func excludeForCompression(header http.Header, object string, cfg compress.Confi
 		return true
 	}
 
+	if crypto.Requested(header) && !cfg.AllowEncrypted {
+		return true
+	}
+
 	// We strictly disable compression for standard extensions/content-types (`compressed`).
 	if hasStringSuffixInSlice(objStr, standardExcludeCompressExtensions) || hasPattern(standardExcludeCompressContentTypes, contentType) {
 		return true

docs/bucket/replication/README.md

@@ -244,6 +244,7 @@ Replication from a source bucket to multiple destination buckets is supported. F
 Note that on the source side, the `X-Amz-Replication-Status` changes from `PENDING` to `COMPLETED` after replication succeeds to each of the targets. On the destination side, a `X-Amz-Replication-Status` status of `REPLICA` indicates that the object was replicated successfully. Any replication failures are automatically re-attempted during a periodic disk scanner cycle.
 
 ### Interaction with extended Bucket Versioning configuration
+
 When Bucket Versioning with excluded prefixes are configured objects matching these prefixes are excluded from being versioned and replicated.
 
 ```
@@ -261,6 +262,7 @@ When Bucket Versioning with excluded prefixes are configured objects matching th
         <!-- .. up to 10 prefixes in all -->
 </VersioningConfiguration>
 ```
+
 In the above sample config, objects under prefixes matching any of the `ExcludedPrefixes` glob patterns will neither be versioned nor replicated.
 
 ## Explore Further

docs/bucket/versioning/DESIGN.md

@@ -131,7 +131,7 @@ Inline data is optional. If no inline data is present, it is encoded as 0 bytes.
 | Entry               | Encoding                    | Content
 | --------------------|-----------------------------|----------------------------------------
 | xlMetaInlineDataVer | byte                        | version identifier
-| id -> data          | msgp map[string][]byte      | Map of string id -> byte content
+| id -> data          | msgp `map[string][]byte`      | Map of string id -> byte content
 
 Currently only xlMetaInlineDataVer == 1 exists.
 

docs/bucket/versioning/README.md

@@ -64,12 +64,15 @@ Similarly to suspend versioning set the configuration with Status set to `Suspen
 ```
 
 ## MinIO extension to Bucket Versioning
+
 ### Motivation
-**PLEASE READ: This feature is meant for advanced usecases only where the setup is using bucket versioning or with replicated buckets, use this feature to optimize versioning behavior for some specific applications. MinIO experts will evaluate and guide on the benefits for your application, please reach out to us on https://subnet.min.io.**
+
+**PLEASE READ: This feature is meant for advanced usecases only where the setup is using bucket versioning or with replicated buckets, use this feature to optimize versioning behavior for some specific applications. MinIO experts will evaluate and guide on the benefits for your application, please reach out to us on <https://subnet.min.io>.**
 
 Spark/Hadoop workloads which use Hadoop MR Committer v1/v2 algorithm upload objects to a temporary prefix in a bucket. These objects are 'renamed' to a different prefix on Job commit. Object storage admins are forced to configure separate ILM policies to expire these objects and their versions to reclaim space.
 
 ### Solution
+
 To exclude objects under a list of prefix (glob) patterns from being versioned, you can send the following versioning configuration with Status set to `Enabled`.
 
 ```
@@ -92,6 +95,7 @@ To exclude objects under a list of prefix (glob) patterns from being versioned,
 ```
 
 ### Features
+
 - Objects matching these prefixes will behave as though versioning were suspended. These objects **will not** be replicated if bucket has replication configured.
 - Objects matching these prefixes will also not leave `null` delete markers, dramatically reduces namespace pollution while keeping the benefits of replication.
 - Users with explicit permissions or the root credential can configure the versioning state of any bucket.

docs/compression/README.md

@@ -54,9 +54,9 @@ The compression settings may also be set through environment variables.
 When set, environment variables override the defined `compress` config settings in the server config.
 
 ```bash
-export MINIO_COMPRESS="on"
+export MINIO_COMPRESSION_ENABLE="on"
-export MINIO_COMPRESS_EXTENSIONS=".txt,.log,.csv,.json,.tar,.xml,.bin"
+export MINIO_COMPRESSION_EXTENSIONS=".txt,.log,.csv,.json,.tar,.xml,.bin"
-export MINIO_COMPRESS_MIME_TYPES="text/*,application/json,application/xml"
+export MINIO_COMPRESSION_MIME_TYPES="text/*,application/json,application/xml"
 ```
 
 ### 3. Compression + Encryption
@@ -76,7 +76,7 @@ To enable compression+encryption use:
 ~ mc admin config set myminio compression allow_encryption=on
 ```
 
-Or alternatively through the environment variable `MINIO_COMPRESS_ALLOW_ENCRYPTION=on`.
+Or alternatively through the environment variable `MINIO_COMPRESSION_ALLOW_ENCRYPTION=on`.
 
 ### 4. Excluded Types
 

docs/distributed/README.md

@@ -28,7 +28,6 @@ MinIO follows strict **read-after-write** and **list-after-write** consistency m
 
 **If MinIO distributed setup is using NFS volumes underneath it is not guaranteed MinIO will provide these consistency guarantees since NFS is not strictly consistent (If you must use NFS we recommend that you atleast use NFSv4 instead of NFSv3 for relatively better outcomes).**
 
-
 ## Get started
 
 If you're aware of stand-alone MinIO set up, the process remains largely the same. MinIO server automatically switches to stand-alone or distributed mode, depending on the command line parameters.

docs/distributed/decom-compressed-sse-s3.sh

@@ -0,0 +1,143 @@
+#!/bin/bash
+
+if [ -n "$TEST_DEBUG" ]; then
+    set -x
+fi
+
+pkill minio
+rm -rf /tmp/xl
+
+if [ ! -f ./mc ]; then
+    wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && \
+	chmod +x mc
+fi
+
+export CI=true
+export MINIO_COMPRESSION_ENABLE="on"
+export MINIO_COMPRESSION_EXTENSIONS=".go"
+export MINIO_COMPRESSION_MIME_TYPES="application/*"
+export MINIO_COMPRESSION_ALLOW_ENCRYPTION="on"
+export MINIO_KMS_AUTO_ENCRYPTION=on
+export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw=
+export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9000/"
+
+(minio server /tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null)&
+pid=$!
+
+sleep 2
+
+./mc admin user add myminio/ minio123 minio123
+./mc admin user add myminio/ minio12345 minio12345
+
+./mc admin policy add myminio/ rw ./docs/distributed/rw.json
+./mc admin policy add myminio/ lake ./docs/distributed/rw.json
+
+./mc admin policy set myminio/ rw user=minio123
+./mc admin policy set myminio/ lake,rw user=minio12345
+
+./mc mb -l myminio/versioned
+
+./mc mirror internal myminio/versioned/ --quiet >/dev/null
+
+## Soft delete (creates delete markers)
+./mc rm -r --force myminio/versioned >/dev/null
+
+## mirror again to create another set of version on top
+./mc mirror internal myminio/versioned/ --quiet >/dev/null
+
+expected_checksum=$(./mc cat internal/dsync/drwmutex.go | md5sum)
+
+user_count=$(./mc admin user list myminio/ | wc -l)
+policy_count=$(./mc admin policy list myminio/ | wc -l)
+
+kill $pid
+
+(minio server /tmp/xl/{1...10}/disk{0...1} /tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded.log) &
+pid=$!
+
+sleep 2
+
+expanded_user_count=$(./mc admin user list myminio/ | wc -l)
+expanded_policy_count=$(./mc admin policy list myminio/ | wc -l)
+
+if [ $user_count -ne $expanded_user_count ]; then
+    echo "BUG: original user count differs from expanded setup"
+    exit 1
+fi
+
+if [ $policy_count -ne $expanded_policy_count ]; then
+    echo "BUG: original policy count  differs from expanded setup"
+    exit 1
+fi
+
+./mc version info myminio/versioned | grep -q "versioning is enabled"
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "expected versioning enabled after expansion"
+    exit 1
+fi
+
+./mc mirror cmd myminio/versioned/ --quiet >/dev/null
+
+./mc ls -r myminio/versioned/ > expanded_ns.txt
+./mc ls -r --versions myminio/versioned/ > expanded_ns_versions.txt
+
+./mc admin decom start myminio/ /tmp/xl/{1...10}/disk{0...1}
+
+until $(./mc admin decom status myminio/ | grep -q Complete)
+do
+    echo "waiting for decom to finish..."
+    sleep 1
+done
+
+kill $pid
+
+(minio server /tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/removed.log)&
+pid=$!
+
+sleep 2
+
+decom_user_count=$(./mc admin user list myminio/ | wc -l)
+decom_policy_count=$(./mc admin policy list myminio/ | wc -l)
+
+if [ $user_count -ne $decom_user_count ]; then
+    echo "BUG: original user count differs after decommission"
+    exit 1
+fi
+
+if [ $policy_count -ne $decom_policy_count ]; then
+    echo "BUG: original policy count differs after decommission"
+    exit 1
+fi
+
+./mc version info myminio/versioned | grep -q "versioning is enabled"
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected versioning enabled after decommission"
+    exit 1
+fi
+
+got_checksum=$(./mc cat myminio/versioned/dsync/drwmutex.go | md5sum)
+if [ "${expected_checksum}" != "${got_checksum}" ]; then
+    echo "BUG: decommission failed on encrypted objects: expected ${expected_checksum} got ${got_checksum}"
+    exit 1
+fi
+
+./mc ls -r myminio/versioned > decommissioned_ns.txt
+./mc ls -r --versions myminio/versioned > decommissioned_ns_versions.txt
+
+out=$(diff -qpruN expanded_ns.txt decommissioned_ns.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected no missing entries after decommission: $out"
+    exit 1
+fi
+
+out=$(diff -qpruN expanded_ns_versions.txt decommissioned_ns_versions.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected no missing entries after decommission: $out"
+    exit 1
+fi
+
+# kill $pid

docs/gateway/nas.md

@@ -1,4 +1,4 @@
-# MinIO NAS Gateway [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) 
+# MinIO NAS Gateway [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io)
 
 > NAS gateway is deprecated and will be removed in future, no more fresh deployments are supported.
 
@@ -96,7 +96,7 @@ NAS gateway implementation allows symlinks on regular files.
 
 ### Behavior
 
-- For reads symlinks resolve to the file the symlink points to. 
+- For reads symlinks resolve to the file the symlink points to.
 - For deletes
   - Deleting a symlink deletes the symlink but not the real file to which the symlink points.
   - Deleting the real file a symlink points to automatically makes the dangling symlink invisible.

docs/iam/access-management-plugin.md

@@ -156,4 +156,3 @@ The following structure is also accepted:
 ```
 
 Any unmentioned JSON object keys in the above are ignored.
-

docs/iam/opa.md

@@ -1,4 +1,5 @@
 # OPA Quickstart Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io)
+
 OPA is a lightweight general-purpose policy engine that can be co-located with MinIO server, in this document we talk about how to use OPA HTTP API to authorize requests. It can be used with any type of credentials (STS based like OpenID or LDAP, regular IAM users or service accounts).
 
 OPA is enabled through MinIO's Access Management Plugin feature.
@@ -21,6 +22,7 @@ podman run -it \
 ### 2. Create a sample OPA Policy
 
 In another terminal, create a policy that allows root user all access and for all other users denies `PutObject`:
+
 ```sh
 cat > example.rego <<EOF
 package httpapi.authz
@@ -43,6 +45,7 @@ EOF
 ```
 
 Then load the policy via OPA's REST API.
+
 ```
 curl -X PUT --data-binary @example.rego \
   localhost:8181/v1/policies/putobject
@@ -78,4 +81,3 @@ mc cat foo/test/issue
 # 3. Attempt to upload an object as user `foo` - this will fail with a permissions error.
 mc cp /etc/issue myminio/test/issue2
 ```
-

docs/kms/README.md

@@ -123,13 +123,14 @@ Encrypted :
 ## Encrypted Private Key
 
 MinIO supports encrypted KES client private keys. Therefore, you can use
-an password-protected private keys for `MINIO_KMS_KES_KEY_FILE`. 
+an password-protected private keys for `MINIO_KMS_KES_KEY_FILE`.
 
 When using password-protected private keys for accessing KES you need to
 provide the password via:
+
 ```
 export MINIO_KMS_KES_KEY_PASSWORD=<your-password>
-``` 
+```
 
 Note that MinIO only supports encrypted private keys - not encrypted certificates.
 Certificates are no secrets and sent in plaintext as part of the TLS handshake.

docs/sts/web-identity.md

@@ -10,7 +10,6 @@ By default, the temporary security credentials created by AssumeRoleWithWebIdent
 
 ## Configuring OpenID identity provider on MinIO
 
-
 Configuration can be performed via MinIO's standard configuration API (i.e. using `mc admin config set/get` commands) or equivalently via environment variables. For brevity we show only environment variables here:
 
 ```

internal/config/compress/compress.go

@@ -39,10 +39,10 @@ const (
 	AllowEncrypted = "allow_encryption"
 	MimeTypes      = "mime_types"
 
-	EnvCompressState           = "MINIO_COMPRESS_ENABLE"
+	EnvCompressState           = "MINIO_COMPRESSION_ENABLE"
-	EnvCompressAllowEncryption = "MINIO_COMPRESS_ALLOW_ENCRYPTION"
+	EnvCompressAllowEncryption = "MINIO_COMPRESSION_ALLOW_ENCRYPTION"
-	EnvCompressExtensions      = "MINIO_COMPRESS_EXTENSIONS"
+	EnvCompressExtensions      = "MINIO_COMPRESSION_EXTENSIONS"
-	EnvCompressMimeTypes       = "MINIO_COMPRESS_MIME_TYPES"
+	EnvCompressMimeTypes       = "MINIO_COMPRESSION_MIME_TYPES"
 
 	// Include-list for compression.
 	DefaultExtensions = ".txt,.log,.csv,.json,.tar,.xml,.bin"
@@ -93,9 +93,9 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 		return cfg, err
 	}
 
-	compress := env.Get(EnvCompress, "")
+	compress := env.Get(EnvCompressState, kvs.Get(config.Enable))
 	if compress == "" {
-		compress = env.Get(EnvCompressState, kvs.Get(config.Enable))
+		compress = env.Get(EnvCompress, "")
 	}
 	cfg.Enabled, err = config.ParseBool(compress)
 	if err != nil {
@@ -110,36 +110,58 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 	}
 
 	allowEnc := env.Get(EnvCompressAllowEncryption, kvs.Get(AllowEncrypted))
+	if allowEnc == "" {
+		allowEnc = env.Get(EnvCompressAllowEncryptionLegacy, "")
+	}
+
 	cfg.AllowEncrypted, err = config.ParseBool(allowEnc)
 	if err != nil {
 		return cfg, err
 	}
 
 	compressExtensions := env.Get(EnvCompressExtensions, kvs.Get(Extensions))
+	compressExtensionsLegacy := env.Get(EnvCompressExtensionsLegacy, "")
 	compressMimeTypes := env.Get(EnvCompressMimeTypes, kvs.Get(MimeTypes))
-	compressMimeTypesLegacy := env.Get(EnvCompressMimeTypesLegacy, kvs.Get(MimeTypes))
+	compressMimeTypesLegacy1 := env.Get(EnvCompressMimeTypesLegacy1, "")
-	if compressExtensions != "" || compressMimeTypes != "" || compressMimeTypesLegacy != "" {
+	compressMimeTypesLegacy2 := env.Get(EnvCompressMimeTypesLegacy2, "")
-		if compressExtensions != "" {
+	if compressExtensions != "" {
-			extensions, err := parseCompressIncludes(compressExtensions)
+		extensions, err := parseCompressIncludes(compressExtensions)
-			if err != nil {
+		if err != nil {
-				return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", err, extensions)
+			return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESSION_EXTENSIONS value (`%s`)", err, extensions)
-			}
-			cfg.Extensions = extensions
 		}
-		if compressMimeTypes != "" {
+		cfg.Extensions = extensions
-			mimeTypes, err := parseCompressIncludes(compressMimeTypes)
+	}
-			if err != nil {
+
-				return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIME_TYPES value (`%s`)", err, mimeTypes)
+	if compressExtensionsLegacy != "" {
-			}
+		extensions, err := parseCompressIncludes(compressExtensions)
-			cfg.MimeTypes = mimeTypes
+		if err != nil {
+			return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", err, extensions)
 		}
-		if compressMimeTypesLegacy != "" {
+		cfg.Extensions = extensions
-			mimeTypes, err := parseCompressIncludes(compressMimeTypesLegacy)
+	}
-			if err != nil {
+
-				return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIME_TYPES value (`%s`)", err, mimeTypes)
+	if compressMimeTypes != "" {
-			}
+		mimeTypes, err := parseCompressIncludes(compressMimeTypes)
-			cfg.MimeTypes = mimeTypes
+		if err != nil {
+			return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESSION_MIME_TYPES value (`%s`)", err, mimeTypes)
 		}
+		cfg.MimeTypes = mimeTypes
+	}
+
+	if compressMimeTypesLegacy1 != "" {
+		mimeTypes, err := parseCompressIncludes(compressMimeTypesLegacy1)
+		if err != nil {
+			return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIMETYPES value (`%s`)", err, mimeTypes)
+		}
+		cfg.MimeTypes = mimeTypes
+	}
+
+	if compressMimeTypesLegacy2 != "" {
+		mimeTypes, err := parseCompressIncludes(compressMimeTypesLegacy2)
+		if err != nil {
+			return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIME_TYPES value (`%s`)", err, mimeTypes)
+		}
+		cfg.MimeTypes = mimeTypes
 	}
 
 	return cfg, nil

internal/config/compress/help.go

@@ -45,6 +45,12 @@ var (
 			Optional:    true,
 			Type:        "csv",
 		},
+		config.HelpKV{
+			Key:         AllowEncrypted,
+			Description: `enable 'encryption' along with compression`,
+			Optional:    true,
+			Type:        "on|off",
+		},
 		config.HelpKV{
 			Key:         config.Comment,
 			Description: config.DefaultComment,

internal/config/compress/legacy.go

@@ -25,8 +25,14 @@ import (
 
 // Legacy envs.
 const (
-	EnvCompress                = "MINIO_COMPRESS"
+	EnvCompress                 = "MINIO_COMPRESS"
-	EnvCompressMimeTypesLegacy = "MINIO_COMPRESS_MIMETYPES"
+	EnvCompressMimeTypesLegacy1 = "MINIO_COMPRESS_MIMETYPES"
+
+	// These envs were wrong but we supported them for a long time
+	// so keep them here to support existing deployments.
+	EnvCompressAllowEncryptionLegacy = "MINIO_COMPRESS_ALLOW_ENCRYPTION"
+	EnvCompressExtensionsLegacy      = "MINIO_COMPRESS_EXTENSIONS"
+	EnvCompressMimeTypesLegacy2      = "MINIO_COMPRESS_MIME_TYPES"
 )
 
 // SetCompressionConfig - One time migration code needed, for migrating from older config to new for Compression.