From b6eb8dff649b0f46c12d24e89aa11254fb0132fa Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sun, 17 Jul 2022 08:43:14 -0700 Subject: [PATCH] Add decommission compression+encryption enabled tests (#15322) update compression environment variables to follow the expected sub-system style, however support fallback mode. --- .github/workflows/markdown-lint.yaml | 2 +- Makefile | 1 + README.fips.md | 2 +- cmd/object-api-utils.go | 9 +- docs/bucket/replication/README.md | 2 + docs/bucket/versioning/DESIGN.md | 2 +- docs/bucket/versioning/README.md | 6 +- docs/compression/README.md | 8 +- docs/distributed/README.md | 1 - docs/distributed/decom-compressed-sse-s3.sh | 143 ++++++++++++++++++++ docs/gateway/nas.md | 4 +- docs/iam/access-management-plugin.md | 1 - docs/iam/opa.md | 4 +- docs/kms/README.md | 5 +- docs/sts/web-identity.md | 1 - internal/config/compress/compress.go | 74 ++++++---- internal/config/compress/help.go | 6 + internal/config/compress/legacy.go | 10 +- 18 files changed, 233 insertions(+), 48 deletions(-) create mode 100644 docs/distributed/decom-compressed-sse-s3.sh diff --git a/.github/workflows/markdown-lint.yaml b/.github/workflows/markdown-lint.yaml index 761b9fc27..0cdc22cf6 100644 --- a/.github/workflows/markdown-lint.yaml +++ b/.github/workflows/markdown-lint.yaml @@ -27,4 +27,4 @@ jobs: npm install -g markdownlint-cli markdownlint --fix '**/*.md' \ --config /home/runner/work/minio/minio/.github/markdown-lint-cfg.yaml \ - --disable MD013 MD040 + --disable MD013 MD040 MD051 diff --git a/Makefile b/Makefile index 01b3efcc7..dd5109a4a 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,7 @@ test-decom: install @env bash $(PWD)/docs/distributed/decom.sh @env bash $(PWD)/docs/distributed/decom-encrypted.sh @env bash $(PWD)/docs/distributed/decom-encrypted-sse-s3.sh + @env bash $(PWD)/docs/distributed/decom-compressed-sse-s3.sh test-upgrade: build @echo "Running minio upgrade tests" diff --git a/README.fips.md b/README.fips.md index 489dccb66..a87ef8e0b 100644 --- a/README.fips.md +++ b/README.fips.md @@ -2,6 +2,6 @@ MinIO creates FIPS builds using a patched version of the Go compiler (that uses BoringCrypto, from BoringSSL, which is [FIPS 140-2 validated](https://csrc.nist.gov/csrc/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp2964.pdf)) published by the Golang Team [here](https://github.com/golang/go/tree/dev.boringcrypto/misc/boring). -MinIO FIPS executables are available at http://dl.min.io - they are only published for `linux-amd64` architecture as binary files with the suffix `.fips`. We also publish corresponding container images to our official image repositories. +MinIO FIPS executables are available at - they are only published for `linux-amd64` architecture as binary files with the suffix `.fips`. We also publish corresponding container images to our official image repositories. We are not making any statements or representations about the suitability of this code or build in relation to the FIPS 140-2 standard. Interested users will have to evaluate for themselves whether this is useful for their own purposes. diff --git a/cmd/object-api-utils.go b/cmd/object-api-utils.go index 590ad1aaf..81f805bd8 100644 --- a/cmd/object-api-utils.go +++ b/cmd/object-api-utils.go @@ -440,10 +440,7 @@ func isCompressible(header http.Header, object string) bool { cfg := globalCompressConfig globalCompressConfigMu.Unlock() - if !cfg.Enabled || (crypto.Requested(header) && !cfg.AllowEncrypted) || excludeForCompression(header, object, cfg) { - return false - } - return true + return !excludeForCompression(header, object, cfg) } // Eliminate the non-compressible objects. @@ -454,6 +451,10 @@ func excludeForCompression(header http.Header, object string, cfg compress.Confi return true } + if crypto.Requested(header) && !cfg.AllowEncrypted { + return true + } + // We strictly disable compression for standard extensions/content-types (`compressed`). if hasStringSuffixInSlice(objStr, standardExcludeCompressExtensions) || hasPattern(standardExcludeCompressContentTypes, contentType) { return true diff --git a/docs/bucket/replication/README.md b/docs/bucket/replication/README.md index 8534f22c1..bd562b898 100644 --- a/docs/bucket/replication/README.md +++ b/docs/bucket/replication/README.md @@ -244,6 +244,7 @@ Replication from a source bucket to multiple destination buckets is supported. F Note that on the source side, the `X-Amz-Replication-Status` changes from `PENDING` to `COMPLETED` after replication succeeds to each of the targets. On the destination side, a `X-Amz-Replication-Status` status of `REPLICA` indicates that the object was replicated successfully. Any replication failures are automatically re-attempted during a periodic disk scanner cycle. ### Interaction with extended Bucket Versioning configuration + When Bucket Versioning with excluded prefixes are configured objects matching these prefixes are excluded from being versioned and replicated. ``` @@ -261,6 +262,7 @@ When Bucket Versioning with excluded prefixes are configured objects matching th ``` + In the above sample config, objects under prefixes matching any of the `ExcludedPrefixes` glob patterns will neither be versioned nor replicated. ## Explore Further diff --git a/docs/bucket/versioning/DESIGN.md b/docs/bucket/versioning/DESIGN.md index b78eae45c..a6e865a42 100644 --- a/docs/bucket/versioning/DESIGN.md +++ b/docs/bucket/versioning/DESIGN.md @@ -131,7 +131,7 @@ Inline data is optional. If no inline data is present, it is encoded as 0 bytes. | Entry | Encoding | Content | --------------------|-----------------------------|---------------------------------------- | xlMetaInlineDataVer | byte | version identifier -| id -> data | msgp map[string][]byte | Map of string id -> byte content +| id -> data | msgp `map[string][]byte` | Map of string id -> byte content Currently only xlMetaInlineDataVer == 1 exists. diff --git a/docs/bucket/versioning/README.md b/docs/bucket/versioning/README.md index 6f20618fc..eaa25ed1c 100644 --- a/docs/bucket/versioning/README.md +++ b/docs/bucket/versioning/README.md @@ -64,12 +64,15 @@ Similarly to suspend versioning set the configuration with Status set to `Suspen ``` ## MinIO extension to Bucket Versioning + ### Motivation -**PLEASE READ: This feature is meant for advanced usecases only where the setup is using bucket versioning or with replicated buckets, use this feature to optimize versioning behavior for some specific applications. MinIO experts will evaluate and guide on the benefits for your application, please reach out to us on https://subnet.min.io.** + +**PLEASE READ: This feature is meant for advanced usecases only where the setup is using bucket versioning or with replicated buckets, use this feature to optimize versioning behavior for some specific applications. MinIO experts will evaluate and guide on the benefits for your application, please reach out to us on .** Spark/Hadoop workloads which use Hadoop MR Committer v1/v2 algorithm upload objects to a temporary prefix in a bucket. These objects are 'renamed' to a different prefix on Job commit. Object storage admins are forced to configure separate ILM policies to expire these objects and their versions to reclaim space. ### Solution + To exclude objects under a list of prefix (glob) patterns from being versioned, you can send the following versioning configuration with Status set to `Enabled`. ``` @@ -92,6 +95,7 @@ To exclude objects under a list of prefix (glob) patterns from being versioned, ``` ### Features + - Objects matching these prefixes will behave as though versioning were suspended. These objects **will not** be replicated if bucket has replication configured. - Objects matching these prefixes will also not leave `null` delete markers, dramatically reduces namespace pollution while keeping the benefits of replication. - Users with explicit permissions or the root credential can configure the versioning state of any bucket. diff --git a/docs/compression/README.md b/docs/compression/README.md index eb08f05aa..8b6509615 100644 --- a/docs/compression/README.md +++ b/docs/compression/README.md @@ -54,9 +54,9 @@ The compression settings may also be set through environment variables. When set, environment variables override the defined `compress` config settings in the server config. ```bash -export MINIO_COMPRESS="on" -export MINIO_COMPRESS_EXTENSIONS=".txt,.log,.csv,.json,.tar,.xml,.bin" -export MINIO_COMPRESS_MIME_TYPES="text/*,application/json,application/xml" +export MINIO_COMPRESSION_ENABLE="on" +export MINIO_COMPRESSION_EXTENSIONS=".txt,.log,.csv,.json,.tar,.xml,.bin" +export MINIO_COMPRESSION_MIME_TYPES="text/*,application/json,application/xml" ``` ### 3. Compression + Encryption @@ -76,7 +76,7 @@ To enable compression+encryption use: ~ mc admin config set myminio compression allow_encryption=on ``` -Or alternatively through the environment variable `MINIO_COMPRESS_ALLOW_ENCRYPTION=on`. +Or alternatively through the environment variable `MINIO_COMPRESSION_ALLOW_ENCRYPTION=on`. ### 4. Excluded Types diff --git a/docs/distributed/README.md b/docs/distributed/README.md index 470489ec1..fe1e39eb9 100644 --- a/docs/distributed/README.md +++ b/docs/distributed/README.md @@ -28,7 +28,6 @@ MinIO follows strict **read-after-write** and **list-after-write** consistency m **If MinIO distributed setup is using NFS volumes underneath it is not guaranteed MinIO will provide these consistency guarantees since NFS is not strictly consistent (If you must use NFS we recommend that you atleast use NFSv4 instead of NFSv3 for relatively better outcomes).** - ## Get started If you're aware of stand-alone MinIO set up, the process remains largely the same. MinIO server automatically switches to stand-alone or distributed mode, depending on the command line parameters. diff --git a/docs/distributed/decom-compressed-sse-s3.sh b/docs/distributed/decom-compressed-sse-s3.sh new file mode 100644 index 000000000..d25104094 --- /dev/null +++ b/docs/distributed/decom-compressed-sse-s3.sh @@ -0,0 +1,143 @@ +#!/bin/bash + +if [ -n "$TEST_DEBUG" ]; then + set -x +fi + +pkill minio +rm -rf /tmp/xl + +if [ ! -f ./mc ]; then + wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && \ + chmod +x mc +fi + +export CI=true +export MINIO_COMPRESSION_ENABLE="on" +export MINIO_COMPRESSION_EXTENSIONS=".go" +export MINIO_COMPRESSION_MIME_TYPES="application/*" +export MINIO_COMPRESSION_ALLOW_ENCRYPTION="on" +export MINIO_KMS_AUTO_ENCRYPTION=on +export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= +export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9000/" + +(minio server /tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null)& +pid=$! + +sleep 2 + +./mc admin user add myminio/ minio123 minio123 +./mc admin user add myminio/ minio12345 minio12345 + +./mc admin policy add myminio/ rw ./docs/distributed/rw.json +./mc admin policy add myminio/ lake ./docs/distributed/rw.json + +./mc admin policy set myminio/ rw user=minio123 +./mc admin policy set myminio/ lake,rw user=minio12345 + +./mc mb -l myminio/versioned + +./mc mirror internal myminio/versioned/ --quiet >/dev/null + +## Soft delete (creates delete markers) +./mc rm -r --force myminio/versioned >/dev/null + +## mirror again to create another set of version on top +./mc mirror internal myminio/versioned/ --quiet >/dev/null + +expected_checksum=$(./mc cat internal/dsync/drwmutex.go | md5sum) + +user_count=$(./mc admin user list myminio/ | wc -l) +policy_count=$(./mc admin policy list myminio/ | wc -l) + +kill $pid + +(minio server /tmp/xl/{1...10}/disk{0...1} /tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded.log) & +pid=$! + +sleep 2 + +expanded_user_count=$(./mc admin user list myminio/ | wc -l) +expanded_policy_count=$(./mc admin policy list myminio/ | wc -l) + +if [ $user_count -ne $expanded_user_count ]; then + echo "BUG: original user count differs from expanded setup" + exit 1 +fi + +if [ $policy_count -ne $expanded_policy_count ]; then + echo "BUG: original policy count differs from expanded setup" + exit 1 +fi + +./mc version info myminio/versioned | grep -q "versioning is enabled" +ret=$? +if [ $ret -ne 0 ]; then + echo "expected versioning enabled after expansion" + exit 1 +fi + +./mc mirror cmd myminio/versioned/ --quiet >/dev/null + +./mc ls -r myminio/versioned/ > expanded_ns.txt +./mc ls -r --versions myminio/versioned/ > expanded_ns_versions.txt + +./mc admin decom start myminio/ /tmp/xl/{1...10}/disk{0...1} + +until $(./mc admin decom status myminio/ | grep -q Complete) +do + echo "waiting for decom to finish..." + sleep 1 +done + +kill $pid + +(minio server /tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/removed.log)& +pid=$! + +sleep 2 + +decom_user_count=$(./mc admin user list myminio/ | wc -l) +decom_policy_count=$(./mc admin policy list myminio/ | wc -l) + +if [ $user_count -ne $decom_user_count ]; then + echo "BUG: original user count differs after decommission" + exit 1 +fi + +if [ $policy_count -ne $decom_policy_count ]; then + echo "BUG: original policy count differs after decommission" + exit 1 +fi + +./mc version info myminio/versioned | grep -q "versioning is enabled" +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected versioning enabled after decommission" + exit 1 +fi + +got_checksum=$(./mc cat myminio/versioned/dsync/drwmutex.go | md5sum) +if [ "${expected_checksum}" != "${got_checksum}" ]; then + echo "BUG: decommission failed on encrypted objects: expected ${expected_checksum} got ${got_checksum}" + exit 1 +fi + +./mc ls -r myminio/versioned > decommissioned_ns.txt +./mc ls -r --versions myminio/versioned > decommissioned_ns_versions.txt + +out=$(diff -qpruN expanded_ns.txt decommissioned_ns.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after decommission: $out" + exit 1 +fi + +out=$(diff -qpruN expanded_ns_versions.txt decommissioned_ns_versions.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after decommission: $out" + exit 1 +fi + +# kill $pid diff --git a/docs/gateway/nas.md b/docs/gateway/nas.md index c0d25f38a..7740144cb 100644 --- a/docs/gateway/nas.md +++ b/docs/gateway/nas.md @@ -1,4 +1,4 @@ -# MinIO NAS Gateway [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) +# MinIO NAS Gateway [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) > NAS gateway is deprecated and will be removed in future, no more fresh deployments are supported. @@ -96,7 +96,7 @@ NAS gateway implementation allows symlinks on regular files. ### Behavior -- For reads symlinks resolve to the file the symlink points to. +- For reads symlinks resolve to the file the symlink points to. - For deletes - Deleting a symlink deletes the symlink but not the real file to which the symlink points. - Deleting the real file a symlink points to automatically makes the dangling symlink invisible. diff --git a/docs/iam/access-management-plugin.md b/docs/iam/access-management-plugin.md index 0ba9350a1..abba4d2d8 100644 --- a/docs/iam/access-management-plugin.md +++ b/docs/iam/access-management-plugin.md @@ -156,4 +156,3 @@ The following structure is also accepted: ``` Any unmentioned JSON object keys in the above are ignored. - diff --git a/docs/iam/opa.md b/docs/iam/opa.md index cc4a2261d..022423f8e 100644 --- a/docs/iam/opa.md +++ b/docs/iam/opa.md @@ -1,4 +1,5 @@ # OPA Quickstart Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) + OPA is a lightweight general-purpose policy engine that can be co-located with MinIO server, in this document we talk about how to use OPA HTTP API to authorize requests. It can be used with any type of credentials (STS based like OpenID or LDAP, regular IAM users or service accounts). OPA is enabled through MinIO's Access Management Plugin feature. @@ -21,6 +22,7 @@ podman run -it \ ### 2. Create a sample OPA Policy In another terminal, create a policy that allows root user all access and for all other users denies `PutObject`: + ```sh cat > example.rego < -``` +``` Note that MinIO only supports encrypted private keys - not encrypted certificates. Certificates are no secrets and sent in plaintext as part of the TLS handshake. diff --git a/docs/sts/web-identity.md b/docs/sts/web-identity.md index 8e5179224..3f39902c5 100644 --- a/docs/sts/web-identity.md +++ b/docs/sts/web-identity.md @@ -10,7 +10,6 @@ By default, the temporary security credentials created by AssumeRoleWithWebIdent ## Configuring OpenID identity provider on MinIO - Configuration can be performed via MinIO's standard configuration API (i.e. using `mc admin config set/get` commands) or equivalently via environment variables. For brevity we show only environment variables here: ``` diff --git a/internal/config/compress/compress.go b/internal/config/compress/compress.go index ffd04bacf..a4fe81a4d 100644 --- a/internal/config/compress/compress.go +++ b/internal/config/compress/compress.go @@ -39,10 +39,10 @@ const ( AllowEncrypted = "allow_encryption" MimeTypes = "mime_types" - EnvCompressState = "MINIO_COMPRESS_ENABLE" - EnvCompressAllowEncryption = "MINIO_COMPRESS_ALLOW_ENCRYPTION" - EnvCompressExtensions = "MINIO_COMPRESS_EXTENSIONS" - EnvCompressMimeTypes = "MINIO_COMPRESS_MIME_TYPES" + EnvCompressState = "MINIO_COMPRESSION_ENABLE" + EnvCompressAllowEncryption = "MINIO_COMPRESSION_ALLOW_ENCRYPTION" + EnvCompressExtensions = "MINIO_COMPRESSION_EXTENSIONS" + EnvCompressMimeTypes = "MINIO_COMPRESSION_MIME_TYPES" // Include-list for compression. DefaultExtensions = ".txt,.log,.csv,.json,.tar,.xml,.bin" @@ -93,9 +93,9 @@ func LookupConfig(kvs config.KVS) (Config, error) { return cfg, err } - compress := env.Get(EnvCompress, "") + compress := env.Get(EnvCompressState, kvs.Get(config.Enable)) if compress == "" { - compress = env.Get(EnvCompressState, kvs.Get(config.Enable)) + compress = env.Get(EnvCompress, "") } cfg.Enabled, err = config.ParseBool(compress) if err != nil { @@ -110,36 +110,58 @@ func LookupConfig(kvs config.KVS) (Config, error) { } allowEnc := env.Get(EnvCompressAllowEncryption, kvs.Get(AllowEncrypted)) + if allowEnc == "" { + allowEnc = env.Get(EnvCompressAllowEncryptionLegacy, "") + } + cfg.AllowEncrypted, err = config.ParseBool(allowEnc) if err != nil { return cfg, err } compressExtensions := env.Get(EnvCompressExtensions, kvs.Get(Extensions)) + compressExtensionsLegacy := env.Get(EnvCompressExtensionsLegacy, "") compressMimeTypes := env.Get(EnvCompressMimeTypes, kvs.Get(MimeTypes)) - compressMimeTypesLegacy := env.Get(EnvCompressMimeTypesLegacy, kvs.Get(MimeTypes)) - if compressExtensions != "" || compressMimeTypes != "" || compressMimeTypesLegacy != "" { - if compressExtensions != "" { - extensions, err := parseCompressIncludes(compressExtensions) - if err != nil { - return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", err, extensions) - } - cfg.Extensions = extensions + compressMimeTypesLegacy1 := env.Get(EnvCompressMimeTypesLegacy1, "") + compressMimeTypesLegacy2 := env.Get(EnvCompressMimeTypesLegacy2, "") + if compressExtensions != "" { + extensions, err := parseCompressIncludes(compressExtensions) + if err != nil { + return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESSION_EXTENSIONS value (`%s`)", err, extensions) } - if compressMimeTypes != "" { - mimeTypes, err := parseCompressIncludes(compressMimeTypes) - if err != nil { - return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIME_TYPES value (`%s`)", err, mimeTypes) - } - cfg.MimeTypes = mimeTypes + cfg.Extensions = extensions + } + + if compressExtensionsLegacy != "" { + extensions, err := parseCompressIncludes(compressExtensions) + if err != nil { + return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", err, extensions) } - if compressMimeTypesLegacy != "" { - mimeTypes, err := parseCompressIncludes(compressMimeTypesLegacy) - if err != nil { - return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIME_TYPES value (`%s`)", err, mimeTypes) - } - cfg.MimeTypes = mimeTypes + cfg.Extensions = extensions + } + + if compressMimeTypes != "" { + mimeTypes, err := parseCompressIncludes(compressMimeTypes) + if err != nil { + return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESSION_MIME_TYPES value (`%s`)", err, mimeTypes) } + cfg.MimeTypes = mimeTypes + } + + if compressMimeTypesLegacy1 != "" { + mimeTypes, err := parseCompressIncludes(compressMimeTypesLegacy1) + if err != nil { + return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIMETYPES value (`%s`)", err, mimeTypes) + } + cfg.MimeTypes = mimeTypes + } + + if compressMimeTypesLegacy2 != "" { + mimeTypes, err := parseCompressIncludes(compressMimeTypesLegacy2) + if err != nil { + return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIME_TYPES value (`%s`)", err, mimeTypes) + } + cfg.MimeTypes = mimeTypes } return cfg, nil diff --git a/internal/config/compress/help.go b/internal/config/compress/help.go index a74a932b1..e7f9a1fcb 100644 --- a/internal/config/compress/help.go +++ b/internal/config/compress/help.go @@ -45,6 +45,12 @@ var ( Optional: true, Type: "csv", }, + config.HelpKV{ + Key: AllowEncrypted, + Description: `enable 'encryption' along with compression`, + Optional: true, + Type: "on|off", + }, config.HelpKV{ Key: config.Comment, Description: config.DefaultComment, diff --git a/internal/config/compress/legacy.go b/internal/config/compress/legacy.go index 15bf5f126..13be9069a 100644 --- a/internal/config/compress/legacy.go +++ b/internal/config/compress/legacy.go @@ -25,8 +25,14 @@ import ( // Legacy envs. const ( - EnvCompress = "MINIO_COMPRESS" - EnvCompressMimeTypesLegacy = "MINIO_COMPRESS_MIMETYPES" + EnvCompress = "MINIO_COMPRESS" + EnvCompressMimeTypesLegacy1 = "MINIO_COMPRESS_MIMETYPES" + + // These envs were wrong but we supported them for a long time + // so keep them here to support existing deployments. + EnvCompressAllowEncryptionLegacy = "MINIO_COMPRESS_ALLOW_ENCRYPTION" + EnvCompressExtensionsLegacy = "MINIO_COMPRESS_EXTENSIONS" + EnvCompressMimeTypesLegacy2 = "MINIO_COMPRESS_MIME_TYPES" ) // SetCompressionConfig - One time migration code needed, for migrating from older config to new for Compression.