rename all remaining packages to internal/ (#12418)

This is to ensure that there are no projects
that try to import `minio/minio/pkg` into
their own repo. Any such common packages should
go to `https://github.com/minio/pkg`

internal/config/compress/compress.go

@@ -0,0 +1,146 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package compress
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/minio/minio/internal/config"
+	"github.com/minio/pkg/env"
+)
+
+// Config represents the compression settings.
+type Config struct {
+	Enabled        bool     `json:"enabled"`
+	AllowEncrypted bool     `json:"allow_encryption"`
+	Extensions     []string `json:"extensions"`
+	MimeTypes      []string `json:"mime-types"`
+}
+
+// Compression environment variables
+const (
+	Extensions     = "extensions"
+	AllowEncrypted = "allow_encryption"
+	MimeTypes      = "mime_types"
+
+	EnvCompressState           = "MINIO_COMPRESS_ENABLE"
+	EnvCompressAllowEncryption = "MINIO_COMPRESS_ALLOW_ENCRYPTION"
+	EnvCompressExtensions      = "MINIO_COMPRESS_EXTENSIONS"
+	EnvCompressMimeTypes       = "MINIO_COMPRESS_MIME_TYPES"
+
+	// Include-list for compression.
+	DefaultExtensions = ".txt,.log,.csv,.json,.tar,.xml,.bin"
+	DefaultMimeTypes  = "text/*,application/json,application/xml,binary/octet-stream"
+)
+
+// DefaultKVS - default KV config for compression settings
+var (
+	DefaultKVS = config.KVS{
+		config.KV{
+			Key:   config.Enable,
+			Value: config.EnableOff,
+		},
+		config.KV{
+			Key:   AllowEncrypted,
+			Value: config.EnableOff,
+		},
+		config.KV{
+			Key:   Extensions,
+			Value: DefaultExtensions,
+		},
+		config.KV{
+			Key:   MimeTypes,
+			Value: DefaultMimeTypes,
+		},
+	}
+)
+
+// Parses the given compression exclude list `extensions` or `content-types`.
+func parseCompressIncludes(include string) ([]string, error) {
+	includes := strings.Split(include, config.ValueSeparator)
+	for _, e := range includes {
+		if len(e) == 0 {
+			return nil, config.ErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type cannot be empty")
+		}
+		if e == "/" {
+			return nil, config.ErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type cannot be '/'")
+		}
+	}
+	return includes, nil
+}
+
+// LookupConfig - lookup compression config.
+func LookupConfig(kvs config.KVS) (Config, error) {
+	var err error
+	cfg := Config{}
+	if err = config.CheckValidKeys(config.CompressionSubSys, kvs, DefaultKVS); err != nil {
+		return cfg, err
+	}
+
+	compress := env.Get(EnvCompress, "")
+	if compress == "" {
+		compress = env.Get(EnvCompressState, kvs.Get(config.Enable))
+	}
+	cfg.Enabled, err = config.ParseBool(compress)
+	if err != nil {
+		// Parsing failures happen due to empty KVS, ignore it.
+		if kvs.Empty() {
+			return cfg, nil
+		}
+		return cfg, err
+	}
+	if !cfg.Enabled {
+		return cfg, nil
+	}
+
+	allowEnc := env.Get(EnvCompressAllowEncryption, kvs.Get(AllowEncrypted))
+	cfg.AllowEncrypted, err = config.ParseBool(allowEnc)
+	if err != nil {
+		return cfg, err
+	}
+
+	compressExtensions := env.Get(EnvCompressExtensions, kvs.Get(Extensions))
+	compressMimeTypes := env.Get(EnvCompressMimeTypes, kvs.Get(MimeTypes))
+	compressMimeTypesLegacy := env.Get(EnvCompressMimeTypesLegacy, kvs.Get(MimeTypes))
+	if compressExtensions != "" || compressMimeTypes != "" || compressMimeTypesLegacy != "" {
+		if compressExtensions != "" {
+			extensions, err := parseCompressIncludes(compressExtensions)
+			if err != nil {
+				return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", err, extensions)
+			}
+			cfg.Extensions = extensions
+		}
+		if compressMimeTypes != "" {
+			mimeTypes, err := parseCompressIncludes(compressMimeTypes)
+			if err != nil {
+				return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIME_TYPES value (`%s`)", err, mimeTypes)
+			}
+			cfg.MimeTypes = mimeTypes
+		}
+		if compressMimeTypesLegacy != "" {
+			mimeTypes, err := parseCompressIncludes(compressMimeTypesLegacy)
+			if err != nil {
+				return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIME_TYPES value (`%s`)", err, mimeTypes)
+			}
+			cfg.MimeTypes = mimeTypes
+		}
+	}
+
+	return cfg, nil
+}

internal/config/compress/compress_test.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package compress
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParseCompressIncludes(t *testing.T) {
+	testCases := []struct {
+		str              string
+		expectedPatterns []string
+		success          bool
+	}{
+		// invalid input
+		{",,,", []string{}, false},
+		{"", []string{}, false},
+		{",", []string{}, false},
+		{"/", []string{}, false},
+		{"text/*,/", []string{}, false},
+
+		// valid input
+		{".txt,.log", []string{".txt", ".log"}, true},
+		{"text/*,application/json", []string{"text/*", "application/json"}, true},
+	}
+
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run(testCase.str, func(t *testing.T) {
+			gotPatterns, err := parseCompressIncludes(testCase.str)
+			if !testCase.success && err == nil {
+				t.Error("expected failure but success instead")
+			}
+			if testCase.success && err != nil {
+				t.Errorf("expected success but failed instead %s", err)
+			}
+			if testCase.success && !reflect.DeepEqual(testCase.expectedPatterns, gotPatterns) {
+				t.Errorf("expected patterns %s but got %s", testCase.expectedPatterns, gotPatterns)
+			}
+		})
+	}
+}

internal/config/compress/help.go

@@ -0,0 +1,44 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package compress
+
+import "github.com/minio/minio/internal/config"
+
+// Help template for compress feature.
+var (
+	Help = config.HelpKVS{
+		config.HelpKV{
+			Key:         Extensions,
+			Description: `comma separated file extensions e.g. ".txt,.log,.csv"`,
+			Optional:    true,
+			Type:        "csv",
+		},
+		config.HelpKV{
+			Key:         MimeTypes,
+			Description: `comma separated wildcard mime-types e.g. "text/*,application/json,application/xml"`,
+			Optional:    true,
+			Type:        "csv",
+		},
+		config.HelpKV{
+			Key:         config.Comment,
+			Description: config.DefaultComment,
+			Optional:    true,
+			Type:        "sentence",
+		},
+	}
+)

internal/config/compress/legacy.go

@@ -0,0 +1,52 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package compress
+
+import (
+	"strings"
+
+	"github.com/minio/minio/internal/config"
+)
+
+// Legacy envs.
+const (
+	EnvCompress                = "MINIO_COMPRESS"
+	EnvCompressMimeTypesLegacy = "MINIO_COMPRESS_MIMETYPES"
+)
+
+// SetCompressionConfig - One time migration code needed, for migrating from older config to new for Compression.
+func SetCompressionConfig(s config.Config, cfg Config) {
+	if !cfg.Enabled {
+		// No need to save disabled settings in new config.
+		return
+	}
+	s[config.CompressionSubSys][config.Default] = config.KVS{
+		config.KV{
+			Key:   config.Enable,
+			Value: config.EnableOn,
+		},
+		config.KV{
+			Key:   Extensions,
+			Value: strings.Join(cfg.Extensions, config.ValueSeparator),
+		},
+		config.KV{
+			Key:   MimeTypes,
+			Value: strings.Join(cfg.MimeTypes, config.ValueSeparator),
+		},
+	}
+}