MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-24 03:27:44 -05:00
Code Issues Packages Projects Releases Wiki Activity

move the dependency to minio/pkg for common libraries (#12397)

Browse Source
This commit is contained in:
Harshavardhana
2021-05-28 15:17:01 -07:00
committed by GitHub
parent 547bb7d0a1
commit 81d5688d56
139 changed files with 122 additions and 10898 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/bucket/object/lock/lock.go
View File

@@ -29,7 +29,7 @@ import (
"github.com/beevik/ntp"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/env"
"github.com/minio/pkg/env"
)
// RetMode - object retention mode.

2
pkg/bucket/policy/condition/stringlikefunc.go
View File

@@ -24,7 +24,7 @@ import (
"github.com/minio/minio-go/v7/pkg/s3utils"
"github.com/minio/minio-go/v7/pkg/set"
"github.com/minio/minio/pkg/wildcard"
"github.com/minio/pkg/wildcard"
)
func toStringLikeFuncString(n name, key Key, values set.StringSet) string {

2
pkg/bucket/policy/principal.go
View File

@@ -21,7 +21,7 @@ import (
"encoding/json"
"github.com/minio/minio-go/v7/pkg/set"
"github.com/minio/minio/pkg/wildcard"
"github.com/minio/pkg/wildcard"
)
// Principal - policy principal.

2
pkg/bucket/policy/resource.go
View File

@@ -22,7 +22,7 @@ import (
"strings"
"github.com/minio/minio/pkg/bucket/policy/condition"
"github.com/minio/minio/pkg/wildcard"
"github.com/minio/pkg/wildcard"
)
// ResourceARNPrefix - resource ARN prefix as per AWS S3 specification.

2
pkg/bucket/replication/destination.go
View File

@@ -22,7 +22,7 @@ import (
"fmt"
"strings"
"github.com/minio/minio/pkg/wildcard"
"github.com/minio/pkg/wildcard"
)
// DestinationARNPrefix - destination ARN prefix as per AWS S3 specification.

93
pkg/certs/ca-certs.go
View File

@@ -1,93 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package certs
import (
"crypto/x509"
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
)
// GetRootCAs loads all X.509 certificates at the given path and adds them
// to the list of system root CAs, if available. The returned CA pool
// is a conjunction of the system root CAs and the certificate(s) at
// the given path.
//
// If path is a regular file, LoadCAs simply adds it to the CA pool
// if the file contains a valid X.509 certificate
//
// If the path points to a directory, LoadCAs iterates over all top-level
// files within the directory and adds them to the CA pool if they contain
// a valid X.509 certificate.
func GetRootCAs(path string) (*x509.CertPool, error) {
rootCAs, _ := loadSystemRoots()
if rootCAs == nil {
// In some systems system cert pool is not supported
// or no certificates are present on the
// system - so we create a new cert pool.
rootCAs = x509.NewCertPool()
}
// Open the file path and check whether its a regular file
// or a directory.
f, err := os.Open(path)
if errors.Is(err, os.ErrNotExist) {
return rootCAs, nil
}
if errors.Is(err, os.ErrPermission) {
return rootCAs, nil
}
if err != nil {
return rootCAs, err
}
defer f.Close()
stat, err := f.Stat()
if err != nil {
return rootCAs, err
}
// In case of a file add it to the root CAs.
if !stat.IsDir() {
bytes, err := ioutil.ReadAll(f)
if err != nil {
return rootCAs, err
}
if !rootCAs.AppendCertsFromPEM(bytes) {
return rootCAs, fmt.Errorf("cert: %q does not contain a valid X.509 PEM-encoded certificate", path)
}
return rootCAs, nil
}
// Otherwise iterate over the files in the directory
// and add each on to the root CAs.
files, err := f.Readdirnames(0)
if err != nil {
return rootCAs, err
}
for _, file := range files {
bytes, err := ioutil.ReadFile(filepath.Join(path, file))
if err == nil { // ignore files which are not readable.
rootCAs.AppendCertsFromPEM(bytes)
}
}
return rootCAs, nil
}

79
pkg/certs/ca-certs_test.go
View File

@@ -1,79 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package certs
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
)
func TestGetRootCAs(t *testing.T) {
emptydir, err := ioutil.TempDir("", "test-get-root-cas")
if err != nil {
t.Fatalf("Unable create temp directory. %v", emptydir)
}
defer os.RemoveAll(emptydir)
dir1, err := ioutil.TempDir("", "test-get-root-cas")
if err != nil {
t.Fatalf("Unable create temp directory. %v", dir1)
}
defer os.RemoveAll(dir1)
if err = os.Mkdir(filepath.Join(dir1, "empty-dir"), 0755); err != nil {
t.Fatalf("Unable create empty dir. %v", err)
}
dir2, err := ioutil.TempDir("", "test-get-root-cas")
if err != nil {
t.Fatalf("Unable create temp directory. %v", dir2)
}
defer os.RemoveAll(dir2)
if err = ioutil.WriteFile(filepath.Join(dir2, "empty-file"), []byte{}, 0644); err != nil {
t.Fatalf("Unable create test file. %v", err)
}
testCases := []struct {
certCAsDir string
expectedErr error
}{
// ignores non-existent directories.
{"nonexistent-dir", nil},
// Ignores directories.
{dir1, nil},
// Ignore empty directory.
{emptydir, nil},
// Loads the cert properly.
{dir2, nil},
}
for _, testCase := range testCases {
_, err := GetRootCAs(testCase.certCAsDir)
if testCase.expectedErr == nil {
if err != nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
}
} else if err == nil {
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
} else if testCase.expectedErr.Error() != err.Error() {
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
}
}
}

85
pkg/certs/cert_pool_nix.go
View File

@@ -1,85 +0,0 @@
// +build !windows
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package certs
import (
"crypto/x509"
"io/ioutil"
"os"
"path/filepath"
"strings"
)
// Possible directories with certificate files, this is an extended
// list from https://golang.org/src/crypto/x509/root_unix.go?#L18
// for k8s platform
var certDirectories = []string{
"/var/run/secrets/kubernetes.io/serviceaccount",
}
// readUniqueDirectoryEntries is like ioutil.ReadDir but omits
// symlinks that point within the directory.
func readUniqueDirectoryEntries(dir string) ([]os.FileInfo, error) {
fis, err := ioutil.ReadDir(dir)
if err != nil {
return nil, err
}
uniq := fis[:0]
for _, fi := range fis {
if !isSameDirSymlink(fi, dir) {
uniq = append(uniq, fi)
}
}
return uniq, nil
}
// isSameDirSymlink reports whether fi in dir is a symlink with a
// target not containing a slash.
func isSameDirSymlink(fi os.FileInfo, dir string) bool {
if fi.Mode()&os.ModeSymlink == 0 {
return false
}
target, err := os.Readlink(filepath.Join(dir, fi.Name()))
return err == nil && !strings.Contains(target, "/")
}
func loadSystemRoots() (*x509.CertPool, error) {
caPool, err := x509.SystemCertPool()
if err != nil {
return caPool, err
}
for _, directory := range certDirectories {
fis, err := readUniqueDirectoryEntries(directory)
if err != nil {
if os.IsNotExist(err) || os.IsPermission(err) {
return caPool, nil
}
return caPool, err
}
for _, fi := range fis {
data, err := ioutil.ReadFile(directory + "/" + fi.Name())
if err == nil {
caPool.AppendCertsFromPEM(data)
}
}
}
return caPool, nil
}

61
pkg/certs/cert_pool_windows.go
View File

@@ -1,61 +0,0 @@
// +build windows
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package certs
import (
"crypto/x509"
"syscall"
"unsafe"
)
func loadSystemRoots() (*x509.CertPool, error) {
const CRYPTENOTFOUND = 0x80092004
store, err := syscall.CertOpenSystemStore(0, syscall.StringToUTF16Ptr("ROOT"))
if err != nil {
return nil, err
}
defer syscall.CertCloseStore(store, 0)
roots := x509.NewCertPool()
var cert *syscall.CertContext
for {
cert, err = syscall.CertEnumCertificatesInStore(store, cert)
if err != nil {
if errno, ok := err.(syscall.Errno); ok {
if errno == CRYPTENOTFOUND {
break
}
}
return nil, err
}
if cert == nil {
break
}
// Copy the buf, since ParseCertificate does not create its own copy.
buf := (*[1 << 20]byte)(unsafe.Pointer(cert.EncodedCert))[:]
buf2 := make([]byte, cert.Length)
copy(buf2, buf)
if c, err := x509.ParseCertificate(buf2); err == nil {
roots.AddCert(c)
}
}
return roots, nil
}

347
pkg/certs/certs.go
View File

@@ -1,347 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package certs
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"os"
"path/filepath"
"sync"
"time"
"github.com/rjeczalik/notify"
)
// LoadX509KeyPairFunc is a function that parses a private key and
// certificate file and returns a TLS certificate on success.
type LoadX509KeyPairFunc func(certFile, keyFile string) (tls.Certificate, error)
// GetCertificateFunc is a callback that allows a TLS stack deliver different
// certificates based on the client trying to establish a TLS connection.
//
// For example, a GetCertificateFunc can return different TLS certificates depending
// upon the TLS SNI sent by the client.
type GetCertificateFunc func(hello *tls.ClientHelloInfo) (*tls.Certificate, error)
// Manager is a TLS certificate manager that can handle multiple certificates.
// When a client tries to establish a TLS connection, Manager will try to
// pick a certificate that can be validated by the client.
//
// For instance, if the client specifies a TLS SNI then Manager will try to
// find the corresponding certificate. If there is no such certificate it
// will fallback to the certificate named public.crt.
//
// Manager will automatically reload certificates if the corresponding file changes.
type Manager struct {
lock sync.RWMutex
certificates map[pair]*tls.Certificate // Mapping: certificate file name => TLS certificates
defaultCert pair
loadX509KeyPair LoadX509KeyPairFunc
events chan notify.EventInfo
ctx context.Context
}
// pair represents a certificate and private key file tuple.
type pair struct {
KeyFile string
CertFile string
}
// NewManager returns a new Manager that handles one certificate specified via
// the certFile and keyFile. It will use the loadX509KeyPair function to (re)load
// certificates.
//
// The certificate loaded from certFile is considered the default certificate.
// If a client does not send the TLS SNI extension then Manager will return
// this certificate.
func NewManager(ctx context.Context, certFile, keyFile string, loadX509KeyPair LoadX509KeyPairFunc) (manager *Manager, err error) {
certFile, err = filepath.Abs(certFile)
if err != nil {
return nil, err
}
keyFile, err = filepath.Abs(keyFile)
if err != nil {
return nil, err
}
manager = &Manager{
certificates: map[pair]*tls.Certificate{},
defaultCert: pair{
KeyFile: keyFile,
CertFile: certFile,
},
loadX509KeyPair: loadX509KeyPair,
events: make(chan notify.EventInfo, 1),
ctx: ctx,
}
if err := manager.AddCertificate(certFile, keyFile); err != nil {
return nil, err
}
go manager.watchFileEvents()
return manager, nil
}
// AddCertificate adds the TLS certificate in certFile resp. keyFile
// to the Manager.
//
// If there is already a certificate with the same base name it will be
// replaced by the newly added one.
func (m *Manager) AddCertificate(certFile, keyFile string) (err error) {
certFile, err = filepath.Abs(certFile)
if err != nil {
return err
}
keyFile, err = filepath.Abs(keyFile)
if err != nil {
return err
}
certFileIsLink, err := isSymlink(certFile)
if err != nil {
return err
}
keyFileIsLink, err := isSymlink(keyFile)
if err != nil {
return err
}
if certFileIsLink && !keyFileIsLink {
return fmt.Errorf("certs: '%s' is a symlink but '%s' is a regular file", certFile, keyFile)
}
if keyFileIsLink && !certFileIsLink {
return fmt.Errorf("certs: '%s' is a symlink but '%s' is a regular file", keyFile, certFile)
}
certificate, err := m.loadX509KeyPair(certFile, keyFile)
if err != nil {
return err
}
// We set the certificate leaf to the actual certificate such that
// we don't have to do the parsing (multiple times) when matching the
// certificate to the client hello. This a performance optimisation.
if certificate.Leaf == nil {
certificate.Leaf, err = x509.ParseCertificate(certificate.Certificate[0])
if err != nil {
return err
}
}
p := pair{
CertFile: certFile,
KeyFile: keyFile,
}
m.lock.Lock()
defer m.lock.Unlock()
// We don't allow IP SANs in certificates - except for the "default" certificate
// which is, by convention, the first certificate added to the manager. The problem
// with allowing IP SANs in more than one certificate is that the manager usually can't
// match the client SNI to a SAN since the SNI is meant to communicate the destination
// host name and clients will not set the SNI to an IP address.
// Allowing multiple certificates with IP SANs lead to errors that confuses users - like:
// "It works for `https://instance.minio.local` but not for `https://10.0.2.1`"
if len(m.certificates) > 0 && len(certificate.Leaf.IPAddresses) > 0 {
return errors.New("cert: certificate must not contain any IP SANs: only the default certificate may contain IP SANs")
}
m.certificates[p] = &certificate
if certFileIsLink && keyFileIsLink {
go m.watchSymlinks(certFile, keyFile)
} else {
// Windows doesn't allow for watching file changes but instead allows
// for directory changes only, while we can still watch for changes
// on files on other platforms. Watch parent directory on all platforms
// for simplicity.
if err = notify.Watch(filepath.Dir(certFile), m.events, eventWrite...); err != nil {
return err
}
if err = notify.Watch(filepath.Dir(keyFile), m.events, eventWrite...); err != nil {
return err
}
}
return nil
}
// watchSymlinks starts an endless loop reloading the
// certFile and keyFile periodically.
func (m *Manager) watchSymlinks(certFile, keyFile string) {
for {
select {
case <-m.ctx.Done():
return // Once stopped exits this routine.
case <-time.After(24 * time.Hour):
certificate, err := m.loadX509KeyPair(certFile, keyFile)
if err != nil {
continue
}
if certificate.Leaf == nil { // This is a performance optimisation
certificate.Leaf, err = x509.ParseCertificate(certificate.Certificate[0])
if err != nil {
continue
}
}
p := pair{
CertFile: certFile,
KeyFile: keyFile,
}
m.lock.Lock()
m.certificates[p] = &certificate
m.lock.Unlock()
}
}
}
// watchFileEvents starts an endless loop waiting for file systems events.
// Once an event occurs it reloads the private key and certificate that
// has changed, if any.
func (m *Manager) watchFileEvents() {
for {
select {
case <-m.ctx.Done():
return
case event := <-m.events:
if !isWriteEvent(event.Event()) {
continue
}
for pair := range m.certificates {
if p := event.Path(); pair.KeyFile == p || pair.CertFile == p {
certificate, err := m.loadX509KeyPair(pair.CertFile, pair.KeyFile)
if err != nil {
continue
}
if certificate.Leaf == nil { // This is performance optimisation
certificate.Leaf, err = x509.ParseCertificate(certificate.Certificate[0])
if err != nil {
continue
}
}
m.lock.Lock()
m.certificates[pair] = &certificate
m.lock.Unlock()
}
}
}
}
}
// GetCertificate returns a TLS certificate based on the client hello.
//
// It tries to find a certificate that would be accepted by the client
// according to the client hello. However, if no certificate can be
// found GetCertificate returns the certificate loaded from the
// Public file.
func (m *Manager) GetCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
m.lock.RLock()
defer m.lock.RUnlock()
// If the client does not send a SNI we return the "default"
// certificate. A client may not send a SNI - e.g. when trying
// to connect to an IP directly (https://<ip>:<port>).
//
// In this case we don't know which the certificate the client
// asks for. It may be a public-facing certificate issued by a
// public CA or an internal certificate containing internal domain
// names.
// Now, we should not serve "the first" certificate that would be
// accepted by the client based on the Client Hello. Otherwise, we
// may expose an internal certificate to the client that contains
// internal domain names. That way we would disclose internal
// infrastructure details.
//
// Therefore, we serve the "default" certificate - which by convention
// is the first certificate added to the Manager. It's the calling code's
// responsibility to ensure that the "public-facing" certificate is used
// when creating a Manager instance.
if hello.ServerName == "" {
certificate := m.certificates[m.defaultCert]
return certificate, nil
}
// Optimization: If there is just one certificate, always serve that one.
if len(m.certificates) == 1 {
for _, certificate := range m.certificates {
return certificate, nil
}
}
// Iterate over all certificates and return the first one that would
// be accepted by the peer (TLS client) based on the client hello.
// In particular, the client usually specifies the requested host/domain
// via SNI.
//
// Note: The certificate.Leaf should be non-nil and contain the actual
// client certificate of MinIO that should be presented to the peer (TLS client).
// Otherwise, the leaf certificate has to be parsed again - which is kind of
// expensive and may cause a performance issue. For more information, check the
// docs of tls.ClientHelloInfo.SupportsCertificate.
for _, certificate := range m.certificates {
if err := hello.SupportsCertificate(certificate); err == nil {
return certificate, nil
}
}
return nil, errors.New("certs: no server certificate is supported by peer")
}
// GetClientCertificate returns a TLS certificate for mTLS based on the
// certificate request.
//
// It tries to find a certificate that would be accepted by the server
// according to the certificate request. However, if no certificate can be
// found GetClientCertificate returns the certificate loaded from the
// Public file.
func (m *Manager) GetClientCertificate(reqInfo *tls.CertificateRequestInfo) (*tls.Certificate, error) {
m.lock.RLock()
defer m.lock.RUnlock()
// Optimization: If there is just one certificate, always serve that one.
if len(m.certificates) == 1 {
for _, certificate := range m.certificates {
return certificate, nil
}
}
// Iterate over all certificates and return the first one that would
// be accepted by the peer (TLS server) based on reqInfo.
//
// Note: The certificate.Leaf should be non-nil and contain the actual
// client certificate of MinIO that should be presented to the peer (TLS server).
// Otherwise, the leaf certificate has to be parsed again - which is kind of
// expensive and may cause a performance issue. For more information, check the
// docs of tls.CertificateRequestInfo.SupportsCertificate.
for _, certificate := range m.certificates {
if err := reqInfo.SupportsCertificate(certificate); err == nil {
return certificate, nil
}
}
return nil, errors.New("certs: no client certificate is supported by peer")
}
// isSymlink returns true if the given file
// is a symbolic link.
func isSymlink(file string) (bool, error) {
st, err := os.Lstat(file)
if err != nil {
return false, err
}
return st.Mode()&os.ModeSymlink == os.ModeSymlink, nil
}

110
pkg/certs/certs_test.go
View File

@@ -1,110 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package certs_test
import (
"context"
"crypto/tls"
"io"
"os"
"reflect"
"testing"
"time"
"github.com/minio/minio/pkg/certs"
)
func updateCerts(crt, key string) {
// ignore error handling
crtSource, _ := os.Open(crt)
defer crtSource.Close()
crtDest, _ := os.Create("public.crt")
defer crtDest.Close()
io.Copy(crtDest, crtSource)
keySource, _ := os.Open(key)
defer keySource.Close()
keyDest, _ := os.Create("private.key")
defer keyDest.Close()
io.Copy(keyDest, keySource)
}
func TestNewManager(t *testing.T) {
ctx, cancelFn := context.WithCancel(context.Background())
defer cancelFn()
c, err := certs.NewManager(ctx, "public.crt", "private.key", tls.LoadX509KeyPair)
if err != nil {
t.Fatal(err)
}
hello := &tls.ClientHelloInfo{}
gcert, err := c.GetCertificate(hello)
if err != nil {
t.Fatal(err)
}
expectedCert, err := tls.LoadX509KeyPair("public.crt", "private.key")
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(gcert.Certificate, expectedCert.Certificate) {
t.Error("certificate doesn't match expected certificate")
}
_, err = certs.NewManager(ctx, "public.crt", "new-private.key", tls.LoadX509KeyPair)
if err == nil {
t.Fatal("Expected to fail but got success")
}
}
func TestValidPairAfterWrite(t *testing.T) {
ctx, cancelFn := context.WithCancel(context.Background())
defer cancelFn()
expectedCert, err := tls.LoadX509KeyPair("new-public.crt", "new-private.key")
if err != nil {
t.Fatal(err)
}
c, err := certs.NewManager(ctx, "public.crt", "private.key", tls.LoadX509KeyPair)
if err != nil {
t.Fatal(err)
}
updateCerts("new-public.crt", "new-private.key")
defer updateCerts("original-public.crt", "original-private.key")
// Wait for the write event..
time.Sleep(200 * time.Millisecond)
hello := &tls.ClientHelloInfo{}
gcert, err := c.GetCertificate(hello)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(gcert.Certificate, expectedCert.Certificate) {
t.Error("certificate doesn't match expected certificate")
}
rInfo := &tls.CertificateRequestInfo{}
gcert, err = c.GetClientCertificate(rInfo)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(gcert.Certificate, expectedCert.Certificate) {
t.Error("client certificate doesn't match expected certificate")
}
}

32
pkg/certs/event.go
View File

@@ -1,32 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package certs
import (
"github.com/rjeczalik/notify"
)
// isWriteEvent checks if the event returned is a write event
func isWriteEvent(event notify.Event) bool {
for _, ev := range eventWrite {
if event&ev != 0 {
return true
}
}
return false
}

27
pkg/certs/event_linux.go
View File

@@ -1,27 +0,0 @@
// +build linux
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package certs
import "github.com/rjeczalik/notify"
var (
// eventWrite contains the notify events that will cause a write
eventWrite = []notify.Event{notify.InCloseWrite}
)

27
pkg/certs/event_others.go
View File

@@ -1,27 +0,0 @@
// +build !linux
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package certs
import "github.com/rjeczalik/notify"
var (
// eventWrite contains the notify events that will cause a write
eventWrite = []notify.Event{notify.Create, notify.Write}
)

28
pkg/certs/new-private.key
View File

@@ -1,28 +0,0 @@
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQD4tnz56oMfKgK3
scZAiFkr5lqAPFfMnrtrZXzyz+scHfOGLV3qFrqrmxyWOYI2qPbnR+NJakNDiGyx
+rI0OZj8hRqmGdP14wE9lfpFdlUVca2lRQW8aKTmF53aDuPkmHL6W7PlAef3NHoT
uuSpM+XdYlnZ97gsR+KPr9vNnYIxQWsrK3+6qfiVpOWbgM77COg5QYAeZoduSmSF
hNv+ptbNXSoyHRnLiBC4VhwFVsBNx0CNXhtE2Qd3QIsSR3cD1p7llp46uDYeQ9Vy
KfGDWWfmMF7YsGoMWoqRa53ObqN43D6gTiUorEN8cfwAPTw3yoVt+zVjFkCMvfAq
lxhXDanNAgMBAAECggEBAIGAI5rNbPCxIzEas6uuUx/0lXLn+J9mlxfYhDK56CV/
wuk+fgQBSblIzp252/8yAz1xxPrZBaUIR/B0JI3k36+8bp/GGwOQ63hxuxqn/q1n
v46qXc44foQAEAUWc7r3Vgbd8NFxKKMjA916Fs2zZCDdsQM5ZQBJfcJrQvvQ45VY
//UtXdNeIBQOb5Wg4o9fHJolKzCHWRaD2ExoIHZ5Fa6JpBmk9JBHcUbrHrlbOeep
/SkbSa0ma9j3k3jqV970XRoQUCJf+K1Li49jmaYPPGXBUAp6AfU+yiAJ1aups38m
BClLAV9g6vgE3xK2xozGPI1+j9lkruYbvGbPNkXexdECgYEA/47XnKITSnxtV+NK
nDbWNOgpeaRbxAdjp1P0b4VI0S0SuRvKUOCp1UlPg5BjGL0JLPQpGlPzEfLlGWAa
68vhyj0V6HL2+PAJNib1eu6yyRBsSbPdrAD5nydHpbxRcdShhVwb2MHMyBeYH5Al
kL+ed5wCF32kXOOGzhoGzJEKNEcCgYEA+SSdcdbuVpQFkAecIoABwdx/qeOAeS19
FsvVSTmWlhal8m2Mn8RWZ0IKXT9AoZJ0KQBIKHViPtyV7UQey05uRgLRHZapHpe8
dhm6SsGYtU3BhLdHJBP0kI79qm2kzqsHp6ghSzaxT9CkRfMniN+TD+w8p7lrOaxv
vV46UHoGX0sCgYB4LlCvVHkF+mXhgv4/YHpz/woiLm0JTwBKXG0DVQbdd/jqHGuU
hVLY/tTp5ij0JVH/VgNOYlRZCIU83blLUmIonXmECyyh/SAX21JuMXram2KRdoi0
rvC1K9/BzUHv6jLbaGmgEeOf5Zign0VLQRHg5fkF2wxEsqtemVbBNSQ7WQKBgBFk
Y/VRervig2zlixnBc93zpZnXft12tnfD7PS6p298z0LYMOvqSdnVe2G9C6b70U4X
bfIdF6mpvnGcwsWQiRQsGCsHnHC9SPO5og6b6ywk7HB2VuoG1pjM0pp2Iv4mZFdo
3kIg5EndF8qmSck9SkffRvCyefDBv98pV8rMaet3AoGBALjlN2hLoNE5Cs5vTYH8
W0AN4lEOaTlBRKG8a1h7Fm2vPgzGGkiwU6bVzsh0oTfytc8v8MW9lNQZpE3dBKne
ms3FrNsnBbTczX+xJmndRnVRocdyON6u476VxAuz/dHSFFnZGXX+2lJse9xnWHUz
OpSHUPq3TrUzhgZClE2ZKpNm
-----END PRIVATE KEY-----

21
pkg/certs/new-public.crt
View File

@@ -1,21 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIDYDCCAkigAwIBAgIJALIHkFXjtZ2yMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV
BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
aWRnaXRzIFB0eSBMdGQwHhcNMTgwNTIwMDg1MzI3WhcNMTkwNTIwMDg1MzI3WjBF
MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50
ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
CgKCAQEA+LZ8+eqDHyoCt7HGQIhZK+ZagDxXzJ67a2V88s/rHB3zhi1d6ha6q5sc
ljmCNqj250fjSWpDQ4hssfqyNDmY/IUaphnT9eMBPZX6RXZVFXGtpUUFvGik5hed
2g7j5Jhy+luz5QHn9zR6E7rkqTPl3WJZ2fe4LEfij6/bzZ2CMUFrKyt/uqn4laTl
m4DO+wjoOUGAHmaHbkpkhYTb/qbWzV0qMh0Zy4gQuFYcBVbATcdAjV4bRNkHd0CL
Ekd3A9ae5ZaeOrg2HkPVcinxg1ln5jBe2LBqDFqKkWudzm6jeNw+oE4lKKxDfHH8
AD08N8qFbfs1YxZAjL3wKpcYVw2pzQIDAQABo1MwUTAdBgNVHQ4EFgQU2Yywgv8p
WfyZxYVx+MnH+VQ5TTUwHwYDVR0jBBgwFoAU2Yywgv8pWfyZxYVx+MnH+VQ5TTUw
DwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEA2maF7DQ7CMpCho9B
9gjGxvt8HqY1pCyuQwcSPb4PTyoKUZ/ZuIDhVOaBX+ox1RzlfGtYs2BUM63/QUDs
dP0GO7/IL/XEqJi1flrFvM7LNSs89qAbPJ440m6jJDzsuL2VeyUX/M72IEsBK2uS
ajtS1+HFQjPMvt7wR6fDPCP7wHPOrkTN4hcHlgzVJShKUnFaHtb2lOnWaoM/Sk91
IsiyAhKRuCM9et7/bnOj7G8448QDVtQNniT8V/HpqQ7ltSuIGvs3QYTLDTege/74
Q8Ph1oH7shyRE/PqPfyIuLq3p0N9Sah3oRMHLohYjJL0zAGt0jxSsnhrBSNUUD/v
bAd5VQ==
-----END CERTIFICATE-----

28
pkg/certs/original-private.key
View File

@@ -1,28 +0,0 @@
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDPszxaYwn+mIz6
IGuUlmvWwUs/yWTH4MC17qey2N5MqcxlfIWHUugcBsbGhi/e1druFW0s7YGMxp+G
+Q1IezxX+VmVaJCN8AgSowbYgpRdpRQ+mhGeQby0JcvO16fyPnUJBz3GGel2bcK8
fcQyT0TVapCiD9oURVmdvDSsRXz+EoPlOve8AWciHHgm1ItO5qdPRP5YtcJfLiwK
noYnpda2d9SzmYk+Q2JFArooF7/A1DYz9bXCMo3qp0gQlMpSMDR+MCbxHBzBBr+f
QG8QdDrzWQ2slhniBhFDk0LuPCBLlSeIzkp+DoAGDXf3hWYhechlabZ7nfngg5er
Ez776WCFAgMBAAECggEBAJcHRyCWmcLm3MRY5MF0K9BKV9R3NnBdTuQ8OPdE2Ui3
w6gcRuBi+eK/TrU3CAIqUXsEW5Hq1mQuXfwAh5cn/XYfG/QXx91eKBCdOTIgqY/6
pODsmVkRhg0c2rl6eWYd4m6BNHsjhm8WWx9C+HJ4z528UpV1n2dUElkvbMHD+aKp
Ndwd0W+0PCn/BjMn/sdyy01f8sfaK2Zoy7HBw/fGeBDNLFFj3Iz7BqXYeS+OyfLN
B4xD5I5fFqt1iJeyqVPzGkOAYSqisijbM1GtZJCeVp37/+IDylCKTO3l8Xd8x73U
qTYcYT3heSHyUC2xCM6Va2YkSrOHeqbq91QgHh9LVrUCgYEA9t/wE2S8TE2l1IG9
68SXdhyaXTnB2qSL7ggY0uazPzBNLQpNMOxicZ6/4QGEi3hSuCqGxxGo9UEoTsVd
pk8oIeDULdPVi4NQxSmkxUyArs/dzOMygUPyosOiEc8z6jWFFKDcQ7mnZnay8dZ4
e4j+/hZDONtDrJ+zH2xu98ZrJPcCgYEA12CbSRbCkTiRj/dq8Qvgp6+ceTVcAbnk
MWpAhZQaXHrG3XP0L7QTIHG/7a09Mln92zjuAFXDp/Vc5NdxeXcnj9j6oUAxq+0I
dq+vibzjROemmvnmQvXGY9tc0ns6u7GjM0+Sicmas+IH4vuum/aRasABfVe2XBwe
4fVs0n7yU2MCgYA7KevFGg0uVCV7yiQTzqdlvPEZim/00B5gyzv3vyYR7KdyNdfN
87ib9imR6OU0738Td82ZA5h0PktEpXQOGUZK6DCxUuUIbE39Ej/UsMLeIh7LrV87
L2eErlG25utQI8di7DIdYO7HVYcJAhcZs/k4N2mgxJtxUUyCKWBmrPycfQKBgAo7
0uUUKcaQs4ntra0qbVBKbdrsiCSk2ozmiY5PTTlbtBtNqSqjGc2O2hnHA4Ni90b1
W4m0iYlvhSxyeDfXS4/wNWh4DmQm7SIGkwaubPYXM7llamWAHB8eiziNFmtYs3J6
s3HMnIczlEBayR8sBhjWaruz8TxLMcR2zubplUYVAoGBAItxeC9IT8BGJoZB++qM
f2LXCqJ383x0sDHhwPMFPtwUTzAwc5BJgQe9zFktW5CBxsER+MnUZjlrarT1HQfH
1Y1mJQXtwuBKG4pPPZphH0yoVlYcWkBTMw/KmlVlwRclEzRQwV3TPD+i6ieKeZhz
9eZwhS3H+Zb/693WbBDyH8L+
-----END PRIVATE KEY-----

22
pkg/certs/original-public.crt
View File

@@ -1,22 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIDqjCCApKgAwIBAgIJAOcv4FsrflS4MA0GCSqGSIb3DQEBCwUAMGoxCzAJBgNV
BAYTAlVTMQswCQYDVQQIDAJDQTEVMBMGA1UEBwwMUmVkd29vZCBDaXR5MQ4wDAYD
VQQKDAVNaW5pbzEUMBIGA1UECwwLRW5naW5lZXJpbmcxETAPBgNVBAMMCG1pbmlv
LmlvMB4XDTE4MDUyMDA4NDc0MFoXDTE5MDUyMDA4NDc0MFowajELMAkGA1UEBhMC
VVMxCzAJBgNVBAgMAkNBMRUwEwYDVQQHDAxSZWR3b29kIENpdHkxDjAMBgNVBAoM
BU1pbmlvMRQwEgYDVQQLDAtFbmdpbmVlcmluZzERMA8GA1UEAwwIbWluaW8uaW8w
ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDPszxaYwn+mIz6IGuUlmvW
wUs/yWTH4MC17qey2N5MqcxlfIWHUugcBsbGhi/e1druFW0s7YGMxp+G+Q1IezxX
+VmVaJCN8AgSowbYgpRdpRQ+mhGeQby0JcvO16fyPnUJBz3GGel2bcK8fcQyT0TV
apCiD9oURVmdvDSsRXz+EoPlOve8AWciHHgm1ItO5qdPRP5YtcJfLiwKnoYnpda2
d9SzmYk+Q2JFArooF7/A1DYz9bXCMo3qp0gQlMpSMDR+MCbxHBzBBr+fQG8QdDrz
WQ2slhniBhFDk0LuPCBLlSeIzkp+DoAGDXf3hWYhechlabZ7nfngg5erEz776WCF
AgMBAAGjUzBRMB0GA1UdDgQWBBRzC09a+3AlbFDg6BsvELolmO8jYjAfBgNVHSME
GDAWgBRzC09a+3AlbFDg6BsvELolmO8jYjAPBgNVHRMBAf8EBTADAQH/MA0GCSqG
SIb3DQEBCwUAA4IBAQBl0cx7qbidKjhoZ1Iv4pCD8xHZgtuWEDApPoGuMtVS66jJ
+oj0ncD5xCtv9XqXtshE65FIsEWnDOIwa+kyjMnxHbFwxveWBT4W0twtqwbVs7NE
I0So6cEmSx4+rB0XorY6mIbD3O9YAStelNhB1jVfQfIMSByYkcGq2Fh+B1LHlOrz
06LJdwYMiILzK0c5fvjZvsDq/9EK+Xo66hphKjs5cl1t9WK7wKOCoZDt2lOTZqEq
UWYGPWlTAxSWQxO4WnvSKqFdsRi8fOO3KlDq1eNqeDSGGCI0DTGgJxidHIpfOPEF
s/zojgc5npE32/1n8og6gLcv7LIKelBfMhUrFTp7
-----END CERTIFICATE-----

28
pkg/certs/private.key
View File

@@ -1,28 +0,0 @@
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDPszxaYwn+mIz6
IGuUlmvWwUs/yWTH4MC17qey2N5MqcxlfIWHUugcBsbGhi/e1druFW0s7YGMxp+G
+Q1IezxX+VmVaJCN8AgSowbYgpRdpRQ+mhGeQby0JcvO16fyPnUJBz3GGel2bcK8
fcQyT0TVapCiD9oURVmdvDSsRXz+EoPlOve8AWciHHgm1ItO5qdPRP5YtcJfLiwK
noYnpda2d9SzmYk+Q2JFArooF7/A1DYz9bXCMo3qp0gQlMpSMDR+MCbxHBzBBr+f
QG8QdDrzWQ2slhniBhFDk0LuPCBLlSeIzkp+DoAGDXf3hWYhechlabZ7nfngg5er
Ez776WCFAgMBAAECggEBAJcHRyCWmcLm3MRY5MF0K9BKV9R3NnBdTuQ8OPdE2Ui3
w6gcRuBi+eK/TrU3CAIqUXsEW5Hq1mQuXfwAh5cn/XYfG/QXx91eKBCdOTIgqY/6
pODsmVkRhg0c2rl6eWYd4m6BNHsjhm8WWx9C+HJ4z528UpV1n2dUElkvbMHD+aKp
Ndwd0W+0PCn/BjMn/sdyy01f8sfaK2Zoy7HBw/fGeBDNLFFj3Iz7BqXYeS+OyfLN
B4xD5I5fFqt1iJeyqVPzGkOAYSqisijbM1GtZJCeVp37/+IDylCKTO3l8Xd8x73U
qTYcYT3heSHyUC2xCM6Va2YkSrOHeqbq91QgHh9LVrUCgYEA9t/wE2S8TE2l1IG9
68SXdhyaXTnB2qSL7ggY0uazPzBNLQpNMOxicZ6/4QGEi3hSuCqGxxGo9UEoTsVd
pk8oIeDULdPVi4NQxSmkxUyArs/dzOMygUPyosOiEc8z6jWFFKDcQ7mnZnay8dZ4
e4j+/hZDONtDrJ+zH2xu98ZrJPcCgYEA12CbSRbCkTiRj/dq8Qvgp6+ceTVcAbnk
MWpAhZQaXHrG3XP0L7QTIHG/7a09Mln92zjuAFXDp/Vc5NdxeXcnj9j6oUAxq+0I
dq+vibzjROemmvnmQvXGY9tc0ns6u7GjM0+Sicmas+IH4vuum/aRasABfVe2XBwe
4fVs0n7yU2MCgYA7KevFGg0uVCV7yiQTzqdlvPEZim/00B5gyzv3vyYR7KdyNdfN
87ib9imR6OU0738Td82ZA5h0PktEpXQOGUZK6DCxUuUIbE39Ej/UsMLeIh7LrV87
L2eErlG25utQI8di7DIdYO7HVYcJAhcZs/k4N2mgxJtxUUyCKWBmrPycfQKBgAo7
0uUUKcaQs4ntra0qbVBKbdrsiCSk2ozmiY5PTTlbtBtNqSqjGc2O2hnHA4Ni90b1
W4m0iYlvhSxyeDfXS4/wNWh4DmQm7SIGkwaubPYXM7llamWAHB8eiziNFmtYs3J6
s3HMnIczlEBayR8sBhjWaruz8TxLMcR2zubplUYVAoGBAItxeC9IT8BGJoZB++qM
f2LXCqJ383x0sDHhwPMFPtwUTzAwc5BJgQe9zFktW5CBxsER+MnUZjlrarT1HQfH
1Y1mJQXtwuBKG4pPPZphH0yoVlYcWkBTMw/KmlVlwRclEzRQwV3TPD+i6ieKeZhz
9eZwhS3H+Zb/693WbBDyH8L+
-----END PRIVATE KEY-----

22
pkg/certs/public.crt
View File

@@ -1,22 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIDqjCCApKgAwIBAgIJAOcv4FsrflS4MA0GCSqGSIb3DQEBCwUAMGoxCzAJBgNV
BAYTAlVTMQswCQYDVQQIDAJDQTEVMBMGA1UEBwwMUmVkd29vZCBDaXR5MQ4wDAYD
VQQKDAVNaW5pbzEUMBIGA1UECwwLRW5naW5lZXJpbmcxETAPBgNVBAMMCG1pbmlv
LmlvMB4XDTE4MDUyMDA4NDc0MFoXDTE5MDUyMDA4NDc0MFowajELMAkGA1UEBhMC
VVMxCzAJBgNVBAgMAkNBMRUwEwYDVQQHDAxSZWR3b29kIENpdHkxDjAMBgNVBAoM
BU1pbmlvMRQwEgYDVQQLDAtFbmdpbmVlcmluZzERMA8GA1UEAwwIbWluaW8uaW8w
ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDPszxaYwn+mIz6IGuUlmvW
wUs/yWTH4MC17qey2N5MqcxlfIWHUugcBsbGhi/e1druFW0s7YGMxp+G+Q1IezxX
+VmVaJCN8AgSowbYgpRdpRQ+mhGeQby0JcvO16fyPnUJBz3GGel2bcK8fcQyT0TV
apCiD9oURVmdvDSsRXz+EoPlOve8AWciHHgm1ItO5qdPRP5YtcJfLiwKnoYnpda2
d9SzmYk+Q2JFArooF7/A1DYz9bXCMo3qp0gQlMpSMDR+MCbxHBzBBr+fQG8QdDrz
WQ2slhniBhFDk0LuPCBLlSeIzkp+DoAGDXf3hWYhechlabZ7nfngg5erEz776WCF
AgMBAAGjUzBRMB0GA1UdDgQWBBRzC09a+3AlbFDg6BsvELolmO8jYjAfBgNVHSME
GDAWgBRzC09a+3AlbFDg6BsvELolmO8jYjAPBgNVHRMBAf8EBTADAQH/MA0GCSqG
SIb3DQEBCwUAA4IBAQBl0cx7qbidKjhoZ1Iv4pCD8xHZgtuWEDApPoGuMtVS66jJ
+oj0ncD5xCtv9XqXtshE65FIsEWnDOIwa+kyjMnxHbFwxveWBT4W0twtqwbVs7NE
I0So6cEmSx4+rB0XorY6mIbD3O9YAStelNhB1jVfQfIMSByYkcGq2Fh+B1LHlOrz
06LJdwYMiILzK0c5fvjZvsDq/9EK+Xo66hphKjs5cl1t9WK7wKOCoZDt2lOTZqEq
UWYGPWlTAxSWQxO4WnvSKqFdsRi8fOO3KlDq1eNqeDSGGCI0DTGgJxidHIpfOPEF
s/zojgc5npE32/1n8og6gLcv7LIKelBfMhUrFTp7
-----END CERTIFICATE-----

22
pkg/certs/server.crt
View File

@@ -1,22 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIDqjCCApKgAwIBAgIJAOcv4FsrflS4MA0GCSqGSIb3DQEBCwUAMGoxCzAJBgNV
BAYTAlVTMQswCQYDVQQIDAJDQTEVMBMGA1UEBwwMUmVkd29vZCBDaXR5MQ4wDAYD
VQQKDAVNaW5pbzEUMBIGA1UECwwLRW5naW5lZXJpbmcxETAPBgNVBAMMCG1pbmlv
LmlvMB4XDTE4MDUyMDA4NDc0MFoXDTE5MDUyMDA4NDc0MFowajELMAkGA1UEBhMC
VVMxCzAJBgNVBAgMAkNBMRUwEwYDVQQHDAxSZWR3b29kIENpdHkxDjAMBgNVBAoM
BU1pbmlvMRQwEgYDVQQLDAtFbmdpbmVlcmluZzERMA8GA1UEAwwIbWluaW8uaW8w
ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDPszxaYwn+mIz6IGuUlmvW
wUs/yWTH4MC17qey2N5MqcxlfIWHUugcBsbGhi/e1druFW0s7YGMxp+G+Q1IezxX
+VmVaJCN8AgSowbYgpRdpRQ+mhGeQby0JcvO16fyPnUJBz3GGel2bcK8fcQyT0TV
apCiD9oURVmdvDSsRXz+EoPlOve8AWciHHgm1ItO5qdPRP5YtcJfLiwKnoYnpda2
d9SzmYk+Q2JFArooF7/A1DYz9bXCMo3qp0gQlMpSMDR+MCbxHBzBBr+fQG8QdDrz
WQ2slhniBhFDk0LuPCBLlSeIzkp+DoAGDXf3hWYhechlabZ7nfngg5erEz776WCF
AgMBAAGjUzBRMB0GA1UdDgQWBBRzC09a+3AlbFDg6BsvELolmO8jYjAfBgNVHSME
GDAWgBRzC09a+3AlbFDg6BsvELolmO8jYjAPBgNVHRMBAf8EBTADAQH/MA0GCSqG
SIb3DQEBCwUAA4IBAQBl0cx7qbidKjhoZ1Iv4pCD8xHZgtuWEDApPoGuMtVS66jJ
+oj0ncD5xCtv9XqXtshE65FIsEWnDOIwa+kyjMnxHbFwxveWBT4W0twtqwbVs7NE
I0So6cEmSx4+rB0XorY6mIbD3O9YAStelNhB1jVfQfIMSByYkcGq2Fh+B1LHlOrz
06LJdwYMiILzK0c5fvjZvsDq/9EK+Xo66hphKjs5cl1t9WK7wKOCoZDt2lOTZqEq
UWYGPWlTAxSWQxO4WnvSKqFdsRi8fOO3KlDq1eNqeDSGGCI0DTGgJxidHIpfOPEF
s/zojgc5npE32/1n8og6gLcv7LIKelBfMhUrFTp7
-----END CERTIFICATE-----

28
pkg/certs/server.key
View File

@@ -1,28 +0,0 @@
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDPszxaYwn+mIz6
IGuUlmvWwUs/yWTH4MC17qey2N5MqcxlfIWHUugcBsbGhi/e1druFW0s7YGMxp+G
+Q1IezxX+VmVaJCN8AgSowbYgpRdpRQ+mhGeQby0JcvO16fyPnUJBz3GGel2bcK8
fcQyT0TVapCiD9oURVmdvDSsRXz+EoPlOve8AWciHHgm1ItO5qdPRP5YtcJfLiwK
noYnpda2d9SzmYk+Q2JFArooF7/A1DYz9bXCMo3qp0gQlMpSMDR+MCbxHBzBBr+f
QG8QdDrzWQ2slhniBhFDk0LuPCBLlSeIzkp+DoAGDXf3hWYhechlabZ7nfngg5er
Ez776WCFAgMBAAECggEBAJcHRyCWmcLm3MRY5MF0K9BKV9R3NnBdTuQ8OPdE2Ui3
w6gcRuBi+eK/TrU3CAIqUXsEW5Hq1mQuXfwAh5cn/XYfG/QXx91eKBCdOTIgqY/6
pODsmVkRhg0c2rl6eWYd4m6BNHsjhm8WWx9C+HJ4z528UpV1n2dUElkvbMHD+aKp
Ndwd0W+0PCn/BjMn/sdyy01f8sfaK2Zoy7HBw/fGeBDNLFFj3Iz7BqXYeS+OyfLN
B4xD5I5fFqt1iJeyqVPzGkOAYSqisijbM1GtZJCeVp37/+IDylCKTO3l8Xd8x73U
qTYcYT3heSHyUC2xCM6Va2YkSrOHeqbq91QgHh9LVrUCgYEA9t/wE2S8TE2l1IG9
68SXdhyaXTnB2qSL7ggY0uazPzBNLQpNMOxicZ6/4QGEi3hSuCqGxxGo9UEoTsVd
pk8oIeDULdPVi4NQxSmkxUyArs/dzOMygUPyosOiEc8z6jWFFKDcQ7mnZnay8dZ4
e4j+/hZDONtDrJ+zH2xu98ZrJPcCgYEA12CbSRbCkTiRj/dq8Qvgp6+ceTVcAbnk
MWpAhZQaXHrG3XP0L7QTIHG/7a09Mln92zjuAFXDp/Vc5NdxeXcnj9j6oUAxq+0I
dq+vibzjROemmvnmQvXGY9tc0ns6u7GjM0+Sicmas+IH4vuum/aRasABfVe2XBwe
4fVs0n7yU2MCgYA7KevFGg0uVCV7yiQTzqdlvPEZim/00B5gyzv3vyYR7KdyNdfN
87ib9imR6OU0738Td82ZA5h0PktEpXQOGUZK6DCxUuUIbE39Ej/UsMLeIh7LrV87
L2eErlG25utQI8di7DIdYO7HVYcJAhcZs/k4N2mgxJtxUUyCKWBmrPycfQKBgAo7
0uUUKcaQs4ntra0qbVBKbdrsiCSk2ozmiY5PTTlbtBtNqSqjGc2O2hnHA4Ni90b1
W4m0iYlvhSxyeDfXS4/wNWh4DmQm7SIGkwaubPYXM7llamWAHB8eiziNFmtYs3J6
s3HMnIczlEBayR8sBhjWaruz8TxLMcR2zubplUYVAoGBAItxeC9IT8BGJoZB++qM
f2LXCqJ383x0sDHhwPMFPtwUTzAwc5BJgQe9zFktW5CBxsER+MnUZjlrarT1HQfH
1Y1mJQXtwuBKG4pPPZphH0yoVlYcWkBTMw/KmlVlwRclEzRQwV3TPD+i6ieKeZhz
9eZwhS3H+Zb/693WbBDyH8L+
-----END PRIVATE KEY-----

178
pkg/cgroup/linux.go
View File

@@ -1,178 +0,0 @@
// +build linux
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Package cgroup implements parsing for all the cgroup
// categories and functionality in a simple way.
package cgroup
import (
"bufio"
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
)
// DO NOT EDIT following constants are chosen defaults for any kernel
// after 3.x, please open a GitHub issue https://github.com/minio/minio/issues
// and discuss first if you wish to change this.
const (
// Default string for looking for kernel memory param.
memoryLimitKernelParam = "memory.limit_in_bytes"
// Points to sys path memory path.
cgroupMemSysPath = "/sys/fs/cgroup/memory"
// Default docker prefix.
dockerPrefixName = "/docker/"
// Proc controller group path.
cgroupFileTemplate = "/proc/%d/cgroup"
)
// CGEntries - represents all the entries in a process cgroup file
// at /proc/<pid>/cgroup as key value pairs.
type CGEntries map[string]string
// GetEntries reads and parses all the cgroup entries for a given process.
func GetEntries(pid int) (CGEntries, error) {
r, err := os.Open(fmt.Sprintf(cgroupFileTemplate, pid))
if err != nil {
return nil, err
}
defer r.Close()
return parseProcCGroup(r)
}
// parseProcCGroup - cgroups are always in the following
// format once enabled you need to know the pid of the
// application you are looking for so that the the
// following parsing logic only parses the file located
// at /proc/<pid>/cgroup.
//
// CGROUP entries id, component and path are always in
// the following format. ``ID:COMPONENT:PATH``
//
// Following code block parses this information and
// returns a procCGroup which is a parsed list of all
// the line by line entires from /proc/<pid>/cgroup.
func parseProcCGroup(r io.Reader) (CGEntries, error) {
var cgEntries = CGEntries{}
// Start reading cgroup categories line by line
// and process them into procCGroup structure.
scanner := bufio.NewScanner(r)
for scanner.Scan() {
line := scanner.Text()
tokens := strings.SplitN(line, ":", 3)
if len(tokens) < 3 {
continue
}
name, path := tokens[1], tokens[2]
for _, token := range strings.Split(name, ",") {
name = strings.TrimPrefix(token, "name=")
cgEntries[name] = path
}
}
// Return upon any error while reading the cgroup categories.
if err := scanner.Err(); err != nil {
return nil, err
}
return cgEntries, nil
}
// Fetch value of the cgroup kernel param from the cgroup manager,
// if cgroup manager is configured we should just rely on `cgm` cli
// to fetch all the values for us.
func getManagerKernValue(cname, path, kernParam string) (limit uint64, err error) {
cmd := exec.Command("cgm", "getvalue", cname, path, kernParam)
var out bytes.Buffer
cmd.Stdout = &out
if err = cmd.Run(); err != nil {
return 0, err
}
// Parse the cgm output.
limit, err = strconv.ParseUint(strings.TrimSpace(out.String()), 10, 64)
return limit, err
}
// Get cgroup memory limit file path.
func getMemoryLimitFilePath(cgPath string) string {
path := cgroupMemSysPath
// Docker generates weird cgroup paths that don't
// really exist on the file system.
//
// For example on regular Linux OS :
// `/user.slice/user-1000.slice/session-1.scope`
//
// But they exist as a bind mount on Docker and
// are not accessible : `/docker/<hash>`
//
// We we will just ignore if there is `/docker` in the
// path ignore and fall back to :
// `/sys/fs/cgroup/memory/memory.limit_in_bytes`
if !strings.HasPrefix(cgPath, dockerPrefixName) {
path = filepath.Join(path, cgPath)
}
// Final path.
return filepath.Join(path, memoryLimitKernelParam)
}
// GetMemoryLimit - Fetches cgroup memory limit either from
// a file path at '/sys/fs/cgroup/memory', if path fails then
// fallback to querying cgroup manager.
func GetMemoryLimit(pid int) (limit uint64, err error) {
var cg CGEntries
cg, err = GetEntries(pid)
if err != nil {
return 0, err
}
path := cg["memory"]
limit, err = getManagerKernValue("memory", path, memoryLimitKernelParam)
if err != nil {
// Upon any failure returned from `cgm`, on some systems cgm
// might not be installed. We fallback to using the the sysfs
// path instead to lookup memory limits.
var b []byte
b, err = ioutil.ReadFile(getMemoryLimitFilePath(path))
if err != nil {
return 0, err
}
limit, err = strconv.ParseUint(strings.TrimSpace(string(b)), 10, 64)
}
return limit, err
}

141
pkg/cgroup/linux_test.go
View File

@@ -1,141 +0,0 @@
// +build linux
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cgroup
import (
"io/ioutil"
"os"
"testing"
)
// Testing parsing correctness for various process cgroup files.
func TestProcCGroup(t *testing.T) {
tmpPath, err := ioutil.TempFile("", "cgroup")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpPath.Name())
cgroup := `
11:memory:/user.slice
10:blkio:/user.slice
9:hugetlb:/
8:net_cls,net_prio:/
7:perf_event:/
6:pids:/user.slice/user-1000.slice
5:devices:/user.slice
4:cpuset:/
3:cpu,cpuacct:/user.slice
2:freezer:/
1:name=systemd:/user.slice/user-1000.slice/session-1.scope
`
_, err = tmpPath.WriteString(cgroup)
if err != nil {
t.Fatal(err)
}
// Seek back to read from the beginning.
tmpPath.Seek(0, 0)
cg, err := parseProcCGroup(tmpPath)
if err != nil {
t.Fatal(err)
}
path := cg["memory"]
if len(path) == 0 {
t.Fatal("Path component cannot be empty")
}
if path != "/user.slice" {
t.Fatal("Path component cannot be empty")
}
path = cg["systemd"]
if path != "/user.slice/user-1000.slice/session-1.scope" {
t.Fatal("Path component cannot be empty")
}
// Mixed cgroups with different group names.
cgroup = `
11:memory:/newtest/newtest
10:blkio:/user.slice
9:hugetlb:/
8:net_cls,net_prio:/
7:perf_event:/
6:pids:/user.slice/user-1000.slice
5:devices:/user.slice
4:cpuset:/
3:cpu,cpuacct:/newtest/newtest
2:freezer:/
1:name=systemd:/user.slice/user-1000.slice/session-1.scope
`
// Seek back to read from the beginning.
tmpPath.Seek(0, 0)
_, err = tmpPath.WriteString(cgroup)
if err != nil {
t.Fatal(err)
}
// Seek back to read from the beginning.
tmpPath.Seek(0, 0)
cg, err = parseProcCGroup(tmpPath)
if err != nil {
t.Fatal(err)
}
path = cg["memory"]
if path != "/newtest/newtest" {
t.Fatal("Path component cannot be empty")
}
path = cg["systemd"]
if path != "/user.slice/user-1000.slice/session-1.scope" {
t.Fatal("Path component cannot be empty")
}
}
// Tests cgroup memory limit path construction.
func TestMemoryLimitPath(t *testing.T) {
testCases := []struct {
cgroupPath string
expectedPath string
}{
{
cgroupPath: "/user.slice",
expectedPath: "/sys/fs/cgroup/memory/user.slice/memory.limit_in_bytes",
},
{
cgroupPath: "/docker/testing",
expectedPath: "/sys/fs/cgroup/memory/memory.limit_in_bytes",
},
}
for i, testCase := range testCases {
actualPath := getMemoryLimitFilePath(testCase.cgroupPath)
if actualPath != testCase.expectedPath {
t.Fatalf("Test: %d: Expected: %s, got %s", i+1, testCase.expectedPath, actualPath)
}
}
}

20
pkg/cgroup/others.go
View File

@@ -1,20 +0,0 @@
// +build !linux
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cgroup

436
pkg/console/console.go
View File

@@ -1,436 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Package console implements console printing helpers
package console
import (
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"github.com/fatih/color"
"github.com/mattn/go-colorable"
"github.com/mattn/go-isatty"
)
var (
// Used by the caller to print multiple lines atomically. Exposed by Lock/Unlock methods.
publicMutex sync.Mutex
// Used internally by console.
privateMutex sync.Mutex
stderrColoredOutput = colorable.NewColorableStderr()
// Print prints a message.
Print = func(data ...interface{}) {
consolePrint("Print", Theme["Print"], data...)
}
// PrintC prints a message with color.
PrintC = func(data ...interface{}) {
consolePrint("PrintC", Theme["PrintC"], data...)
}
// Printf prints a formatted message.
Printf = func(format string, data ...interface{}) {
consolePrintf("Print", Theme["Print"], format, data...)
}
// Println prints a message with a newline.
Println = func(data ...interface{}) {
consolePrintln("Print", Theme["Print"], data...)
}
// Fatal print a error message and exit.
Fatal = func(data ...interface{}) {
consolePrint("Fatal", Theme["Fatal"], data...)
os.Exit(1)
}
// Fatalf print a error message with a format specified and exit.
Fatalf = func(format string, data ...interface{}) {
consolePrintf("Fatal", Theme["Fatal"], format, data...)
os.Exit(1)
}
// Fatalln print a error message with a new line and exit.
Fatalln = func(data ...interface{}) {
consolePrintln("Fatal", Theme["Fatal"], data...)
os.Exit(1)
}
// Error prints a error message.
Error = func(data ...interface{}) {
consolePrint("Error", Theme["Error"], data...)
}
// Errorf print a error message with a format specified.
Errorf = func(format string, data ...interface{}) {
consolePrintf("Error", Theme["Error"], format, data...)
}
// Errorln prints a error message with a new line.
Errorln = func(data ...interface{}) {
consolePrintln("Error", Theme["Error"], data...)
}
// Info prints a informational message.
Info = func(data ...interface{}) {
consolePrint("Info", Theme["Info"], data...)
}
// Infof prints a informational message in custom format.
Infof = func(format string, data ...interface{}) {
consolePrintf("Info", Theme["Info"], format, data...)
}
// Infoln prints a informational message with a new line.
Infoln = func(data ...interface{}) {
consolePrintln("Info", Theme["Info"], data...)
}
// Debug prints a debug message without a new line
// Debug prints a debug message.
Debug = func(data ...interface{}) {
consolePrint("Debug", Theme["Debug"], data...)
}
// Debugf prints a debug message with a new line.
Debugf = func(format string, data ...interface{}) {
consolePrintf("Debug", Theme["Debug"], format, data...)
}
// Debugln prints a debug message with a new line.
Debugln = func(data ...interface{}) {
consolePrintln("Debug", Theme["Debug"], data...)
}
// Colorize prints message in a colorized form, dictated by the corresponding tag argument.
Colorize = func(tag string, data interface{}) string {
if isatty.IsTerminal(os.Stdout.Fd()) {
colorized, ok := Theme[tag]
if ok {
return colorized.SprintFunc()(data)
} // else: No theme found. Return as string.
}
return fmt.Sprint(data)
}
// Eraseline Print in new line and adjust to top so that we don't print over the ongoing progress bar.
Eraseline = func() {
consolePrintf("Print", Theme["Print"], "%c[2K\n", 27)
consolePrintf("Print", Theme["Print"], "%c[A", 27)
}
)
// wrap around standard fmt functions.
// consolePrint prints a message prefixed with message type and program name.
func consolePrint(tag string, c *color.Color, a ...interface{}) {
privateMutex.Lock()
defer privateMutex.Unlock()
switch tag {
case "Debug":
// if no arguments are given do not invoke debug printer.
if len(a) == 0 {
return
}
output := color.Output
color.Output = stderrColoredOutput
if isatty.IsTerminal(os.Stderr.Fd()) {
c.Print(ProgramName() + ": <DEBUG> ")
c.Print(a...)
} else {
fmt.Fprint(color.Output, ProgramName()+": <DEBUG> ")
fmt.Fprint(color.Output, a...)
}
color.Output = output
case "Fatal":
fallthrough
case "Error":
// if no arguments are given do not invoke fatal and error printer.
if len(a) == 0 {
return
}
output := color.Output
color.Output = stderrColoredOutput
if isatty.IsTerminal(os.Stderr.Fd()) {
c.Print(ProgramName() + ": <ERROR> ")
c.Print(a...)
} else {
fmt.Fprint(color.Output, ProgramName()+": <ERROR> ")
fmt.Fprint(color.Output, a...)
}
color.Output = output
case "Info":
// if no arguments are given do not invoke info printer.
if len(a) == 0 {
return
}
if isatty.IsTerminal(os.Stdout.Fd()) {
c.Print(ProgramName() + ": ")
c.Print(a...)
} else {
fmt.Fprint(color.Output, ProgramName()+": ")
fmt.Fprint(color.Output, a...)
}
default:
if isatty.IsTerminal(os.Stdout.Fd()) {
c.Print(a...)
} else {
fmt.Fprint(color.Output, a...)
}
}
}
// consolePrintf - same as print with a new line.
func consolePrintf(tag string, c *color.Color, format string, a ...interface{}) {
privateMutex.Lock()
defer privateMutex.Unlock()
switch tag {
case "Debug":
// if no arguments are given do not invoke debug printer.
if len(a) == 0 {
return
}
output := color.Output
color.Output = stderrColoredOutput
if isatty.IsTerminal(os.Stderr.Fd()) {
c.Print(ProgramName() + ": <DEBUG> ")
c.Printf(format, a...)
} else {
fmt.Fprint(color.Output, ProgramName()+": <DEBUG> ")
fmt.Fprintf(color.Output, format, a...)
}
color.Output = output
case "Fatal":
fallthrough
case "Error":
// if no arguments are given do not invoke fatal and error printer.
if len(a) == 0 {
return
}
output := color.Output
color.Output = stderrColoredOutput
if isatty.IsTerminal(os.Stderr.Fd()) {
c.Print(ProgramName() + ": <ERROR> ")
c.Printf(format, a...)
} else {
fmt.Fprint(color.Output, ProgramName()+": <ERROR> ")
fmt.Fprintf(color.Output, format, a...)
}
color.Output = output
case "Info":
// if no arguments are given do not invoke info printer.
if len(a) == 0 {
return
}
if isatty.IsTerminal(os.Stdout.Fd()) {
c.Print(ProgramName() + ": ")
c.Printf(format, a...)
} else {
fmt.Fprint(color.Output, ProgramName()+": ")
fmt.Fprintf(color.Output, format, a...)
}
default:
if isatty.IsTerminal(os.Stdout.Fd()) {
c.Printf(format, a...)
} else {
fmt.Fprintf(color.Output, format, a...)
}
}
}
// consolePrintln - same as print with a new line.
func consolePrintln(tag string, c *color.Color, a ...interface{}) {
privateMutex.Lock()
defer privateMutex.Unlock()
switch tag {
case "Debug":
// if no arguments are given do not invoke debug printer.
if len(a) == 0 {
return
}
output := color.Output
color.Output = stderrColoredOutput
if isatty.IsTerminal(os.Stderr.Fd()) {
c.Print(ProgramName() + ": <DEBUG> ")
c.Println(a...)
} else {
fmt.Fprint(color.Output, ProgramName()+": <DEBUG> ")
fmt.Fprintln(color.Output, a...)
}
color.Output = output
case "Fatal":
fallthrough
case "Error":
// if no arguments are given do not invoke fatal and error printer.
if len(a) == 0 {
return
}
output := color.Output
color.Output = stderrColoredOutput
if isatty.IsTerminal(os.Stderr.Fd()) {
c.Print(ProgramName() + ": <ERROR> ")
c.Println(a...)
} else {
fmt.Fprint(color.Output, ProgramName()+": <ERROR> ")
fmt.Fprintln(color.Output, a...)
}
color.Output = output
case "Info":
// if no arguments are given do not invoke info printer.
if len(a) == 0 {
return
}
if isatty.IsTerminal(os.Stdout.Fd()) {
c.Print(ProgramName() + ": ")
c.Println(a...)
} else {
fmt.Fprint(color.Output, ProgramName()+": ")
fmt.Fprintln(color.Output, a...)
}
default:
if isatty.IsTerminal(os.Stdout.Fd()) {
c.Println(a...)
} else {
fmt.Fprintln(color.Output, a...)
}
}
}
// Lock console.
func Lock() {
publicMutex.Lock()
}
// Unlock locked console.
func Unlock() {
publicMutex.Unlock()
}
// ProgramName - return the name of the executable program.
func ProgramName() string {
_, progName := filepath.Split(os.Args[0])
return progName
}
// Table - data to print in table format with fixed row widths.
type Table struct {
// per-row colors
RowColors []*color.Color
// per-column align-right flag (aligns left by default)
AlignRight []bool
// Left margin width for table
TableIndentWidth int
// Flag to print separator under heading. Row 0 is considered heading
HeaderRowSeparator bool
}
// NewTable - create a new Table instance. Takes per-row colors and
// per-column right-align flags and table indentation width (i.e. left
// margin width)
func NewTable(rowColors []*color.Color, alignRight []bool, indentWidth int) *Table {
return &Table{rowColors, alignRight, indentWidth, false}
}
// DisplayTable - prints the table
func (t *Table) DisplayTable(rows [][]string) error {
numRows := len(rows)
numCols := len(rows[0])
if numRows != len(t.RowColors) {
return fmt.Errorf("row count and row-colors mismatch")
}
// Compute max. column widths
maxColWidths := make([]int, numCols)
for _, row := range rows {
if len(row) != len(t.AlignRight) {
return fmt.Errorf("col count and align-right mismatch")
}
for i, v := range row {
if len([]rune(v)) > maxColWidths[i] {
maxColWidths[i] = len([]rune(v))
}
}
}
// Compute per-cell text with padding and alignment applied.
paddedText := make([][]string, numRows)
for r, row := range rows {
paddedText[r] = make([]string, numCols)
for c, cell := range row {
if t.AlignRight[c] {
fmtStr := fmt.Sprintf("%%%ds", maxColWidths[c])
paddedText[r][c] = fmt.Sprintf(fmtStr, cell)
} else {
extraWidth := maxColWidths[c] - len([]rune(cell))
fmtStr := fmt.Sprintf("%%s%%%ds", extraWidth)
paddedText[r][c] = fmt.Sprintf(fmtStr, cell, "")
}
}
}
// Draw table top border
segments := make([]string, numCols)
for i, c := range maxColWidths {
segments[i] = strings.Repeat("─", c+2)
}
indentText := strings.Repeat(" ", t.TableIndentWidth)
border := fmt.Sprintf("%s┌%s┐", indentText, strings.Join(segments, "┬"))
fmt.Println(border)
// Print the table with colors
for r, row := range paddedText {
if t.HeaderRowSeparator && r == 1 {
// Draw table header-row border
border = fmt.Sprintf("%s├%s┤", indentText, strings.Join(segments, "┼"))
fmt.Println(border)
}
fmt.Print(indentText + "│ ")
for c, text := range row {
t.RowColors[r].Print(text)
if c != numCols-1 {
fmt.Print(" │ ")
}
}
fmt.Println(" │")
}
// Draw table bottom border
border = fmt.Sprintf("%s└%s┘", indentText, strings.Join(segments, "┴"))
fmt.Println(border)
return nil
}
// RewindLines - uses terminal escape symbols to clear and rewind
// upwards on the console for `n` lines.
func RewindLines(n int) {
for i := 0; i < n; i++ {
fmt.Printf("\033[1A\033[K")
}
}

38
pkg/console/console_test.go
View File

@@ -1,38 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package console
import (
"testing"
"github.com/fatih/color"
)
func TestSetColor(t *testing.T) {
SetColor("unknown", color.New(color.FgWhite))
_, ok := Theme["unknown"]
if !ok {
t.Fatal("missing theme")
}
}
func TestColorLock(t *testing.T) {
Lock()
Print("") // Test for deadlocks.
Unlock()
}

55
pkg/console/themes.go
View File

@@ -1,55 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package console
import "github.com/fatih/color"
var (
// Theme contains default color mapping.
Theme = map[string]*color.Color{
"Debug": color.New(color.FgWhite, color.Faint, color.Italic),
"Fatal": color.New(color.FgRed, color.Italic, color.Bold),
"Error": color.New(color.FgYellow, color.Italic),
"Info": color.New(color.FgGreen, color.Bold),
"Print": color.New(),
"PrintB": color.New(color.FgBlue, color.Bold),
"PrintC": color.New(color.FgGreen, color.Bold),
}
)
// SetColorOff disables coloring for the entire session.
func SetColorOff() {
privateMutex.Lock()
defer privateMutex.Unlock()
color.NoColor = true
}
// SetColorOn enables coloring for the entire session.
func SetColorOn() {
privateMutex.Lock()
defer privateMutex.Unlock()
color.NoColor = false
}
// SetColor sets a color for a particular tag.
func SetColor(tag string, cl *color.Color) {
privateMutex.Lock()
defer privateMutex.Unlock()
// add new theme
Theme[tag] = cl
}

2
pkg/dsync/drwmutex.go
View File

@@ -25,7 +25,7 @@ import (
"sync"
"time"
"github.com/minio/minio/pkg/console"
"github.com/minio/pkg/console"
)
// Indicator if logging is enabled.

227
pkg/ellipses/ellipses.go
View File

@@ -1,227 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ellipses
import (
"errors"
"fmt"
"regexp"
"strconv"
"strings"
)
var (
// Regex to extract ellipses syntax inputs.
regexpEllipses = regexp.MustCompile(`(.*)({[0-9a-z]*\.\.\.[0-9a-z]*})(.*)`)
// Ellipses constants
openBraces = "{"
closeBraces = "}"
ellipses = "..."
)
// Parses an ellipses range pattern of following style
// `{1...64}`
// `{33...64}`
func parseEllipsesRange(pattern string) (seq []string, err error) {
if !strings.Contains(pattern, openBraces) {
return nil, errors.New("Invalid argument")
}
if !strings.Contains(pattern, closeBraces) {
return nil, errors.New("Invalid argument")
}
pattern = strings.TrimPrefix(pattern, openBraces)
pattern = strings.TrimSuffix(pattern, closeBraces)
ellipsesRange := strings.Split(pattern, ellipses)
if len(ellipsesRange) != 2 {
return nil, errors.New("Invalid argument")
}
var hexadecimal bool
var start, end uint64
if start, err = strconv.ParseUint(ellipsesRange[0], 10, 64); err != nil {
// Look for hexadecimal conversions if any.
start, err = strconv.ParseUint(ellipsesRange[0], 16, 64)
if err != nil {
return nil, err
}
hexadecimal = true
}
if end, err = strconv.ParseUint(ellipsesRange[1], 10, 64); err != nil {
// Look for hexadecimal conversions if any.
end, err = strconv.ParseUint(ellipsesRange[1], 16, 64)
if err != nil {
return nil, err
}
hexadecimal = true
}
if start > end {
return nil, fmt.Errorf("Incorrect range start %d cannot be bigger than end %d", start, end)
}
for i := start; i <= end; i++ {
if strings.HasPrefix(ellipsesRange[0], "0") && len(ellipsesRange[0]) > 1 || strings.HasPrefix(ellipsesRange[1], "0") {
if hexadecimal {
seq = append(seq, fmt.Sprintf(fmt.Sprintf("%%0%dx", len(ellipsesRange[1])), i))
} else {
seq = append(seq, fmt.Sprintf(fmt.Sprintf("%%0%dd", len(ellipsesRange[1])), i))
}
} else {
if hexadecimal {
seq = append(seq, fmt.Sprintf("%x", i))
} else {
seq = append(seq, fmt.Sprintf("%d", i))
}
}
}
return seq, nil
}
// Pattern - ellipses pattern, describes the range and also the
// associated prefix and suffixes.
type Pattern struct {
Prefix string
Suffix string
Seq []string
}
// argExpander - recursively expands labels into its respective forms.
func argExpander(labels [][]string) (out [][]string) {
if len(labels) == 1 {
for _, v := range labels[0] {
out = append(out, []string{v})
}
return out
}
for _, lbl := range labels[0] {
rs := argExpander(labels[1:])
for _, rlbls := range rs {
r := append(rlbls, []string{lbl}...)
out = append(out, r)
}
}
return out
}
// ArgPattern contains a list of patterns provided in the input.
type ArgPattern []Pattern
// Expand - expands all the ellipses patterns in
// the given argument.
func (a ArgPattern) Expand() [][]string {
labels := make([][]string, len(a))
for i := range labels {
labels[i] = a[i].Expand()
}
return argExpander(labels)
}
// Expand - expands a ellipses pattern.
func (p Pattern) Expand() []string {
var labels []string
for i := range p.Seq {
switch {
case p.Prefix != "" && p.Suffix == "":
labels = append(labels, fmt.Sprintf("%s%s", p.Prefix, p.Seq[i]))
case p.Suffix != "" && p.Prefix == "":
labels = append(labels, fmt.Sprintf("%s%s", p.Seq[i], p.Suffix))
case p.Suffix == "" && p.Prefix == "":
labels = append(labels, p.Seq[i])
default:
labels = append(labels, fmt.Sprintf("%s%s%s", p.Prefix, p.Seq[i], p.Suffix))
}
}
return labels
}
// HasEllipses - returns true if input arg has ellipses type pattern.
func HasEllipses(args ...string) bool {
var ok = true
for _, arg := range args {
ok = ok && (strings.Count(arg, ellipses) > 0 || (strings.Count(arg, openBraces) > 0 && strings.Count(arg, closeBraces) > 0))
}
return ok
}
// ErrInvalidEllipsesFormatFn error returned when invalid ellipses format is detected.
var ErrInvalidEllipsesFormatFn = func(arg string) error {
return fmt.Errorf("Invalid ellipsis format in (%s), Ellipsis range must be provided in format {N...M} where N and M are positive integers, M must be greater than N, with an allowed minimum range of 4", arg)
}
// FindEllipsesPatterns - finds all ellipses patterns, recursively and parses the ranges numerically.
func FindEllipsesPatterns(arg string) (ArgPattern, error) {
var patterns []Pattern
parts := regexpEllipses.FindStringSubmatch(arg)
if len(parts) == 0 {
// We throw an error if arg doesn't have any recognizable ellipses pattern.
return nil, ErrInvalidEllipsesFormatFn(arg)
}
parts = parts[1:]
patternFound := regexpEllipses.MatchString(parts[0])
for patternFound {
seq, err := parseEllipsesRange(parts[1])
if err != nil {
return patterns, err
}
patterns = append(patterns, Pattern{
Prefix: "",
Suffix: parts[2],
Seq: seq,
})
parts = regexpEllipses.FindStringSubmatch(parts[0])
if len(parts) > 0 {
parts = parts[1:]
patternFound = HasEllipses(parts[0])
continue
}
break
}
if len(parts) > 0 {
seq, err := parseEllipsesRange(parts[1])
if err != nil {
return patterns, err
}
patterns = append(patterns, Pattern{
Prefix: parts[0],
Suffix: parts[2],
Seq: seq,
})
}
// Check if any of the prefix or suffixes now have flower braces
// left over, in such a case we generally think that there is
// perhaps a typo in users input and error out accordingly.
for _, pattern := range patterns {
if strings.Count(pattern.Prefix, openBraces) > 0 || strings.Count(pattern.Prefix, closeBraces) > 0 {
return nil, ErrInvalidEllipsesFormatFn(arg)
}
if strings.Count(pattern.Suffix, openBraces) > 0 || strings.Count(pattern.Suffix, closeBraces) > 0 {
return nil, ErrInvalidEllipsesFormatFn(arg)
}
}
return patterns, nil
}

256
pkg/ellipses/ellipses_test.go
View File

@@ -1,256 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ellipses
import (
"fmt"
"testing"
)
// Test tests args with ellipses.
func TestHasEllipses(t *testing.T) {
testCases := []struct {
args []string
expectedOk bool
}{
// Tests for all args without ellipses.
{
[]string{"64"},
false,
},
// Found flower braces, still attempt to parse and throw an error.
{
[]string{"{1..64}"},
true,
},
{
[]string{"{1..2..}"},
true,
},
// Test for valid input.
{
[]string{"1...64"},
true,
},
{
[]string{"{1...2O}"},
true,
},
{
[]string{"..."},
true,
},
{
[]string{"{-1...1}"},
true,
},
{
[]string{"{0...-1}"},
true,
},
{
[]string{"{1....4}"},
true,
},
{
[]string{"{1...64}"},
true,
},
{
[]string{"{...}"},
true,
},
{
[]string{"{1...64}", "{65...128}"},
true,
},
{
[]string{"http://minio{2...3}/export/set{1...64}"},
true,
},
{
[]string{
"http://minio{2...3}/export/set{1...64}",
"http://minio{2...3}/export/set{65...128}",
},
true,
},
{
[]string{
"mydisk-{a...z}{1...20}",
},
true,
},
{
[]string{
"mydisk-{1...4}{1..2.}",
},
true,
},
}
for i, testCase := range testCases {
t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
gotOk := HasEllipses(testCase.args...)
if gotOk != testCase.expectedOk {
t.Errorf("Expected %t, got %t", testCase.expectedOk, gotOk)
}
})
}
}
// Test tests find ellipses patterns.
func TestFindEllipsesPatterns(t *testing.T) {
testCases := []struct {
pattern string
success bool
expectedCount int
}{
// Tests for all invalid inputs
{
"{1..64}",
false,
0,
},
{
"1...64",
false,
0,
},
{
"...",
false,
0,
},
{
"{1...",
false,
0,
},
{
"...64}",
false,
0,
},
{
"{...}",
false,
0,
},
{
"{-1...1}",
false,
0,
},
{
"{0...-1}",
false,
0,
},
{
"{1...2O}",
false,
0,
},
{
"{64...1}",
false,
0,
},
{
"{1....4}",
false,
0,
},
{
"mydisk-{a...z}{1...20}",
false,
0,
},
{
"mydisk-{1...4}{1..2.}",
false,
0,
},
{
"{1..2.}-mydisk-{1...4}",
false,
0,
},
{
"{{1...4}}",
false,
0,
},
{
"{4...02}",
false,
0,
},
{
"{f...z}",
false,
0,
},
// Test for valid input.
{
"{1...64}",
true,
64,
},
{
"{1...64} {65...128}",
true,
4096,
},
{
"{01...036}",
true,
36,
},
{
"{001...036}",
true,
36,
},
{
"{1...a}",
true,
10,
},
}
for i, testCase := range testCases {
t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
argP, err := FindEllipsesPatterns(testCase.pattern)
if err != nil && testCase.success {
t.Errorf("Expected success but failed instead %s", err)
}
if err == nil && !testCase.success {
t.Errorf("Expected failure but passed instead")
}
if err == nil {
gotCount := len(argP.Expand())
if gotCount != testCase.expectedCount {
t.Errorf("Expected %d, got %d", testCase.expectedCount, gotCount)
}
}
})
}
}

100
pkg/env/env.go vendored
View File

@@ -1,100 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package env
import (
"strconv"
"strings"
"sync"
)
var (
privateMutex sync.RWMutex
lockEnvMutex sync.Mutex
envOff bool
)
// LockSetEnv locks modifications to environment.
// Call returned function to unlock.
func LockSetEnv() func() {
lockEnvMutex.Lock()
return lockEnvMutex.Unlock
}
// SetEnvOff - turns off env lookup
// A global lock above this MUST ensure that
func SetEnvOff() {
privateMutex.Lock()
defer privateMutex.Unlock()
envOff = true
}
// SetEnvOn - turns on env lookup
func SetEnvOn() {
privateMutex.Lock()
defer privateMutex.Unlock()
envOff = false
}
// IsSet returns if the given env key is set.
func IsSet(key string) bool {
_, _, _, ok := LookupEnv(key)
return ok
}
// Get retrieves the value of the environment variable named
// by the key. If the variable is present in the environment the
// value (which may be empty) is returned. Otherwise it returns
// the specified default value.
func Get(key, defaultValue string) string {
privateMutex.RLock()
ok := envOff
privateMutex.RUnlock()
if ok {
return defaultValue
}
if v, _, _, ok := LookupEnv(key); ok {
return v
}
return defaultValue
}
// GetInt returns an integer if found in the environment
// and returns the default value otherwise.
func GetInt(key string, defaultValue int) (int, error) {
v := Get(key, "")
if v == "" {
return defaultValue, nil
}
return strconv.Atoi(v)
}
// List all envs with a given prefix.
func List(prefix string) (envs []string) {
for _, env := range Environ() {
if strings.HasPrefix(env, prefix) {
values := strings.SplitN(env, "=", 2)
if len(values) == 2 {
envs = append(envs, values[0])
}
}
}
return envs
}

187
pkg/env/web_env.go vendored
View File

@@ -1,187 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package env
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"io/ioutil"
"net"
"net/http"
"net/url"
"os"
"regexp"
"strings"
"time"
"github.com/dgrijalva/jwt-go"
)
const (
webEnvScheme = "env"
webEnvSchemeSecure = "env+tls"
)
var (
globalRootCAs *x509.CertPool
)
// RegisterGlobalCAs register the global root CAs
func RegisterGlobalCAs(CAs *x509.CertPool) {
globalRootCAs = CAs
}
var (
hostKeys = regexp.MustCompile("^(https?://)(.*?):(.*?)@(.*?)$")
)
func fetchHTTPConstituentParts(u *url.URL) (username string, password string, envURL string, err error) {
envURL = u.String()
if hostKeys.MatchString(envURL) {
parts := hostKeys.FindStringSubmatch(envURL)
if len(parts) != 5 {
return "", "", "", errors.New("invalid arguments")
}
username = parts[2]
password = parts[3]
envURL = fmt.Sprintf("%s%s", parts[1], parts[4])
}
if username == "" && password == "" && u.User != nil {
username = u.User.Username()
password, _ = u.User.Password()
}
return username, password, envURL, nil
}
func getEnvValueFromHTTP(urlStr, envKey string) (string, string, string, error) {
u, err := url.Parse(urlStr)
if err != nil {
return "", "", "", err
}
switch u.Scheme {
case webEnvScheme:
u.Scheme = "http"
case webEnvSchemeSecure:
u.Scheme = "https"
default:
return "", "", "", errors.New("invalid arguments")
}
username, password, envURL, err := fetchHTTPConstituentParts(u)
if err != nil {
return "", "", "", err
}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, envURL+"?key="+envKey, nil)
if err != nil {
return "", "", "", err
}
claims := &jwt.StandardClaims{
ExpiresAt: int64(15 * time.Minute),
Issuer: username,
Subject: envKey,
}
token := jwt.NewWithClaims(jwt.SigningMethodHS512, claims)
ss, err := token.SignedString([]byte(password))
if err != nil {
return "", "", "", err
}
req.Header.Set("Authorization", "Bearer "+ss)
clnt := &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 3 * time.Second,
KeepAlive: 5 * time.Second,
}).DialContext,
ResponseHeaderTimeout: 3 * time.Second,
TLSHandshakeTimeout: 3 * time.Second,
ExpectContinueTimeout: 3 * time.Second,
TLSClientConfig: &tls.Config{
RootCAs: globalRootCAs,
},
// Go net/http automatically unzip if content-type is
// gzip disable this feature, as we are always interested
// in raw stream.
DisableCompression: true,
},
}
resp, err := clnt.Do(req)
if err != nil {
return "", "", "", err
}
envValueBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", "", "", err
}
return string(envValueBytes), username, password, nil
}
// Environ returns a copy of strings representing the
// environment, in the form "key=value".
func Environ() []string {
return os.Environ()
}
// LookupEnv retrieves the value of the environment variable
// named by the key. If the variable is present in the
// environment the value (which may be empty) is returned
// and the boolean is true. Otherwise the returned value
// will be empty and the boolean will be false.
//
// Additionally if the input is env://username:password@remote:port/
// to fetch ENV values for the env value from a remote server.
// In this case, it also returns the credentials username and password
func LookupEnv(key string) (string, string, string, bool) {
v, ok := os.LookupEnv(key)
if ok && strings.HasPrefix(v, webEnvScheme) {
// If env value starts with `env*://`
// continue to parse and fetch from remote
var err error
v, user, pwd, err := getEnvValueFromHTTP(strings.TrimSpace(v), key)
if err != nil {
env, eok := os.LookupEnv("_" + key)
if eok {
// fallback to cached value if-any.
return env, user, pwd, eok
}
}
// Set the ENV value to _env value,
// this value is a fallback in-case of
// server restarts when webhook server
// is down.
os.Setenv("_"+key, v)
return v, user, pwd, true
}
return v, "", "", ok
}

89
pkg/env/web_env_test.go vendored
View File

@@ -1,89 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package env
import (
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"testing"
"github.com/gorilla/mux"
)
func GetenvHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
if vars["namespace"] != "default" {
http.Error(w, "namespace not found", http.StatusNotFound)
return
}
if vars["name"] != "minio" {
http.Error(w, "tenant not found", http.StatusNotFound)
return
}
if vars["key"] != "MINIO_ARGS" {
http.Error(w, "key not found", http.StatusNotFound)
return
}
w.Write([]byte("http://127.0.0.{1..4}:9000/data{1...4}"))
w.(http.Flusher).Flush()
}
func startTestServer(t *testing.T) *httptest.Server {
router := mux.NewRouter().SkipClean(true).UseEncodedPath()
router.Methods(http.MethodGet).
Path("/webhook/v1/getenv/{namespace}/{name}").
HandlerFunc(GetenvHandler).Queries("key", "{key:.*}")
ts := httptest.NewServer(router)
t.Cleanup(func() {
ts.Close()
})
return ts
}
func TestWebEnv(t *testing.T) {
ts := startTestServer(t)
u, err := url.Parse(ts.URL)
if err != nil {
t.Fatal(err)
}
v, user, pwd, err := getEnvValueFromHTTP(
fmt.Sprintf("env://minio:minio123@%s/webhook/v1/getenv/default/minio",
u.Host),
"MINIO_ARGS")
if err != nil {
t.Fatal(err)
}
if v != "http://127.0.0.{1..4}:9000/data{1...4}" {
t.Fatalf("Unexpected value %s", v)
}
if user != "minio" {
t.Fatalf("Unexpected value %s", v)
}
if pwd != "minio123" {
t.Fatalf("Unexpected value %s", v)
}
}

2
pkg/event/rules.go
View File

@@ -20,7 +20,7 @@ package event
import (
"strings"
"github.com/minio/minio/pkg/wildcard"
"github.com/minio/pkg/wildcard"
)
// NewPattern - create new pattern for prefix/suffix.

2
pkg/event/target/queuestore.go
View File

@@ -27,7 +27,7 @@ import (
"sync"
"github.com/minio/minio/pkg/event"
"github.com/minio/minio/pkg/sys"
"github.com/minio/pkg/sys"
)
const (

2
pkg/event/target/webhook.go
View File

@@ -32,9 +32,9 @@ import (
"path/filepath"
"time"
"github.com/minio/minio/pkg/certs"
"github.com/minio/minio/pkg/event"
xnet "github.com/minio/minio/pkg/net"
"github.com/minio/pkg/certs"
)
// Webhook constants

2
pkg/iam/policy/action.go
View File

@@ -19,7 +19,7 @@ package iampolicy
import (
"github.com/minio/minio/pkg/bucket/policy/condition"
"github.com/minio/minio/pkg/wildcard"
"github.com/minio/pkg/wildcard"
)
// Action - policy action.

2
pkg/iam/policy/resource.go
View File

@@ -23,7 +23,7 @@ import (
"strings"
"github.com/minio/minio/pkg/bucket/policy/condition"
"github.com/minio/minio/pkg/wildcard"
"github.com/minio/pkg/wildcard"
)
// ResourceARNPrefix - resource ARN prefix as per AWS S3 specification.

118
pkg/licverifier/verifier.go
View File

@@ -1,118 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Package licverifier implements a simple library to verify MinIO Subnet license keys.
package licverifier
import (
"crypto/ecdsa"
"errors"
"fmt"
"time"
"github.com/dgrijalva/jwt-go"
)
// LicenseVerifier needs an ECDSA public key in PEM format for initialization.
type LicenseVerifier struct {
ecPubKey *ecdsa.PublicKey
}
// LicenseInfo holds customer metadata present in the license key.
type LicenseInfo struct {
Email string // Email of the license key requestor
Organization string // Subnet organization name
AccountID int64 // Subnet account id
StorageCapacity int64 // Storage capacity used in TB
Plan string // Subnet plan
ExpiresAt time.Time // Time of license expiry
}
// license key JSON field names
const (
accountID = "aid"
sub = "sub"
expiresAt = "exp"
organization = "org"
capacity = "cap"
plan = "plan"
)
// NewLicenseVerifier returns an initialized license verifier with the given
// ECDSA public key in PEM format.
func NewLicenseVerifier(pemBytes []byte) (*LicenseVerifier, error) {
pbKey, err := jwt.ParseECPublicKeyFromPEM(pemBytes)
if err != nil {
return nil, fmt.Errorf("Failed to parse public key: %s", err)
}
return &LicenseVerifier{
ecPubKey: pbKey,
}, nil
}
// toLicenseInfo extracts LicenseInfo from claims. It returns an error if any of
// the claim values are invalid.
func toLicenseInfo(claims jwt.MapClaims) (LicenseInfo, error) {
accID, ok := claims[accountID].(float64)
if !ok || ok && accID <= 0 {
return LicenseInfo{}, errors.New("Invalid accountId in claims")
}
email, ok := claims[sub].(string)
if !ok {
return LicenseInfo{}, errors.New("Invalid email in claims")
}
expiryTS, ok := claims[expiresAt].(float64)
if !ok {
return LicenseInfo{}, errors.New("Invalid time of expiry in claims")
}
expiresAt := time.Unix(int64(expiryTS), 0)
orgName, ok := claims[organization].(string)
if !ok {
return LicenseInfo{}, errors.New("Invalid organization in claims")
}
storageCap, ok := claims[capacity].(float64)
if !ok {
return LicenseInfo{}, errors.New("Invalid storage capacity in claims")
}
plan, ok := claims[plan].(string)
if !ok {
return LicenseInfo{}, errors.New("Invalid plan in claims")
}
return LicenseInfo{
Email: email,
Organization: orgName,
AccountID: int64(accID),
StorageCapacity: int64(storageCap),
Plan: plan,
ExpiresAt: expiresAt,
}, nil
}
// Verify verifies the license key and validates the claims present in it.
func (lv *LicenseVerifier) Verify(license string) (LicenseInfo, error) {
token, err := jwt.ParseWithClaims(license, &jwt.MapClaims{}, func(token *jwt.Token) (interface{}, error) {
return lv.ecPubKey, nil
})
if err != nil {
return LicenseInfo{}, fmt.Errorf("Failed to verify license: %s", err)
}
if claims, ok := token.Claims.(*jwt.MapClaims); ok && token.Valid {
return toLicenseInfo(*claims)
}
return LicenseInfo{}, errors.New("Invalid claims found in license")
}

111
pkg/licverifier/verifier_test.go
View File

@@ -1,111 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package licverifier
import (
"fmt"
"testing"
"time"
"github.com/dgrijalva/jwt-go"
)
// at fixes the jwt.TimeFunc at t and calls f in that context.
func at(t time.Time, f func()) {
jwt.TimeFunc = func() time.Time { return t }
f()
jwt.TimeFunc = time.Now
}
func areEqLicenseInfo(a, b LicenseInfo) bool {
if a.Email == b.Email && a.Organization == b.Organization && a.AccountID == b.AccountID && a.Plan == b.Plan && a.StorageCapacity == b.StorageCapacity && a.ExpiresAt.Equal(b.ExpiresAt) {
return true
}
return false
}
// TestLicenseVerify tests the license key verification process with a valid and
// an invalid key.
func TestLicenseVerify(t *testing.T) {
pemBytes := []byte(`-----BEGIN PUBLIC KEY-----
MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEbo+e1wpBY4tBq9AONKww3Kq7m6QP/TBQ
mr/cKCUyBL7rcAvg0zNq1vcSrUSGlAmY3SEDCu3GOKnjG/U4E7+p957ocWSV+mQU
9NKlTdQFGF3+aO6jbQ4hX/S5qPyF+a3z
-----END PUBLIC KEY-----`)
lv, err := NewLicenseVerifier(pemBytes)
if err != nil {
t.Fatalf("Failed to create license verifier: %s", err)
}
testCases := []struct {
lic string
expectedLicInfo LicenseInfo
shouldPass bool
}{{"", LicenseInfo{}, false},
{"eyJhbGciOiJFUzM4NCIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJrYW5hZ2FyYWorYzFAbWluaW8uaW8iLCJjYXAiOjUwLCJvcmciOiJHcmluZ290dHMgSW5jLiIsImV4cCI6MS42NDE0NDYxNjkwMDExOTg4OTRlOSwicGxhbiI6IlNUQU5EQVJEIiwiaXNzIjoic3VibmV0QG1pbi5pbyIsImFpZCI6MSwiaWF0IjoxLjYwOTkxMDE2OTAwMTE5ODg5NGU5fQ.EhTL2xwMHnUoLQF4UR-5bjUCja3whseLU5mb9XEj7PvAae6HEIDCOMEF8Hhh20DN_v_LRE283j2ZlA5zulcXSZXS0CLcrKqbVy6QLvZfvvLuerOjJI-NBa9dSJWJ0WoN", LicenseInfo{
Email: "kanagaraj+c1@minio.io",
Organization: "Gringotts Inc.",
AccountID: 1,
StorageCapacity: 50,
Plan: "STANDARD",
ExpiresAt: time.Date(2022, time.January, 6, 5, 16, 9, 0, time.UTC),
}, true},
}
for i, tc := range testCases {
// Fixing the jwt.TimeFunc at 2021-01-06 05:16:09 +0000 UTC to
// ensure that the license JWT doesn't expire ever.
at(time.Unix(int64(1609910169), 0), func() {
licInfo, err := lv.Verify(tc.lic)
if err != nil && tc.shouldPass {
t.Fatalf("%d: Expected license to pass verification but failed with %s", i+1, err)
}
if err == nil {
if !tc.shouldPass {
t.Fatalf("%d: Expected license to fail verification but passed", i+1)
}
if !areEqLicenseInfo(tc.expectedLicInfo, licInfo) {
t.Fatalf("%d: Expected license info %v but got %v", i+1, tc.expectedLicInfo, licInfo)
}
}
})
}
}
// Example creates a LicenseVerifier using the ECDSA public key in pemBytes. It
// uses the Verify method of the LicenseVerifier to verify and extract the
// claims present in the license key.
func Example() {
pemBytes := []byte(`-----BEGIN PUBLIC KEY-----
MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEbo+e1wpBY4tBq9AONKww3Kq7m6QP/TBQ
mr/cKCUyBL7rcAvg0zNq1vcSrUSGlAmY3SEDCu3GOKnjG/U4E7+p957ocWSV+mQU
9NKlTdQFGF3+aO6jbQ4hX/S5qPyF+a3z
-----END PUBLIC KEY-----`)
lv, err := NewLicenseVerifier(pemBytes)
if err != nil {
fmt.Println("Failed to create license verifier", err)
}
licenseKey := "eyJhbGciOiJFUzM4NCIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJrYW5hZ2FyYWorYzFAbWluaW8uaW8iLCJjYXAiOjUwLCJvcmciOiJHcmluZ290dHMgSW5jLiIsImV4cCI6MS42NDE0NDYxNjkwMDExOTg4OTRlOSwicGxhbiI6IlNUQU5EQVJEIiwiaXNzIjoic3VibmV0QG1pbi5pbyIsImFpZCI6MSwiaWF0IjoxLjYwOTkxMDE2OTAwMTE5ODg5NGU5fQ.EhTL2xwMHnUoLQF4UR-5bjUCja3whseLU5mb9XEj7PvAae6HEIDCOMEF8Hhh20DN_v_LRE283j2ZlA5zulcXSZXS0CLcrKqbVy6QLvZfvvLuerOjJI-NBa9dSJWJ0WoN"
licInfo, err := lv.Verify(licenseKey)
if err != nil {
fmt.Println("Failed to verify license key", err)
}
fmt.Println("License metadata", licInfo)
}

17
pkg/mimedb/Makefile
View File

@@ -1,17 +0,0 @@
# Generate db.go from db.json downloaded nodejs mime-db project.
# NOTE: Autogenerated db.go needs to be vet proofed. \
Manually edit json -> JSON for all variable names
all: download build
# Download db.json from NodeJS's mime-db project. It is under MIT license.
download:
@mkdir db
@wget -nv -q https://cdn.rawgit.com/jshttp/mime-db/master/db.json -O db/db.json
# After generating db.go, clean up downloaded db.json.
build: download
@go run util/gen-db.go db/db.json > db.go
@rm -f db/db.json
@rm -rf db
@echo Generated \"db.go\".

4411
pkg/mimedb/db.go
View File

File diff suppressed because it is too large Load Diff

45
pkg/mimedb/db_test.go
View File

@@ -1,45 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package mimedb
import "testing"
func TestMimeLookup(t *testing.T) {
// Test mimeLookup.
contentType := DB["txt"].ContentType
if contentType != "text/plain" {
t.Fatalf("Invalid content type are found expected \"application/x-msdownload\", got %s", contentType)
}
compressible := DB["txt"].Compressible
if compressible {
t.Fatalf("Invalid content type are found expected \"false\", got %t", compressible)
}
}
func TestTypeByExtension(t *testing.T) {
// Test TypeByExtension.
contentType := TypeByExtension(".txt")
if contentType != "text/plain" {
t.Fatalf("Invalid content type are found expected \"text/plain\", got %s", contentType)
}
// Test non-existent type resolution
contentType = TypeByExtension(".abc")
if contentType != "application/octet-stream" {
t.Fatalf("Invalid content type are found expected \"application/octet-stream\", got %s", contentType)
}
}

34
pkg/mimedb/resolve-db.go
View File

@@ -1,34 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package mimedb
import (
"strings"
)
// TypeByExtension resolves the extension to its respective content-type.
func TypeByExtension(ext string) string {
// Set default to "application/octet-stream".
var contentType = "application/octet-stream"
if ext != "" {
if content, ok := DB[strings.ToLower(strings.TrimPrefix(ext, "."))]; ok {
contentType = content.ContentType
}
}
return contentType
}

146
pkg/mimedb/util/gen-db.go
View File

@@ -1,146 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Package mimedb is a database of file extension to mime content-type.
// Definitions are imported from NodeJS mime-db project under MIT license.
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"strings"
"text/template"
)
const progTempl = `// DO NOT EDIT THIS FILE. IT IS AUTO-GENERATED BY "gen-db.go".
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Package mimedb is a database of file extension to mime content-type.
// Definitions are imported from NodeJS mime-db project under MIT license.
package mimedb
// DB - Mime is a collection of mime types with extension as key and content-type as value.
var DB = map[string]struct {
ContentType string
Compressible bool
}{
{{range $extension, $entry := . }} "{{$extension}}": {
ContentType: "{{$entry.ContentType}}",
Compressible: {{$entry.Compressible}},
},
{{end}}}
`
type mimeEntry struct {
ContentType string `json:"contentType"`
Compressible bool `json:"compresible"`
}
type mimeDB map[string]mimeEntry
// JSON data from gobindata and parse them into extDB.
func convertDB(jsonFile string) (mimeDB, error) {
// Structure of JSON data from mime-db project.
type dbEntry struct {
Source string `json:"source"`
Compressible bool `json:"compresible"`
Extensions []string `json:"extensions"`
}
// Access embedded "db.json" inside go-bindata.
jsonDB, err := ioutil.ReadFile(jsonFile)
if err != nil {
return nil, err
}
// Convert db.json into go's typed structure.
db := make(map[string]dbEntry)
if err := json.Unmarshal(jsonDB, &db); err != nil {
return nil, err
}
mDB := make(mimeDB)
// Generate a new database from mime-db.
for key, val := range db {
if len(val.Extensions) > 0 {
/* Denormalize - each extension has its own
unique content-type now. Looks will be fast. */
for _, ext := range val.Extensions {
/* Single extension type may map to
multiple content-types. In that case,
simply prefer the longest content-type
to maintain some level of
consistency. Only guarantee is,
whatever content type is assigned, it
is appropriate and valid type. */
if strings.Compare(mDB[ext].ContentType, key) < 0 {
mDB[ext] = mimeEntry{
ContentType: key,
Compressible: val.Compressible,
}
}
}
}
}
return mDB, nil
}
func main() {
// Take input json file from command-line".
if len(os.Args) != 2 {
fmt.Print("Syntax:\n\tgen-db /path/to/db.json\n")
os.Exit(1)
}
// Load and convert db.json into new database with extension
// as key.
mDB, err := convertDB(os.Args[1])
if err != nil {
panic(err)
}
// Generate db embedded go program.
tmpl := template.New("mimedb")
mimeTmpl, err := tmpl.Parse(progTempl)
if err != nil {
panic(err)
}
err = mimeTmpl.Execute(os.Stdout, mDB)
if err != nil {
panic(err)
}
}

191
pkg/quick/encoding.go
View File

@@ -1,191 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package quick
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"strings"
"time"
etcd "go.etcd.io/etcd/client/v3"
yaml "gopkg.in/yaml.v2"
)
// ConfigEncoding is a generic interface which
// marshal/unmarshal configuration.
type ConfigEncoding interface {
Unmarshal([]byte, interface{}) error
Marshal(interface{}) ([]byte, error)
}
// YAML encoding implements ConfigEncoding
type yamlEncoding struct{}
func (y yamlEncoding) Unmarshal(b []byte, v interface{}) error {
return yaml.Unmarshal(b, v)
}
func (y yamlEncoding) Marshal(v interface{}) ([]byte, error) {
return yaml.Marshal(v)
}
// JSON encoding implements ConfigEncoding
type jsonEncoding struct{}
func (j jsonEncoding) Unmarshal(b []byte, v interface{}) error {
err := json.Unmarshal(b, v)
if err != nil {
// Try to return a sophisticated json error message if possible
switch jerr := err.(type) {
case *json.SyntaxError:
return fmt.Errorf("Unable to parse JSON schema due to a syntax error at '%s'",
FormatJSONSyntaxError(bytes.NewReader(b), jerr.Offset))
case *json.UnmarshalTypeError:
return fmt.Errorf("Unable to parse JSON, type '%v' cannot be converted into the Go '%v' type",
jerr.Value, jerr.Type)
}
return err
}
return nil
}
func (j jsonEncoding) Marshal(v interface{}) ([]byte, error) {
return json.MarshalIndent(v, "", "\t")
}
// Convert a file extension to the appropriate struct capable
// to marshal/unmarshal data
func ext2EncFormat(fileExtension string) ConfigEncoding {
// Lower the file extension
ext := strings.ToLower(fileExtension)
ext = strings.TrimPrefix(ext, ".")
// Return the appropriate encoder/decoder according
// to the extension
switch ext {
case "yml", "yaml":
// YAML
return yamlEncoding{}
default:
// JSON
return jsonEncoding{}
}
}
// toMarshaller returns the right marshal function according
// to the given file extension
func toMarshaller(ext string) func(interface{}) ([]byte, error) {
return ext2EncFormat(ext).Marshal
}
// toUnmarshaller returns the right marshal function according
// to the given file extension
func toUnmarshaller(ext string) func([]byte, interface{}) error {
return ext2EncFormat(ext).Unmarshal
}
// saveFileConfig marshals with the right encoding format
// according to the filename extension, if no extension is
// provided, json will be selected.
func saveFileConfig(filename string, v interface{}) error {
// Fetch filename's extension
ext := filepath.Ext(filename)
// Marshal data
dataBytes, err := toMarshaller(ext)(v)
if err != nil {
return err
}
if runtime.GOOS == "windows" {
dataBytes = []byte(strings.Replace(string(dataBytes), "\n", "\r\n", -1))
}
// Save data.
return writeFile(filename, dataBytes)
}
func saveFileConfigEtcd(filename string, clnt *etcd.Client, v interface{}) error {
// Fetch filename's extension
ext := filepath.Ext(filename)
// Marshal data
dataBytes, err := toMarshaller(ext)(v)
if err != nil {
return err
}
if runtime.GOOS == "windows" {
dataBytes = []byte(strings.Replace(string(dataBytes), "\n", "\r\n", -1))
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
_, err = clnt.Put(ctx, filename, string(dataBytes))
if err == context.DeadlineExceeded {
return fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", clnt.Endpoints())
} else if err != nil {
return fmt.Errorf("unexpected error %w returned by etcd setup, please check your endpoints %s", err, clnt.Endpoints())
}
return nil
}
func loadFileConfigEtcd(filename string, clnt *etcd.Client, v interface{}) error {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
resp, err := clnt.Get(ctx, filename)
if err != nil {
if err == context.DeadlineExceeded {
return fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", clnt.Endpoints())
}
return fmt.Errorf("unexpected error %w returned by etcd setup, please check your endpoints %s", err, clnt.Endpoints())
}
if resp.Count == 0 {
return os.ErrNotExist
}
for _, ev := range resp.Kvs {
if string(ev.Key) == filename {
fileData := ev.Value
if runtime.GOOS == "windows" {
fileData = bytes.Replace(fileData, []byte("\r\n"), []byte("\n"), -1)
}
// Unmarshal file's content
return toUnmarshaller(filepath.Ext(filename))(fileData, v)
}
}
return os.ErrNotExist
}
// loadFileConfig unmarshals the file's content with the right
// decoder format according to the filename extension. If no
// extension is provided, json will be selected by default.
func loadFileConfig(filename string, v interface{}) error {
fileData, err := ioutil.ReadFile(filename)
if err != nil {
return err
}
if runtime.GOOS == "windows" {
fileData = []byte(strings.Replace(string(fileData), "\r\n", "\n", -1))
}
// Unmarshal file's content
return toUnmarshaller(filepath.Ext(filename))(fileData, v)
}

81
pkg/quick/errorutil.go
View File

@@ -1,81 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package quick
import (
"bufio"
"bytes"
"fmt"
"io"
"github.com/cheggaaa/pb"
)
const errorFmt = "%5d: %s <<<<"
// FormatJSONSyntaxError generates a pretty printed json syntax error since
// golang doesn't provide an easy way to report the location of the error
func FormatJSONSyntaxError(data io.Reader, offset int64) (highlight string) {
var readLine bytes.Buffer
var errLine = 1
var readBytes int64
bio := bufio.NewReader(data)
// termWidth is set to a default one to use when we are
// not able to calculate terminal width via OS syscalls
termWidth := 25
// errorShift is the length of the minimum needed place for
// error msg accessories, like <--, etc.. We calculate it
// dynamically to avoid an eventual bug after modifying errorFmt
errorShift := len(fmt.Sprintf(errorFmt, 1, ""))
if width, err := pb.GetTerminalWidth(); err == nil {
termWidth = width
}
for {
b, err := bio.ReadByte()
if err != nil {
break
}
readBytes++
if readBytes > offset {
break
}
if b == '\n' {
readLine.Reset()
errLine++
continue
} else if b == '\t' {
readLine.WriteByte(' ')
} else if b == '\r' {
break
}
readLine.WriteByte(b)
}
lineLen := readLine.Len()
idx := lineLen - termWidth + errorShift
if idx < 0 || idx > lineLen-1 {
idx = 0
}
return fmt.Sprintf(errorFmt, errLine, readLine.String()[idx:])
}

236
pkg/quick/quick.go
View File

@@ -1,236 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package quick
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"reflect"
"sync"
"github.com/fatih/structs"
"github.com/minio/minio/pkg/safe"
etcd "go.etcd.io/etcd/client/v3"
)
// Config - generic config interface functions
type Config interface {
String() string
Version() string
Save(string) error
Load(string) error
Data() interface{}
Diff(Config) ([]structs.Field, error)
DeepDiff(Config) ([]structs.Field, error)
}
// config - implements quick.Config interface
type config struct {
data interface{}
clnt *etcd.Client
lock *sync.RWMutex
}
// Version returns the current config file format version
func (d config) Version() string {
st := structs.New(d.data)
f := st.Field("Version")
return f.Value().(string)
}
// String converts JSON config to printable string
func (d config) String() string {
configBytes, _ := json.MarshalIndent(d.data, "", "\t")
return string(configBytes)
}
// Save writes config data to a file. Data format
// is selected based on file extension or JSON if
// not provided.
func (d config) Save(filename string) error {
d.lock.Lock()
defer d.lock.Unlock()
if d.clnt != nil {
return saveFileConfigEtcd(filename, d.clnt, d.data)
}
// Backup if given file exists
oldData, err := ioutil.ReadFile(filename)
if err != nil {
// Ignore if file does not exist.
if !os.IsNotExist(err) {
return err
}
} else {
// Save read data to the backup file.
backupFilename := filename + ".old"
if err = writeFile(backupFilename, oldData); err != nil {
return err
}
}
// Save data.
return saveFileConfig(filename, d.data)
}
// Load - loads config from file and merge with currently set values
// File content format is guessed from the file name extension, if not
// available, consider that we have JSON.
func (d config) Load(filename string) error {
d.lock.Lock()
defer d.lock.Unlock()
if d.clnt != nil {
return loadFileConfigEtcd(filename, d.clnt, d.data)
}
return loadFileConfig(filename, d.data)
}
// Data - grab internal data map for reading
func (d config) Data() interface{} {
return d.data
}
// Diff - list fields that are in A but not in B
func (d config) Diff(c Config) ([]structs.Field, error) {
var fields []structs.Field
currFields := structs.Fields(d.Data())
newFields := structs.Fields(c.Data())
var found bool
for _, currField := range currFields {
found = false
for _, newField := range newFields {
if reflect.DeepEqual(currField.Name(), newField.Name()) {
found = true
}
}
if !found {
fields = append(fields, *currField)
}
}
return fields, nil
}
// DeepDiff - list fields in A that are missing or not equal to fields in B
func (d config) DeepDiff(c Config) ([]structs.Field, error) {
var fields []structs.Field
currFields := structs.Fields(d.Data())
newFields := structs.Fields(c.Data())
var found bool
for _, currField := range currFields {
found = false
for _, newField := range newFields {
if reflect.DeepEqual(currField.Value(), newField.Value()) {
found = true
}
}
if !found {
fields = append(fields, *currField)
}
}
return fields, nil
}
// CheckData - checks the validity of config data. Data should be of
// type struct and contain a string type field called "Version".
func CheckData(data interface{}) error {
if !structs.IsStruct(data) {
return fmt.Errorf("interface must be struct type")
}
st := structs.New(data)
f, ok := st.FieldOk("Version")
if !ok {
return fmt.Errorf("struct ‘%s’ must have field ‘Version’", st.Name())
}
if f.Kind() != reflect.String {
return fmt.Errorf("‘Version’ field in struct ‘%s’ must be a string type", st.Name())
}
return nil
}
// writeFile writes data to a file named by filename.
// If the file does not exist, writeFile creates it;
// otherwise writeFile truncates it before writing.
func writeFile(filename string, data []byte) error {
safeFile, err := safe.CreateFile(filename)
if err != nil {
return err
}
_, err = safeFile.Write(data)
if err != nil {
return err
}
return safeFile.Close()
}
// GetVersion - extracts the version information.
func GetVersion(filename string, clnt *etcd.Client) (version string, err error) {
var qc Config
qc, err = LoadConfig(filename, clnt, &struct {
Version string
}{})
if err != nil {
return "", err
}
return qc.Version(), nil
}
// LoadConfig - loads json config from filename for the a given struct data
func LoadConfig(filename string, clnt *etcd.Client, data interface{}) (qc Config, err error) {
qc, err = NewConfig(data, clnt)
if err != nil {
return nil, err
}
return qc, qc.Load(filename)
}
// SaveConfig - saves given configuration data into given file as JSON.
func SaveConfig(data interface{}, filename string, clnt *etcd.Client) (err error) {
if err = CheckData(data); err != nil {
return err
}
var qc Config
qc, err = NewConfig(data, clnt)
if err != nil {
return err
}
return qc.Save(filename)
}
// NewConfig loads config from etcd client if provided, otherwise loads from a local filename.
// fails when all else fails.
func NewConfig(data interface{}, clnt *etcd.Client) (cfg Config, err error) {
if err := CheckData(data); err != nil {
return nil, err
}
d := new(config)
d.data = data
d.clnt = clnt
d.lock = new(sync.RWMutex)
return d, nil
}

505
pkg/quick/quick_test.go
View File

@@ -1,505 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package quick
import (
"bytes"
"encoding/json"
"io/ioutil"
"os"
"reflect"
"runtime"
"strings"
"testing"
)
func TestReadVersion(t *testing.T) {
type myStruct struct {
Version string
}
saveMe := myStruct{"1"}
config, err := NewConfig(&saveMe, nil)
if err != nil {
t.Fatal(err)
}
err = config.Save("test.json")
if err != nil {
t.Fatal(err)
}
version, err := GetVersion("test.json", nil)
if err != nil {
t.Fatal(err)
}
if version != "1" {
t.Fatalf("Expected version '1', got '%v'", version)
}
}
func TestReadVersionErr(t *testing.T) {
type myStruct struct {
Version int
}
saveMe := myStruct{1}
_, err := NewConfig(&saveMe, nil)
if err == nil {
t.Fatal("Unexpected should fail in initialization for bad input")
}
err = ioutil.WriteFile("test.json", []byte("{ \"version\":2,"), 0644)
if err != nil {
t.Fatal(err)
}
_, err = GetVersion("test.json", nil)
if err == nil {
t.Fatal("Unexpected should fail to fetch version")
}
err = ioutil.WriteFile("test.json", []byte("{ \"version\":2 }"), 0644)
if err != nil {
t.Fatal(err)
}
_, err = GetVersion("test.json", nil)
if err == nil {
t.Fatal("Unexpected should fail to fetch version")
}
}
func TestSaveFailOnDir(t *testing.T) {
defer os.RemoveAll("test-1.json")
err := os.MkdirAll("test-1.json", 0644)
if err != nil {
t.Fatal(err)
}
type myStruct struct {
Version string
}
saveMe := myStruct{"1"}
config, err := NewConfig(&saveMe, nil)
if err != nil {
t.Fatal(err)
}
err = config.Save("test-1.json")
if err == nil {
t.Fatal("Unexpected should fail to save if test-1.json is a directory")
}
}
func TestCheckData(t *testing.T) {
err := CheckData(nil)
if err == nil {
t.Fatal("Unexpected should fail")
}
type myStructBadNoVersion struct {
User string
Password string
Directories []string
}
saveMeBadNoVersion := myStructBadNoVersion{"guest", "nopassword", []string{"Work", "Documents", "Music"}}
err = CheckData(&saveMeBadNoVersion)
if err == nil {
t.Fatal("Unexpected should fail if Version is not set")
}
type myStructBadVersionInt struct {
Version int
User string
Password string
}
saveMeBadVersionInt := myStructBadVersionInt{1, "guest", "nopassword"}
err = CheckData(&saveMeBadVersionInt)
if err == nil {
t.Fatal("Unexpected should fail if Version is integer")
}
type myStructGood struct {
Version string
User string
Password string
Directories []string
}
saveMeGood := myStructGood{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
err = CheckData(&saveMeGood)
if err != nil {
t.Fatal(err)
}
}
func TestLoadFile(t *testing.T) {
type myStruct struct {
Version string
User string
Password string
Directories []string
}
saveMe := myStruct{}
_, err := LoadConfig("test.json", nil, &saveMe)
if err == nil {
t.Fatal(err)
}
file, err := os.Create("test.json")
if err != nil {
t.Fatal(err)
}
if err = file.Close(); err != nil {
t.Fatal(err)
}
_, err = LoadConfig("test.json", nil, &saveMe)
if err == nil {
t.Fatal("Unexpected should fail to load empty JSON")
}
config, err := NewConfig(&saveMe, nil)
if err != nil {
t.Fatal(err)
}
err = config.Load("test-non-exist.json")
if err == nil {
t.Fatal("Unexpected should fail to Load non-existent config")
}
err = config.Load("test.json")
if err == nil {
t.Fatal("Unexpected should fail to load empty JSON")
}
saveMe = myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
config, err = NewConfig(&saveMe, nil)
if err != nil {
t.Fatal(err)
}
err = config.Save("test.json")
if err != nil {
t.Fatal(err)
}
saveMe1 := myStruct{}
_, err = LoadConfig("test.json", nil, &saveMe1)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(saveMe1, saveMe) {
t.Fatalf("Expected %v, got %v", saveMe1, saveMe)
}
saveMe2 := myStruct{}
err = json.Unmarshal([]byte(config.String()), &saveMe2)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(saveMe2, saveMe1) {
t.Fatalf("Expected %v, got %v", saveMe2, saveMe1)
}
}
func TestYAMLFormat(t *testing.T) {
testYAML := "test.yaml"
defer os.RemoveAll(testYAML)
type myStruct struct {
Version string
User string
Password string
Directories []string
}
plainYAML := `version: "1"
user: guest
password: nopassword
directories:
- Work
- Documents
- Music
`
if runtime.GOOS == "windows" {
plainYAML = strings.Replace(plainYAML, "\n", "\r\n", -1)
}
saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
// Save format using
config, err := NewConfig(&saveMe, nil)
if err != nil {
t.Fatal(err)
}
err = config.Save(testYAML)
if err != nil {
t.Fatal(err)
}
// Check if the saved structure in actually an YAML format
b, err := ioutil.ReadFile(testYAML)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal([]byte(plainYAML), b) {
t.Fatalf("Expected %v, got %v", plainYAML, string(b))
}
// Check if the loaded data is the same as the saved one
loadMe := myStruct{}
config, err = NewConfig(&loadMe, nil)
if err != nil {
t.Fatal(err)
}
err = config.Load(testYAML)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(saveMe, loadMe) {
t.Fatalf("Expected %v, got %v", saveMe, loadMe)
}
}
func TestJSONFormat(t *testing.T) {
testJSON := "test.json"
defer os.RemoveAll(testJSON)
type myStruct struct {
Version string
User string
Password string
Directories []string
}
plainJSON := `{
"Version": "1",
"User": "guest",
"Password": "nopassword",
"Directories": [
"Work",
"Documents",
"Music"
]
}`
if runtime.GOOS == "windows" {
plainJSON = strings.Replace(plainJSON, "\n", "\r\n", -1)
}
saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
// Save format using
config, err := NewConfig(&saveMe, nil)
if err != nil {
t.Fatal(err)
}
err = config.Save(testJSON)
if err != nil {
t.Fatal(err)
}
// Check if the saved structure in actually an JSON format
b, err := ioutil.ReadFile(testJSON)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal([]byte(plainJSON), b) {
t.Fatalf("Expected %v, got %v", plainJSON, string(b))
}
// Check if the loaded data is the same as the saved one
loadMe := myStruct{}
config, err = NewConfig(&loadMe, nil)
if err != nil {
t.Fatal(err)
}
err = config.Load(testJSON)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(saveMe, loadMe) {
t.Fatalf("Expected %v, got %v", saveMe, loadMe)
}
}
func TestSaveLoad(t *testing.T) {
defer os.RemoveAll("test.json")
type myStruct struct {
Version string
User string
Password string
Directories []string
}
saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
config, err := NewConfig(&saveMe, nil)
if err != nil {
t.Fatal(err)
}
err = config.Save("test.json")
if err != nil {
t.Fatal(err)
}
loadMe := myStruct{Version: "1"}
newConfig, err := NewConfig(&loadMe, nil)
if err != nil {
t.Fatal(err)
}
err = newConfig.Load("test.json")
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(config.Data(), newConfig.Data()) {
t.Fatalf("Expected %v, got %v", config.Data(), newConfig.Data())
}
if !reflect.DeepEqual(config.Data(), &loadMe) {
t.Fatalf("Expected %v, got %v", config.Data(), &loadMe)
}
mismatch := myStruct{"1.1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
if reflect.DeepEqual(config.Data(), &mismatch) {
t.Fatal("Expected to mismatch but succeeded instead")
}
}
func TestSaveBackup(t *testing.T) {
defer os.RemoveAll("test.json")
defer os.RemoveAll("test.json.old")
type myStruct struct {
Version string
User string
Password string
Directories []string
}
saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
config, err := NewConfig(&saveMe, nil)
if err != nil {
t.Fatal(err)
}
err = config.Save("test.json")
if err != nil {
t.Fatal(err)
}
loadMe := myStruct{Version: "1"}
newConfig, err := NewConfig(&loadMe, nil)
if err != nil {
t.Fatal(err)
}
err = newConfig.Load("test.json")
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(config.Data(), newConfig.Data()) {
t.Fatalf("Expected %v, got %v", config.Data(), newConfig.Data())
}
if !reflect.DeepEqual(config.Data(), &loadMe) {
t.Fatalf("Expected %v, got %v", config.Data(), &loadMe)
}
mismatch := myStruct{"1.1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
if reflect.DeepEqual(newConfig.Data(), &mismatch) {
t.Fatal("Expected to mismatch but succeeded instead")
}
config, err = NewConfig(&mismatch, nil)
if err != nil {
t.Fatal(err)
}
err = config.Save("test.json")
if err != nil {
t.Fatal(err)
}
}
func TestDiff(t *testing.T) {
type myStruct struct {
Version string
User string
Password string
Directories []string
}
saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
config, err := NewConfig(&saveMe, nil)
if err != nil {
t.Fatal(err)
}
type myNewConfigStruct struct {
Version string
// User string
Password string
Directories []string
}
mismatch := myNewConfigStruct{"1", "nopassword", []string{"Work", "documents", "Music"}}
newConfig, err := NewConfig(&mismatch, nil)
if err != nil {
t.Fatal(err)
}
fields, err := config.Diff(newConfig)
if err != nil {
t.Fatal(err)
}
if len(fields) != 1 {
t.Fatalf("Expected len 1, got %v", len(fields))
}
// Uncomment for debugging
// for i, field := range fields {
// fmt.Printf("Diff[%d]: %s=%v\n", i, field.Name(), field.Value())
// }
}
func TestDeepDiff(t *testing.T) {
type myStruct struct {
Version string
User string
Password string
Directories []string
}
saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
config, err := NewConfig(&saveMe, nil)
if err != nil {
t.Fatal(err)
}
mismatch := myStruct{"1", "Guest", "nopassword", []string{"Work", "documents", "Music"}}
newConfig, err := NewConfig(&mismatch, nil)
if err != nil {
t.Fatal(err)
}
fields, err := config.DeepDiff(newConfig)
if err != nil {
t.Fatal(err)
}
if len(fields) != 2 {
t.Fatalf("Expected len 2, got %v", len(fields))
}
// Uncomment for debugging
// for i, field := range fields {
// fmt.Printf("DeepDiff[%d]: %s=%v\n", i, field.Name(), field.Value())
// }
}

133
pkg/safe/safe.go
View File

@@ -1,133 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package safe
import (
"errors"
"io/ioutil"
"os"
"path/filepath"
)
// File represents safe file descriptor.
type File struct {
name string
tmpfile *os.File
closed bool
aborted bool
}
// Write writes len(b) bytes to the temporary File. In case of error, the temporary file is removed.
func (file *File) Write(b []byte) (n int, err error) {
if file.closed {
err = errors.New("write on closed file")
return
}
if file.aborted {
err = errors.New("write on aborted file")
return
}
defer func() {
if err != nil {
os.Remove(file.tmpfile.Name())
file.aborted = true
}
}()
n, err = file.tmpfile.Write(b)
return
}
// Close closes the temporary File and renames to the named file. In case of error, the temporary file is removed.
func (file *File) Close() (err error) {
defer func() {
if err != nil {
os.Remove(file.tmpfile.Name())
file.aborted = true
}
}()
if file.closed {
err = errors.New("close on closed file")
return
}
if file.aborted {
err = errors.New("close on aborted file")
return
}
if err = file.tmpfile.Close(); err != nil {
return
}
err = os.Rename(file.tmpfile.Name(), file.name)
file.closed = true
return
}
// Abort aborts the temporary File by closing and removing the temporary file.
func (file *File) Abort() (err error) {
if file.closed {
err = errors.New("abort on closed file")
return
}
if file.aborted {
err = errors.New("abort on aborted file")
return
}
file.tmpfile.Close()
err = os.Remove(file.tmpfile.Name())
file.aborted = true
return
}
// CreateFile creates the named file safely from unique temporary file.
// The temporary file is renamed to the named file upon successful close
// to safeguard intermediate state in the named file. The temporary file
// is created in the name of the named file with suffixed unique number
// and prefixed "$tmpfile" string. While creating the temporary file,
// missing parent directories are also created. The temporary file is
// removed if case of any intermediate failure. Not removed temporary
// files can be cleaned up by identifying them using "$tmpfile" prefix
// string.
func CreateFile(name string) (*File, error) {
// ioutil.TempFile() fails if parent directory is missing.
// Create parent directory to avoid such error.
dname := filepath.Dir(name)
if err := os.MkdirAll(dname, 0700); err != nil {
return nil, err
}
fname := filepath.Base(name)
tmpfile, err := ioutil.TempFile(dname, "$tmpfile."+fname+".")
if err != nil {
return nil, err
}
if err = os.Chmod(tmpfile.Name(), 0600); err != nil {
if rerr := os.Remove(tmpfile.Name()); rerr != nil {
err = rerr
}
return nil, err
}
return &File{name: name, tmpfile: tmpfile}, nil
}

193
pkg/safe/safe_test.go
View File

@@ -1,193 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package safe
import (
"io/ioutil"
"os"
"path"
"testing"
)
type MySuite struct {
root string
}
func (s *MySuite) SetUpSuite(t *testing.T) {
root, err := ioutil.TempDir(os.TempDir(), "safe_test.go.")
if err != nil {
t.Fatal(err)
}
s.root = root
}
func (s *MySuite) TearDownSuite(t *testing.T) {
err := os.RemoveAll(s.root)
if err != nil {
t.Fatal(err)
}
}
func TestSafeAbort(t *testing.T) {
s := &MySuite{}
s.SetUpSuite(t)
defer s.TearDownSuite(t)
f, err := CreateFile(path.Join(s.root, "testfile-abort"))
if err != nil {
t.Fatal(err)
}
_, err = os.Stat(path.Join(s.root, "testfile-abort"))
if !os.IsNotExist(err) {
t.Fatal(err)
}
err = f.Abort()
if err != nil {
t.Fatal(err)
}
err = f.Close()
if err != nil {
if err.Error() != "close on aborted file" {
t.Fatal(err)
}
}
}
func TestSafeClose(t *testing.T) {
s := &MySuite{}
s.SetUpSuite(t)
defer s.TearDownSuite(t)
f, err := CreateFile(path.Join(s.root, "testfile-close"))
if err != nil {
t.Fatal(err)
}
_, err = os.Stat(path.Join(s.root, "testfile-close"))
if !os.IsNotExist(err) {
t.Fatal(err)
}
err = f.Close()
if err != nil {
t.Fatal(err)
}
_, err = os.Stat(path.Join(s.root, "testfile-close"))
if err != nil {
t.Fatal(err)
}
err = os.Remove(path.Join(s.root, "testfile-close"))
if err != nil {
t.Fatal(err)
}
err = f.Abort()
if err != nil {
if err.Error() != "abort on closed file" {
t.Fatal(err)
}
}
}
func TestSafe(t *testing.T) {
s := &MySuite{}
s.SetUpSuite(t)
defer s.TearDownSuite(t)
f, err := CreateFile(path.Join(s.root, "testfile-safe"))
if err != nil {
t.Fatal(err)
}
_, err = os.Stat(path.Join(s.root, "testfile-safe"))
if !os.IsNotExist(err) {
t.Fatal(err)
}
err = f.Close()
if err != nil {
t.Fatal(err)
}
_, err = f.Write([]byte("Test"))
if err != nil {
if err.Error() != "write on closed file" {
t.Fatal(err)
}
}
err = f.Close()
if err != nil {
if err.Error() != "close on closed file" {
t.Fatal(err)
}
}
_, err = os.Stat(path.Join(s.root, "testfile-safe"))
if err != nil {
t.Fatal(err)
}
err = os.Remove(path.Join(s.root, "testfile-safe"))
if err != nil {
t.Fatal(err)
}
}
func TestSafeAbortWrite(t *testing.T) {
s := &MySuite{}
s.SetUpSuite(t)
defer s.TearDownSuite(t)
f, err := CreateFile(path.Join(s.root, "purgefile-abort"))
if err != nil {
t.Fatal(err)
}
_, err = os.Stat(path.Join(s.root, "purgefile-abort"))
if !os.IsNotExist(err) {
t.Fatal(err)
}
err = f.Abort()
if err != nil {
t.Fatal(err)
}
_, err = os.Stat(path.Join(s.root, "purgefile-abort"))
if !os.IsNotExist(err) {
t.Fatal(err)
}
err = f.Abort()
if err != nil {
if err.Error() != "abort on aborted file" {
t.Fatal(err)
}
}
_, err = f.Write([]byte("Test"))
if err != nil {
if err.Error() != "write on aborted file" {
t.Fatal(err)
}
}
}

41
pkg/sys/rlimit-file_bsd.go
View File

@@ -1,41 +0,0 @@
// +build freebsd dragonfly
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sys
import (
"syscall"
)
// GetMaxOpenFileLimit - returns maximum file descriptor number that can be opened by this process.
func GetMaxOpenFileLimit() (curLimit, maxLimit uint64, err error) {
var rlimit syscall.Rlimit
if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit); err == nil {
curLimit = uint64(rlimit.Cur)
maxLimit = uint64(rlimit.Max)
}
return curLimit, maxLimit, err
}
// SetMaxOpenFileLimit - sets maximum file descriptor number that can be opened by this process.
func SetMaxOpenFileLimit(curLimit, maxLimit uint64) error {
rlimit := syscall.Rlimit{Cur: int64(curLimit), Max: int64(curLimit)}
return syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlimit)
}

49
pkg/sys/rlimit-file_nix.go
View File

@@ -1,49 +0,0 @@
// +build linux darwin openbsd netbsd solaris
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sys
import (
"runtime"
"syscall"
)
// GetMaxOpenFileLimit - returns maximum file descriptor number that can be opened by this process.
func GetMaxOpenFileLimit() (curLimit, maxLimit uint64, err error) {
var rlimit syscall.Rlimit
if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit); err == nil {
curLimit = rlimit.Cur
maxLimit = rlimit.Max
}
return curLimit, maxLimit, err
}
// SetMaxOpenFileLimit - sets maximum file descriptor number that can be opened by this process.
func SetMaxOpenFileLimit(curLimit, maxLimit uint64) error {
if runtime.GOOS == "darwin" && curLimit > 10240 {
// The max file limit is 10240, even though
// the max returned by Getrlimit is 1<<63-1.
// This is OPEN_MAX in sys/syslimits.h.
// refer https://github.com/golang/go/issues/30401
curLimit = 10240
}
rlimit := syscall.Rlimit{Cur: curLimit, Max: maxLimit}
return syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlimit)
}

41
pkg/sys/rlimit-file_test.go
View File

@@ -1,41 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sys
import "testing"
// Test get max open file limit.
func TestGetMaxOpenFileLimit(t *testing.T) {
_, _, err := GetMaxOpenFileLimit()
if err != nil {
t.Errorf("expected: nil, got: %v", err)
}
}
// Test set open file limit
func TestSetMaxOpenFileLimit(t *testing.T) {
curLimit, maxLimit, err := GetMaxOpenFileLimit()
if err != nil {
t.Fatalf("Unable to get max open file limit. %v", err)
}
err = SetMaxOpenFileLimit(curLimit, maxLimit)
if err != nil {
t.Errorf("expected: nil, got: %v", err)
}
}

32
pkg/sys/rlimit-file_windows.go
View File

@@ -1,32 +0,0 @@
// +build windows
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sys
// GetMaxOpenFileLimit - returns maximum file descriptor number that can be opened by this process.
func GetMaxOpenFileLimit() (curLimit, maxLimit uint64, err error) {
// Nothing to do for windows.
return curLimit, maxLimit, err
}
// SetMaxOpenFileLimit - sets maximum file descriptor number that can be opened by this process.
func SetMaxOpenFileLimit(curLimit, maxLimit uint64) error {
// Nothing to do for windows.
return nil
}

39
pkg/sys/rlimit-memory_bsd.go
View File

@@ -1,39 +0,0 @@
// +build freebsd dragonfly
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sys
import "syscall"
// GetMaxMemoryLimit - returns the maximum size of the process's virtual memory (address space) in bytes.
func GetMaxMemoryLimit() (curLimit, maxLimit uint64, err error) {
var rlimit syscall.Rlimit
if err = syscall.Getrlimit(syscall.RLIMIT_DATA, &rlimit); err == nil {
curLimit = uint64(rlimit.Cur)
maxLimit = uint64(rlimit.Max)
}
return curLimit, maxLimit, err
}
// SetMaxMemoryLimit - sets the maximum size of the process's virtual memory (address space) in bytes.
func SetMaxMemoryLimit(curLimit, maxLimit uint64) error {
rlimit := syscall.Rlimit{Cur: int64(curLimit), Max: int64(maxLimit)}
return syscall.Setrlimit(syscall.RLIMIT_DATA, &rlimit)
}

39
pkg/sys/rlimit-memory_nix.go
View File

@@ -1,39 +0,0 @@
// +build linux darwin netbsd solaris
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sys
import "syscall"
// GetMaxMemoryLimit - returns the maximum size of the process's virtual memory (address space) in bytes.
func GetMaxMemoryLimit() (curLimit, maxLimit uint64, err error) {
var rlimit syscall.Rlimit
if err = syscall.Getrlimit(syscall.RLIMIT_AS, &rlimit); err == nil {
curLimit = rlimit.Cur
maxLimit = rlimit.Max
}
return curLimit, maxLimit, err
}
// SetMaxMemoryLimit - sets the maximum size of the process's virtual memory (address space) in bytes.
func SetMaxMemoryLimit(curLimit, maxLimit uint64) error {
rlimit := syscall.Rlimit{Cur: curLimit, Max: maxLimit}
return syscall.Setrlimit(syscall.RLIMIT_AS, &rlimit)
}

39
pkg/sys/rlimit-memory_openbsd.go
View File

@@ -1,39 +0,0 @@
// +build openbsd
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sys
import "syscall"
// GetMaxMemoryLimit - returns the maximum size of the process's virtual memory (address space) in bytes.
func GetMaxMemoryLimit() (curLimit, maxLimit uint64, err error) {
var rlimit syscall.Rlimit
if err = syscall.Getrlimit(syscall.RLIMIT_DATA, &rlimit); err == nil {
curLimit = rlimit.Cur
maxLimit = rlimit.Max
}
return curLimit, maxLimit, err
}
// SetMaxMemoryLimit - sets the maximum size of the process's virtual memory (address space) in bytes.
func SetMaxMemoryLimit(curLimit, maxLimit uint64) error {
rlimit := syscall.Rlimit{Cur: curLimit, Max: maxLimit}
return syscall.Setrlimit(syscall.RLIMIT_DATA, &rlimit)
}

41
pkg/sys/rlimit-memory_test.go
View File

@@ -1,41 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sys
import "testing"
// Test get max memory limit.
func TestGetMaxMemoryLimit(t *testing.T) {
_, _, err := GetMaxMemoryLimit()
if err != nil {
t.Errorf("expected: nil, got: %v", err)
}
}
// Test set memory limit
func TestSetMaxMemoryLimit(t *testing.T) {
curLimit, maxLimit, err := GetMaxMemoryLimit()
if err != nil {
t.Fatalf("Unable to get max memory limit. %v", err)
}
err = SetMaxMemoryLimit(curLimit, maxLimit)
if err != nil {
t.Errorf("expected: nil, got: %v", err)
}
}

32
pkg/sys/rlimit-memory_windows.go
View File

@@ -1,32 +0,0 @@
// +build windows
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sys
// GetMaxMemoryLimit - returns the maximum size of the process's virtual memory (address space) in bytes.
func GetMaxMemoryLimit() (curLimit, maxLimit uint64, err error) {
// Nothing to do for windows.
return curLimit, maxLimit, err
}
// SetMaxMemoryLimit - sets the maximum size of the process's virtual memory (address space) in bytes.
func SetMaxMemoryLimit(curLimit, maxLimit uint64) error {
// Nothing to do for windows.
return nil
}

23
pkg/sys/stats.go
View File

@@ -1,23 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sys
// Stats - system statistics.
type Stats struct {
TotalRAM uint64 // Physical RAM size in bytes,
}

46
pkg/sys/stats_bsd.go
View File

@@ -1,46 +0,0 @@
// +build openbsd freebsd dragonfly
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sys
import (
"encoding/binary"
"syscall"
)
func getHwPhysmem() (uint64, error) {
totalString, err := syscall.Sysctl("hw.physmem")
if err != nil {
return 0, err
}
// syscall.sysctl() helpfully assumes the result is a null-terminated string and
// removes the last byte of the result if it's 0 :/
totalString += "\x00"
total := uint64(binary.LittleEndian.Uint64([]byte(totalString)))
return total, nil
}
// GetStats - return system statistics for bsd.
func GetStats() (stats Stats, err error) {
stats.TotalRAM, err = getHwPhysmem()
return stats, err
}

44
pkg/sys/stats_darwin.go
View File

@@ -1,44 +0,0 @@
// +build darwin
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sys
import (
"encoding/binary"
"syscall"
)
func getHwMemsize() (uint64, error) {
totalString, err := syscall.Sysctl("hw.memsize")
if err != nil {
return 0, err
}
// syscall.sysctl() helpfully assumes the result is a null-terminated string and
// removes the last byte of the result if it's 0 :/
totalString += "\x00"
return binary.LittleEndian.Uint64([]byte(totalString)), nil
}
// GetStats - return system statistics for macOS.
func GetStats() (stats Stats, err error) {
stats.TotalRAM, err = getHwMemsize()
return stats, err
}

88
pkg/sys/stats_linux.go
View File

@@ -1,88 +0,0 @@
// +build linux,!arm,!386
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sys
import (
"os"
"syscall"
"github.com/minio/minio/pkg/cgroup"
)
// Get the final system memory limit chosen by the user.
// by default without any configuration on a vanilla Linux
// system you would see physical RAM limit. If cgroup
// is configured at some point in time this function
// would return the memory limit chosen for the given pid.
func getMemoryLimit() (sysLimit uint64, err error) {
if sysLimit, err = getSysinfoMemoryLimit(); err != nil {
// Physical memory info is not accessible, just exit here.
return 0, err
}
// Following code is deliberately ignoring the error.
cGroupLimit, gerr := cgroup.GetMemoryLimit(os.Getpid())
if gerr != nil {
// Upon error just return system limit.
return sysLimit, nil
}
// cgroup limit is lesser than system limit means
// user wants to limit the memory usage further
// treat cgroup limit as the system limit.
if cGroupLimit <= sysLimit {
sysLimit = cGroupLimit
}
// Final system limit.
return sysLimit, nil
}
// Get physical RAM size of the node.
func getSysinfoMemoryLimit() (limit uint64, err error) {
var si syscall.Sysinfo_t
if err = syscall.Sysinfo(&si); err != nil {
return 0, err
}
// Some fields in syscall.Sysinfo_t have different integer sizes
// in different platform architectures. Cast all fields to uint64.
unit := si.Unit
totalRAM := si.Totalram
// Total RAM is always the multiplicative value
// of unit size and total ram.
//nolint:unconvert
return uint64(unit) * uint64(totalRAM), nil
}
// GetStats - return system statistics, currently only
// supported value is TotalRAM.
func GetStats() (stats Stats, err error) {
var limit uint64
limit, err = getMemoryLimit()
if err != nil {
return Stats{}, err
}
stats.TotalRAM = limit
return stats, nil
}

87
pkg/sys/stats_linux_32bit.go
View File

@@ -1,87 +0,0 @@
// +build linux,arm linux,386
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sys
import (
"os"
"syscall"
"github.com/minio/minio/pkg/cgroup"
)
// Get the final system memory limit chosen by the user.
// by default without any configuration on a vanilla Linux
// system you would see physical RAM limit. If cgroup
// is configured at some point in time this function
// would return the memory limit chosen for the given pid.
func getMemoryLimit() (sysLimit uint64, err error) {
if sysLimit, err = getSysinfoMemoryLimit(); err != nil {
// Physical memory info is not accessible, just exit here.
return 0, err
}
// Following code is deliberately ignoring the error.
cGroupLimit, gerr := cgroup.GetMemoryLimit(os.Getpid())
if gerr != nil {
// Upon error just return system limit.
return sysLimit, nil
}
// cgroup limit is lesser than system limit means
// user wants to limit the memory usage further
// treat cgroup limit as the system limit.
if cGroupLimit <= sysLimit {
sysLimit = cGroupLimit
}
// Final system limit.
return sysLimit, nil
}
// Get physical RAM size of the node.
func getSysinfoMemoryLimit() (limit uint64, err error) {
var si syscall.Sysinfo_t
if err = syscall.Sysinfo(&si); err != nil {
return 0, err
}
// Some fields in syscall.Sysinfo_t have different integer sizes
// in different platform architectures. Cast all fields to uint64.
unit := si.Unit
totalRAM := si.Totalram
// Total RAM is always the multiplicative value
// of unit size and total ram.
return uint64(unit) * uint64(totalRAM), nil
}
// GetStats - return system statistics, currently only
// supported value is TotalRAM.
func GetStats() (stats Stats, err error) {
var limit uint64
limit, err = getMemoryLimit()
if err != nil {
return Stats{}, err
}
stats.TotalRAM = limit
return stats, nil
}

46
pkg/sys/stats_netbsd.go
View File

@@ -1,46 +0,0 @@
// +build netbsd
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sys
import (
"encoding/binary"
"syscall"
)
func getHwPhysmem() (uint64, error) {
totalString, err := syscall.Sysctl("hw.physmem64")
if err != nil {
return 0, err
}
// syscall.sysctl() helpfully assumes the result is a null-terminated string and
// removes the last byte of the result if it's 0 :/
totalString += "\x00"
total := uint64(binary.LittleEndian.Uint64([]byte(totalString)))
return total, nil
}
// GetStats - return system statistics for bsd.
func GetStats() (stats Stats, err error) {
stats.TotalRAM, err = getHwPhysmem()
return stats, err
}

26
pkg/sys/stats_solaris.go
View File

@@ -1,26 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sys
import "errors"
// GetStats - stub implementation for Solaris, this will not give us
// complete functionality but will enable fs setups on Solaris.
func GetStats() (stats Stats, err error) {
return Stats{}, errors.New("Not implemented")
}

31
pkg/sys/stats_test.go
View File

@@ -1,31 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sys
import "testing"
// Test get stats result.
func TestGetStats(t *testing.T) {
stats, err := GetStats()
if err != nil {
t.Errorf("Tests: Expected `nil`, Got %s", err)
}
if stats.TotalRAM == 0 {
t.Errorf("Tests: Expected `n > 0`, Got %d", stats.TotalRAM)
}
}

57
pkg/sys/stats_windows.go
View File

@@ -1,57 +0,0 @@
// +build windows
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sys
import (
"syscall"
"unsafe"
"golang.org/x/sys/windows"
)
var (
modkernel32 = windows.NewLazySystemDLL("kernel32.dll")
procGlobalMemoryStatusEx = modkernel32.NewProc("GlobalMemoryStatusEx")
)
type memoryStatusEx struct {
cbSize uint32
dwMemoryLoad uint32
ullTotalPhys uint64 // in bytes
ullAvailPhys uint64
ullTotalPageFile uint64
ullAvailPageFile uint64
ullTotalVirtual uint64
ullAvailVirtual uint64
ullAvailExtendedVirtual uint64
}
// GetStats - return system statistics for windows.
func GetStats() (stats Stats, err error) {
var memInfo memoryStatusEx
memInfo.cbSize = uint32(unsafe.Sizeof(memInfo))
if mem, _, _ := procGlobalMemoryStatusEx.Call(uintptr(unsafe.Pointer(&memInfo))); mem == 0 {
err = syscall.GetLastError()
} else {
stats.TotalRAM = memInfo.ullTotalPhys
}
return stats, err
}

39
pkg/sys/threads.go
View File

@@ -1,39 +0,0 @@
// +build linux
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sys
import (
"io/ioutil"
"strconv"
"strings"
)
// GetMaxThreads returns the maximum number of threads that the system can create.
func GetMaxThreads() (int, error) {
sysMaxThreadsStr, err := ioutil.ReadFile("/proc/sys/kernel/threads-max")
if err != nil {
return 0, err
}
sysMaxThreads, err := strconv.Atoi(strings.TrimSpace(string(sysMaxThreadsStr)))
if err != nil {
return 0, err
}
return sysMaxThreads, nil
}

27
pkg/sys/threads_other.go
View File

@@ -1,27 +0,0 @@
// +build !linux
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sys
import "errors"
// GetMaxThreads returns the maximum number of threads that the system can create.
func GetMaxThreads() (int, error) {
return 0, errors.New("getting max threads is not supported")
}

119
pkg/trie/trie.go
View File

@@ -1,119 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Package trie implements a simple trie tree for minio server/tools borrows
// idea from - https://godoc.org/golang.org/x/text/internal/triegen.
package trie
// Node trie tree node container carries value and children.
type Node struct {
exists bool
value string
child map[rune]*Node // runes as child.
}
// newNode create a new trie node.
func newNode() *Node {
return &Node{
exists: false,
value: "",
child: make(map[rune]*Node),
}
}
// Trie is a trie container.
type Trie struct {
root *Node
size int
}
// Root returns root node.
func (t *Trie) Root() *Node {
return t.root
}
// Insert insert a key.
func (t *Trie) Insert(key string) {
curNode := t.root
for _, v := range key {
if curNode.child[v] == nil {
curNode.child[v] = newNode()
}
curNode = curNode.child[v]
}
if !curNode.exists {
// increment when new rune child is added.
t.size++
curNode.exists = true
}
// value is stored for retrieval in future.
curNode.value = key
}
// PrefixMatch - prefix match.
func (t *Trie) PrefixMatch(key string) []string {
node, _ := t.findNode(key)
if node == nil {
return nil
}
return t.Walk(node)
}
// Walk the tree.
func (t *Trie) Walk(node *Node) (ret []string) {
if node.exists {
ret = append(ret, node.value)
}
for _, v := range node.child {
ret = append(ret, t.Walk(v)...)
}
return
}
// find nodes corresponding to key.
func (t *Trie) findNode(key string) (node *Node, index int) {
curNode := t.root
f := false
for k, v := range key {
if f {
index = k
f = false
}
if curNode.child[v] == nil {
return nil, index
}
curNode = curNode.child[v]
if curNode.exists {
f = true
}
}
if curNode.exists {
index = len(key)
}
return curNode, index
}
// NewTrie create a new trie.
func NewTrie() *Trie {
return &Trie{
root: newNode(),
size: 0,
}
}

69
pkg/trie/trie_test.go
View File

@@ -1,69 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package trie
import (
"testing"
)
// Simply make sure creating a new tree works.
func TestNewTrie(t *testing.T) {
trie := NewTrie()
if trie.size != 0 {
t.Errorf("expected size 0, got: %d", trie.size)
}
}
// Ensure that we can insert new keys into the tree, then check the size.
func TestInsert(t *testing.T) {
trie := NewTrie()
// We need to have an empty tree to begin with.
if trie.size != 0 {
t.Errorf("expected size 0, got: %d", trie.size)
}
trie.Insert("key")
trie.Insert("keyy")
// After inserting, we should have a size of two.
if trie.size != 2 {
t.Errorf("expected size 2, got: %d", trie.size)
}
}
// Ensure that PrefixMatch gives us the correct two keys in the tree.
func TestPrefixMatch(t *testing.T) {
trie := NewTrie()
// Feed it some fodder: only 'minio' and 'miny-os' should trip the matcher.
trie.Insert("minio")
trie.Insert("amazon")
trie.Insert("cheerio")
trie.Insert("miny-o's")
matches := trie.PrefixMatch("min")
if len(matches) != 2 {
t.Errorf("expected two matches, got: %d", len(matches))
}
if matches[0] != "minio" && matches[1] != "minio" {
t.Errorf("expected one match to be 'minio', got: '%s' and '%s'", matches[0], matches[1])
}
}

68
pkg/wildcard/match.go
View File

@@ -1,68 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package wildcard
// MatchSimple - finds whether the text matches/satisfies the pattern string.
// supports only '*' wildcard in the pattern.
// considers a file system path as a flat name space.
func MatchSimple(pattern, name string) bool {
if pattern == "" {
return name == pattern
}
if pattern == "*" {
return true
}
// Does only wildcard '*' match.
return deepMatchRune([]rune(name), []rune(pattern), true)
}
// Match - finds whether the text matches/satisfies the pattern string.
// supports '*' and '?' wildcards in the pattern string.
// unlike path.Match(), considers a path as a flat name space while matching the pattern.
// The difference is illustrated in the example here https://play.golang.org/p/Ega9qgD4Qz .
func Match(pattern, name string) (matched bool) {
if pattern == "" {
return name == pattern
}
if pattern == "*" {
return true
}
// Does extended wildcard '*' and '?' match.
return deepMatchRune([]rune(name), []rune(pattern), false)
}
func deepMatchRune(str, pattern []rune, simple bool) bool {
for len(pattern) > 0 {
switch pattern[0] {
default:
if len(str) == 0 || str[0] != pattern[0] {
return false
}
case '?':
if len(str) == 0 && !simple {
return false
}
case '*':
return deepMatchRune(str, pattern[1:], simple) ||
(len(str) > 0 && deepMatchRune(str[1:], pattern, simple))
}
str = str[1:]
pattern = pattern[1:]
}
return len(str) == 0 && len(pattern) == 0
}

543
pkg/wildcard/match_test.go
View File

@@ -1,543 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package wildcard_test
import (
"testing"
"github.com/minio/minio/pkg/wildcard"
)
// TestMatch - Tests validate the logic of wild card matching.
// `Match` supports '*' and '?' wildcards.
// Sample usage: In resource matching for bucket policy validation.
func TestMatch(t *testing.T) {
testCases := []struct {
pattern string
text string
matched bool
}{
// Test case - 1.
// Test case with pattern "*". Expected to match any text.
{
pattern: "*",
text: "s3:GetObject",
matched: true,
},
// Test case - 2.
// Test case with empty pattern. This only matches empty string.
{
pattern: "",
text: "s3:GetObject",
matched: false,
},
// Test case - 3.
// Test case with empty pattern. This only matches empty string.
{
pattern: "",
text: "",
matched: true,
},
// Test case - 4.
// Test case with single "*" at the end.
{
pattern: "s3:*",
text: "s3:ListMultipartUploadParts",
matched: true,
},
// Test case - 5.
// Test case with a no "*". In this case the pattern and text should be the same.
{
pattern: "s3:ListBucketMultipartUploads",
text: "s3:ListBucket",
matched: false,
},
// Test case - 6.
// Test case with a no "*". In this case the pattern and text should be the same.
{
pattern: "s3:ListBucket",
text: "s3:ListBucket",
matched: true,
},
// Test case - 7.
// Test case with a no "*". In this case the pattern and text should be the same.
{
pattern: "s3:ListBucketMultipartUploads",
text: "s3:ListBucketMultipartUploads",
matched: true,
},
// Test case - 8.
// Test case with pattern containing key name with a prefix. Should accept the same text without a "*".
{
pattern: "my-bucket/oo*",
text: "my-bucket/oo",
matched: true,
},
// Test case - 9.
// Test case with "*" at the end of the pattern.
{
pattern: "my-bucket/In*",
text: "my-bucket/India/Karnataka/",
matched: true,
},
// Test case - 10.
// Test case with prefixes shuffled.
// This should fail.
{
pattern: "my-bucket/In*",
text: "my-bucket/Karnataka/India/",
matched: false,
},
// Test case - 11.
// Test case with text expanded to the wildcards in the pattern.
{
pattern: "my-bucket/In*/Ka*/Ban",
text: "my-bucket/India/Karnataka/Ban",
matched: true,
},
// Test case - 12.
// Test case with the keyname part is repeated as prefix several times.
// This is valid.
{
pattern: "my-bucket/In*/Ka*/Ban",
text: "my-bucket/India/Karnataka/Ban/Ban/Ban/Ban/Ban",
matched: true,
},
// Test case - 13.
// Test case to validate that `*` can be expanded into multiple prefixes.
{
pattern: "my-bucket/In*/Ka*/Ban",
text: "my-bucket/India/Karnataka/Area1/Area2/Area3/Ban",
matched: true,
},
// Test case - 14.
// Test case to validate that `*` can be expanded into multiple prefixes.
{
pattern: "my-bucket/In*/Ka*/Ban",
text: "my-bucket/India/State1/State2/Karnataka/Area1/Area2/Area3/Ban",
matched: true,
},
// Test case - 15.
// Test case where the keyname part of the pattern is expanded in the text.
{
pattern: "my-bucket/In*/Ka*/Ban",
text: "my-bucket/India/Karnataka/Bangalore",
matched: false,
},
// Test case - 16.
// Test case with prefixes and wildcard expanded for all "*".
{
pattern: "my-bucket/In*/Ka*/Ban*",
text: "my-bucket/India/Karnataka/Bangalore",
matched: true,
},
// Test case - 17.
// Test case with keyname part being a wildcard in the pattern.
{
pattern: "my-bucket/*",
text: "my-bucket/India",
matched: true,
},
// Test case - 18.
{
pattern: "my-bucket/oo*",
text: "my-bucket/odo",
matched: false,
},
// Test case with pattern containing wildcard '?'.
// Test case - 19.
// "my-bucket?/" matches "my-bucket1/", "my-bucket2/", "my-bucket3" etc...
// doesn't match "mybucket/".
{
pattern: "my-bucket?/abc*",
text: "mybucket/abc",
matched: false,
},
// Test case - 20.
{
pattern: "my-bucket?/abc*",
text: "my-bucket1/abc",
matched: true,
},
// Test case - 21.
{
pattern: "my-?-bucket/abc*",
text: "my--bucket/abc",
matched: false,
},
// Test case - 22.
{
pattern: "my-?-bucket/abc*",
text: "my-1-bucket/abc",
matched: true,
},
// Test case - 23.
{
pattern: "my-?-bucket/abc*",
text: "my-k-bucket/abc",
matched: true,
},
// Test case - 24.
{
pattern: "my??bucket/abc*",
text: "mybucket/abc",
matched: false,
},
// Test case - 25.
{
pattern: "my??bucket/abc*",
text: "my4abucket/abc",
matched: true,
},
// Test case - 26.
{
pattern: "my-bucket?abc*",
text: "my-bucket/abc",
matched: true,
},
// Test case 27-28.
// '?' matches '/' too. (works with s3).
// This is because the namespace is considered flat.
// "abc?efg" matches both "abcdefg" and "abc/efg".
{
pattern: "my-bucket/abc?efg",
text: "my-bucket/abcdefg",
matched: true,
},
{
pattern: "my-bucket/abc?efg",
text: "my-bucket/abc/efg",
matched: true,
},
// Test case - 29.
{
pattern: "my-bucket/abc????",
text: "my-bucket/abc",
matched: false,
},
// Test case - 30.
{
pattern: "my-bucket/abc????",
text: "my-bucket/abcde",
matched: false,
},
// Test case - 31.
{
pattern: "my-bucket/abc????",
text: "my-bucket/abcdefg",
matched: true,
},
// Test case 32-34.
// test case with no '*'.
{
pattern: "my-bucket/abc?",
text: "my-bucket/abc",
matched: false,
},
{
pattern: "my-bucket/abc?",
text: "my-bucket/abcd",
matched: true,
},
{
pattern: "my-bucket/abc?",
text: "my-bucket/abcde",
matched: false,
},
// Test case 35.
{
pattern: "my-bucket/mnop*?",
text: "my-bucket/mnop",
matched: false,
},
// Test case 36.
{
pattern: "my-bucket/mnop*?",
text: "my-bucket/mnopqrst/mnopqr",
matched: true,
},
// Test case 37.
{
pattern: "my-bucket/mnop*?",
text: "my-bucket/mnopqrst/mnopqrs",
matched: true,
},
// Test case 38.
{
pattern: "my-bucket/mnop*?",
text: "my-bucket/mnop",
matched: false,
},
// Test case 39.
{
pattern: "my-bucket/mnop*?",
text: "my-bucket/mnopq",
matched: true,
},
// Test case 40.
{
pattern: "my-bucket/mnop*?",
text: "my-bucket/mnopqr",
matched: true,
},
// Test case 41.
{
pattern: "my-bucket/mnop*?and",
text: "my-bucket/mnopqand",
matched: true,
},
// Test case 42.
{
pattern: "my-bucket/mnop*?and",
text: "my-bucket/mnopand",
matched: false,
},
// Test case 43.
{
pattern: "my-bucket/mnop*?and",
text: "my-bucket/mnopqand",
matched: true,
},
// Test case 44.
{
pattern: "my-bucket/mnop*?",
text: "my-bucket/mn",
matched: false,
},
// Test case 45.
{
pattern: "my-bucket/mnop*?",
text: "my-bucket/mnopqrst/mnopqrs",
matched: true,
},
// Test case 46.
{
pattern: "my-bucket/mnop*??",
text: "my-bucket/mnopqrst",
matched: true,
},
// Test case 47.
{
pattern: "my-bucket/mnop*qrst",
text: "my-bucket/mnopabcdegqrst",
matched: true,
},
// Test case 48.
{
pattern: "my-bucket/mnop*?and",
text: "my-bucket/mnopqand",
matched: true,
},
// Test case 49.
{
pattern: "my-bucket/mnop*?and",
text: "my-bucket/mnopand",
matched: false,
},
// Test case 50.
{
pattern: "my-bucket/mnop*?and?",
text: "my-bucket/mnopqanda",
matched: true,
},
// Test case 51.
{
pattern: "my-bucket/mnop*?and",
text: "my-bucket/mnopqanda",
matched: false,
},
// Test case 52.
{
pattern: "my-?-bucket/abc*",
text: "my-bucket/mnopqanda",
matched: false,
},
}
// Iterating over the test cases, call the function under test and asert the output.
for i, testCase := range testCases {
actualResult := wildcard.Match(testCase.pattern, testCase.text)
if testCase.matched != actualResult {
t.Errorf("Test %d: Expected the result to be `%v`, but instead found it to be `%v`", i+1, testCase.matched, actualResult)
}
}
}
// TestMatchSimple - Tests validate the logic of wild card matching.
// `MatchSimple` supports matching for only '*' in the pattern string.
func TestMatchSimple(t *testing.T) {
testCases := []struct {
pattern string
text string
matched bool
}{
// Test case - 1.
// Test case with pattern "*". Expected to match any text.
{
pattern: "*",
text: "s3:GetObject",
matched: true,
},
// Test case - 2.
// Test case with empty pattern. This only matches empty string.
{
pattern: "",
text: "s3:GetObject",
matched: false,
},
// Test case - 3.
// Test case with empty pattern. This only matches empty string.
{
pattern: "",
text: "",
matched: true,
},
// Test case - 4.
// Test case with single "*" at the end.
{
pattern: "s3:*",
text: "s3:ListMultipartUploadParts",
matched: true,
},
// Test case - 5.
// Test case with a no "*". In this case the pattern and text should be the same.
{
pattern: "s3:ListBucketMultipartUploads",
text: "s3:ListBucket",
matched: false,
},
// Test case - 6.
// Test case with a no "*". In this case the pattern and text should be the same.
{
pattern: "s3:ListBucket",
text: "s3:ListBucket",
matched: true,
},
// Test case - 7.
// Test case with a no "*". In this case the pattern and text should be the same.
{
pattern: "s3:ListBucketMultipartUploads",
text: "s3:ListBucketMultipartUploads",
matched: true,
},
// Test case - 8.
// Test case with pattern containing key name with a prefix. Should accept the same text without a "*".
{
pattern: "my-bucket/oo*",
text: "my-bucket/oo",
matched: true,
},
// Test case - 9.
// Test case with "*" at the end of the pattern.
{
pattern: "my-bucket/In*",
text: "my-bucket/India/Karnataka/",
matched: true,
},
// Test case - 10.
// Test case with prefixes shuffled.
// This should fail.
{
pattern: "my-bucket/In*",
text: "my-bucket/Karnataka/India/",
matched: false,
},
// Test case - 11.
// Test case with text expanded to the wildcards in the pattern.
{
pattern: "my-bucket/In*/Ka*/Ban",
text: "my-bucket/India/Karnataka/Ban",
matched: true,
},
// Test case - 12.
// Test case with the keyname part is repeated as prefix several times.
// This is valid.
{
pattern: "my-bucket/In*/Ka*/Ban",
text: "my-bucket/India/Karnataka/Ban/Ban/Ban/Ban/Ban",
matched: true,
},
// Test case - 13.
// Test case to validate that `*` can be expanded into multiple prefixes.
{
pattern: "my-bucket/In*/Ka*/Ban",
text: "my-bucket/India/Karnataka/Area1/Area2/Area3/Ban",
matched: true,
},
// Test case - 14.
// Test case to validate that `*` can be expanded into multiple prefixes.
{
pattern: "my-bucket/In*/Ka*/Ban",
text: "my-bucket/India/State1/State2/Karnataka/Area1/Area2/Area3/Ban",
matched: true,
},
// Test case - 15.
// Test case where the keyname part of the pattern is expanded in the text.
{
pattern: "my-bucket/In*/Ka*/Ban",
text: "my-bucket/India/Karnataka/Bangalore",
matched: false,
},
// Test case - 16.
// Test case with prefixes and wildcard expanded for all "*".
{
pattern: "my-bucket/In*/Ka*/Ban*",
text: "my-bucket/India/Karnataka/Bangalore",
matched: true,
},
// Test case - 17.
// Test case with keyname part being a wildcard in the pattern.
{
pattern: "my-bucket/*",
text: "my-bucket/India",
matched: true,
},
// Test case - 18.
{
pattern: "my-bucket/oo*",
text: "my-bucket/odo",
matched: false,
},
// Test case - 11.
{
pattern: "my-bucket/oo?*",
text: "my-bucket/oo???",
matched: true,
},
// Test case - 12:
{
pattern: "my-bucket/oo??*",
text: "my-bucket/odo",
matched: false,
},
// Test case - 13:
{
pattern: "?h?*",
text: "?h?hello",
matched: true,
},
}
// Iterating over the test cases, call the function under test and asert the output.
for i, testCase := range testCases {
actualResult := wildcard.MatchSimple(testCase.pattern, testCase.text)
if testCase.matched != actualResult {
t.Errorf("Test %d: Expected the result to be `%v`, but instead found it to be `%v`", i+1, testCase.matched, actualResult)
}
}
}

65
pkg/words/damerau-levenshtein.go
View File

@@ -1,65 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package words
import "math"
// Returns the minimum value of a slice of integers
func minimum(integers []int) (minVal int) {
minVal = math.MaxInt32
for _, v := range integers {
if v < minVal {
minVal = v
}
}
return
}
// DamerauLevenshteinDistance calculates distance between two strings using an algorithm
// described in https://en.wikipedia.org/wiki/Damerau-Levenshtein_distance
func DamerauLevenshteinDistance(a string, b string) int {
var cost int
d := make([][]int, len(a)+1)
for i := 1; i <= len(a)+1; i++ {
d[i-1] = make([]int, len(b)+1)
}
for i := 0; i <= len(a); i++ {
d[i][0] = i
}
for j := 0; j <= len(b); j++ {
d[0][j] = j
}
for i := 1; i <= len(a); i++ {
for j := 1; j <= len(b); j++ {
if a[i-1] == b[j-1] {
cost = 0
} else {
cost = 1
}
d[i][j] = minimum([]int{
d[i-1][j] + 1,
d[i][j-1] + 1,
d[i-1][j-1] + cost,
})
if i > 1 && j > 1 && a[i-1] == b[j-2] && a[i-2] == b[j-1] {
d[i][j] = minimum([]int{d[i][j], d[i-2][j-2] + cost}) // transposition
}
}
}
return d[len(a)][len(b)]
}

66
pkg/words/damerau-levenshtein_test.go
View File

@@ -1,66 +0,0 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package words
import (
"math"
"testing"
)
// Test minimum function which calculates the minimal value in a list of integers
func TestMinimum(t *testing.T) {
type testCase struct {
listval []int
expected int
}
testCases := []testCase{
{listval: []int{3, 4, 15}, expected: 3},
{listval: []int{}, expected: math.MaxInt32},
}
// Validate all the test cases.
for i, tt := range testCases {
val := minimum(tt.listval)
if val != tt.expected {
t.Errorf("Test %d:, Expected %d, got %d", i+1, tt.expected, val)
}
}
}
// Test DamerauLevenshtein which calculates the difference distance between two words
func TestDamerauLevenshtein(t *testing.T) {
type testCase struct {
word1 string
word2 string
distance int
}
testCases := []testCase{
{word1: "", word2: "", distance: 0},
{word1: "a", word2: "a", distance: 0},
{word1: "a", word2: "b", distance: 1},
{word1: "rm", word2: "tm", distance: 1},
{word1: "version", word2: "evrsion", distance: 1},
{word1: "version", word2: "bersio", distance: 2},
}
// Validate all the test cases.
for i, tt := range testCases {
d := DamerauLevenshteinDistance(tt.word1, tt.word2)
if d != tt.distance {
t.Errorf("Test %d:, Expected %d, got %d", i+1, tt.distance, d)
}
}
}