Initialize configs correctly, move notification config (#8367)

This PR also removes deprecated tests, adds checks
to avoid races reproduced on CI/CD.
This commit is contained in:
Harshavardhana 2019-10-08 23:11:15 -07:00 committed by kannappanr
parent d2a8be6fc2
commit 6a4ef2e48e
15 changed files with 149 additions and 321 deletions

View File

@ -31,7 +31,7 @@ matrix:
- make
- diff -au <(gofmt -s -d cmd) <(printf "")
- diff -au <(gofmt -s -d pkg) <(printf "")
- for d in $(go list ./... | grep -v browser); do CGO_ENABLED=1 go test -v -race --timeout 15m "$d"; done
- for d in $(go list ./... | grep -v browser); do CGO_ENABLED=1 go test -v -race --timeout 20m "$d"; done
- make verifiers
- make crosscompile
- make verify

View File

@ -64,10 +64,12 @@ test: verifiers build
@echo "Running unit tests"
@GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null
# Verify minio binary, enable races as well
# Verify minio binary
# TODO: enable races as well
# @GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
verify:
@echo "Verifying build"
@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
@GO111MODULE=on CGO_ENABLED=1 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
@(env bash $(PWD)/buildscripts/verify-build.sh)
coverage: build

View File

@ -20,7 +20,6 @@ import (
"context"
"errors"
"fmt"
"reflect"
"strings"
"sync"
@ -28,6 +27,7 @@ import (
"github.com/minio/minio/cmd/config/cache"
"github.com/minio/minio/cmd/config/compress"
xldap "github.com/minio/minio/cmd/config/ldap"
"github.com/minio/minio/cmd/config/notify"
"github.com/minio/minio/cmd/config/storageclass"
"github.com/minio/minio/cmd/crypto"
xhttp "github.com/minio/minio/cmd/http"
@ -465,56 +465,6 @@ func (s *serverConfig) TestNotificationTargets() error {
return nil
}
// Returns the string describing a difference with the given
// configuration object. If the given configuration object is
// identical, an empty string is returned.
func (s *serverConfig) ConfigDiff(t *serverConfig) string {
switch {
case t == nil:
return "Given configuration is empty"
case s.Credential != t.Credential:
return "Credential configuration differs"
case s.Region != t.Region:
return "Region configuration differs"
case s.StorageClass != t.StorageClass:
return "StorageClass configuration differs"
case !reflect.DeepEqual(s.Cache, t.Cache):
return "Cache configuration differs"
case !reflect.DeepEqual(s.Compression, t.Compression):
return "Compression configuration differs"
case !reflect.DeepEqual(s.Notify.AMQP, t.Notify.AMQP):
return "AMQP Notification configuration differs"
case !reflect.DeepEqual(s.Notify.NATS, t.Notify.NATS):
return "NATS Notification configuration differs"
case !reflect.DeepEqual(s.Notify.NSQ, t.Notify.NSQ):
return "NSQ Notification configuration differs"
case !reflect.DeepEqual(s.Notify.Elasticsearch, t.Notify.Elasticsearch):
return "ElasticSearch Notification configuration differs"
case !reflect.DeepEqual(s.Notify.Redis, t.Notify.Redis):
return "Redis Notification configuration differs"
case !reflect.DeepEqual(s.Notify.PostgreSQL, t.Notify.PostgreSQL):
return "PostgreSQL Notification configuration differs"
case !reflect.DeepEqual(s.Notify.Kafka, t.Notify.Kafka):
return "Kafka Notification configuration differs"
case !reflect.DeepEqual(s.Notify.Webhook, t.Notify.Webhook):
return "Webhook Notification configuration differs"
case !reflect.DeepEqual(s.Notify.MySQL, t.Notify.MySQL):
return "MySQL Notification configuration differs"
case !reflect.DeepEqual(s.Notify.MQTT, t.Notify.MQTT):
return "MQTT Notification configuration differs"
case !reflect.DeepEqual(s.Logger, t.Logger):
return "Logger configuration differs"
case !reflect.DeepEqual(s.KMS, t.KMS):
return "KMS configuration differs"
case reflect.DeepEqual(s, t):
return ""
default:
// This case will not happen unless this comparison
// function has become stale.
return "Configuration differs"
}
}
func newServerConfig() *serverConfig {
cred, err := auth.GetNewCredentials()
logger.FatalIf(err, "")
@ -534,47 +484,15 @@ func newServerConfig() *serverConfig {
MaxUse: globalCacheMaxUse,
},
KMS: crypto.KMSConfig{},
Notify: notifier{},
Notify: notify.NewConfig(),
Compression: compress.Config{
Enabled: false,
Extensions: globalCompressExtensions,
MimeTypes: globalCompressMimeTypes,
},
Logger: logger.NewConfig(),
}
// Make sure to initialize notification configs.
srvCfg.Notify.AMQP = make(map[string]target.AMQPArgs)
srvCfg.Notify.AMQP["1"] = target.AMQPArgs{}
srvCfg.Notify.MQTT = make(map[string]target.MQTTArgs)
srvCfg.Notify.MQTT["1"] = target.MQTTArgs{}
srvCfg.Notify.Elasticsearch = make(map[string]target.ElasticsearchArgs)
srvCfg.Notify.Elasticsearch["1"] = target.ElasticsearchArgs{}
srvCfg.Notify.Redis = make(map[string]target.RedisArgs)
srvCfg.Notify.Redis["1"] = target.RedisArgs{}
srvCfg.Notify.NATS = make(map[string]target.NATSArgs)
srvCfg.Notify.NATS["1"] = target.NATSArgs{}
srvCfg.Notify.NSQ = make(map[string]target.NSQArgs)
srvCfg.Notify.NSQ["1"] = target.NSQArgs{}
srvCfg.Notify.PostgreSQL = make(map[string]target.PostgreSQLArgs)
srvCfg.Notify.PostgreSQL["1"] = target.PostgreSQLArgs{}
srvCfg.Notify.MySQL = make(map[string]target.MySQLArgs)
srvCfg.Notify.MySQL["1"] = target.MySQLArgs{}
srvCfg.Notify.Kafka = make(map[string]target.KafkaArgs)
srvCfg.Notify.Kafka["1"] = target.KafkaArgs{}
srvCfg.Notify.Webhook = make(map[string]target.WebhookArgs)
srvCfg.Notify.Webhook["1"] = target.WebhookArgs{}
srvCfg.Cache.Drives = make([]string, 0)
srvCfg.Cache.Exclude = make([]string, 0)
srvCfg.Cache.Expiry = globalCacheExpiry
srvCfg.Cache.MaxUse = globalCacheMaxUse
// Console logging is on by default
srvCfg.Logger.Console.Enabled = true
// Create an example of HTTP logger
srvCfg.Logger.HTTP = make(map[string]logger.HTTP)
srvCfg.Logger.HTTP["target1"] = logger.HTTP{Endpoint: "https://username:password@example.com/api"}
return srvCfg
}

View File

@ -21,11 +21,6 @@ import (
"os"
"path"
"testing"
"github.com/minio/minio/cmd/config/storageclass"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/event/target"
)
func TestServerConfig(t *testing.T) {
@ -182,128 +177,3 @@ func TestValidateConfig(t *testing.T) {
}
}
func TestConfigDiff(t *testing.T) {
testCases := []struct {
s, t *serverConfig
diff string
}{
// 1
{&serverConfig{}, nil, "Given configuration is empty"},
// 2
{
&serverConfig{Credential: auth.Credentials{
AccessKey: "u1",
SecretKey: "p1",
Expiration: timeSentinel,
}},
&serverConfig{Credential: auth.Credentials{
AccessKey: "u1",
SecretKey: "p2",
Expiration: timeSentinel,
}},
"Credential configuration differs",
},
// 3
{&serverConfig{Region: "us-east-1"}, &serverConfig{Region: "us-west-1"}, "Region configuration differs"},
// 4
{
&serverConfig{StorageClass: storageclass.Config{
Standard: storageclass.StorageClass{
Parity: 8,
},
RRS: storageclass.StorageClass{
Parity: 6,
},
}},
&serverConfig{StorageClass: storageclass.Config{
Standard: storageclass.StorageClass{
Parity: 8,
},
RRS: storageclass.StorageClass{
Parity: 4,
},
}},
"StorageClass configuration differs",
},
// 5
{
&serverConfig{Notify: notifier{AMQP: map[string]target.AMQPArgs{"1": {Enable: true}}}},
&serverConfig{Notify: notifier{AMQP: map[string]target.AMQPArgs{"1": {Enable: false}}}},
"AMQP Notification configuration differs",
},
// 6
{
&serverConfig{Notify: notifier{NATS: map[string]target.NATSArgs{"1": {Enable: true}}}},
&serverConfig{Notify: notifier{NATS: map[string]target.NATSArgs{"1": {Enable: false}}}},
"NATS Notification configuration differs",
},
// 7
{
&serverConfig{Notify: notifier{NSQ: map[string]target.NSQArgs{"1": {Enable: true}}}},
&serverConfig{Notify: notifier{NSQ: map[string]target.NSQArgs{"1": {Enable: false}}}},
"NSQ Notification configuration differs",
},
// 8
{
&serverConfig{Notify: notifier{Elasticsearch: map[string]target.ElasticsearchArgs{"1": {Enable: true}}}},
&serverConfig{Notify: notifier{Elasticsearch: map[string]target.ElasticsearchArgs{"1": {Enable: false}}}},
"ElasticSearch Notification configuration differs",
},
// 9
{
&serverConfig{Notify: notifier{Redis: map[string]target.RedisArgs{"1": {Enable: true}}}},
&serverConfig{Notify: notifier{Redis: map[string]target.RedisArgs{"1": {Enable: false}}}},
"Redis Notification configuration differs",
},
// 10
{
&serverConfig{Notify: notifier{PostgreSQL: map[string]target.PostgreSQLArgs{"1": {Enable: true}}}},
&serverConfig{Notify: notifier{PostgreSQL: map[string]target.PostgreSQLArgs{"1": {Enable: false}}}},
"PostgreSQL Notification configuration differs",
},
// 11
{
&serverConfig{Notify: notifier{Kafka: map[string]target.KafkaArgs{"1": {Enable: true}}}},
&serverConfig{Notify: notifier{Kafka: map[string]target.KafkaArgs{"1": {Enable: false}}}},
"Kafka Notification configuration differs",
},
// 12
{
&serverConfig{Notify: notifier{Webhook: map[string]target.WebhookArgs{"1": {Enable: true}}}},
&serverConfig{Notify: notifier{Webhook: map[string]target.WebhookArgs{"1": {Enable: false}}}},
"Webhook Notification configuration differs",
},
// 13
{
&serverConfig{Notify: notifier{MySQL: map[string]target.MySQLArgs{"1": {Enable: true}}}},
&serverConfig{Notify: notifier{MySQL: map[string]target.MySQLArgs{"1": {Enable: false}}}},
"MySQL Notification configuration differs",
},
// 14
{
&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: true}}}},
&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: false}}}},
"MQTT Notification configuration differs",
},
// 15
{
&serverConfig{Logger: logger.Config{
Console: logger.Console{Enabled: false},
HTTP: map[string]logger.HTTP{"1": {Endpoint: "http://address1"}},
}},
&serverConfig{Logger: logger.Config{
Console: logger.Console{Enabled: false},
HTTP: map[string]logger.HTTP{"1": {Endpoint: "http://address2"}},
}},
"Logger configuration differs",
},
}
for i, testCase := range testCases {
got := testCase.s.ConfigDiff(testCase.t)
if got != testCase.diff {
t.Errorf("Test %d: got %s expected %s", i+1, got, testCase.diff)
}
}
}

View File

@ -23,6 +23,7 @@ import (
"github.com/minio/minio/cmd/config/cache"
"github.com/minio/minio/cmd/config/compress"
xldap "github.com/minio/minio/cmd/config/ldap"
"github.com/minio/minio/cmd/config/notify"
"github.com/minio/minio/cmd/config/storageclass"
"github.com/minio/minio/cmd/crypto"
"github.com/minio/minio/cmd/logger"
@ -271,7 +272,7 @@ type serverConfigV7 struct {
Notify notifierV1 `json:"notify"`
}
// serverConfigV8 server configuration version '8'. Adds NATS notifier
// serverConfigV8 server configuration version '8'. Adds NATS notify.Config
// configuration.
type serverConfigV8 struct {
Version string `json:"version"`
@ -288,7 +289,7 @@ type serverConfigV8 struct {
}
// serverConfigV9 server configuration version '9'. Adds PostgreSQL
// notifier configuration.
// notify.Config configuration.
type serverConfigV9 struct {
Version string `json:"version"`
@ -562,9 +563,6 @@ type serverConfigV21 struct {
// serverConfigV22 is just like version '21' with added support
// for StorageClass.
//
// IMPORTANT NOTE: When updating this struct make sure that
// serverConfig.ConfigDiff() is updated as necessary.
type serverConfigV22 struct {
Version string `json:"version"`
@ -582,9 +580,6 @@ type serverConfigV22 struct {
}
// serverConfigV23 is just like version '22' with addition of cache field.
//
// IMPORTANT NOTE: When updating this struct make sure that
// serverConfig.ConfigDiff() is updated as necessary.
type serverConfigV23 struct {
Version string `json:"version"`
@ -606,9 +601,6 @@ type serverConfigV23 struct {
// serverConfigV24 is just like version '23', we had to revert
// the changes which were made in 6fb06045028b7a57c37c60a612c8e50735279ab4
//
// IMPORTANT NOTE: When updating this struct make sure that
// serverConfig.ConfigDiff() is updated as necessary.
type serverConfigV24 struct {
Version string `json:"version"`
@ -630,9 +622,6 @@ type serverConfigV24 struct {
// serverConfigV25 is just like version '24', stores additionally
// worm variable.
//
// IMPORTANT NOTE: When updating this struct make sure that
// serverConfig.ConfigDiff() is updated as necessary.
type serverConfigV25 struct {
quick.Config `json:"-"` // ignore interfaces
@ -681,9 +670,6 @@ type serverConfigV26 struct {
// serverConfigV27 is just like version '26', stores additionally
// the logger field
//
// IMPORTANT NOTE: When updating this struct make sure that
// serverConfig.ConfigDiff() is updated as necessary.
type serverConfigV27 struct {
quick.Config `json:"-"` // ignore interfaces
@ -711,9 +697,6 @@ type serverConfigV27 struct {
// serverConfigV28 is just like version '27', additionally
// storing KMS config
//
// IMPORTANT NOTE: When updating this struct make sure that
// serverConfig.ConfigDiff() is updated as necessary.
type serverConfigV28 struct {
quick.Config `json:"-"` // ignore interfaces
@ -814,19 +797,6 @@ type serverConfigV31 struct {
} `json:"policy"`
}
type notifier struct {
AMQP map[string]target.AMQPArgs `json:"amqp"`
Elasticsearch map[string]target.ElasticsearchArgs `json:"elasticsearch"`
Kafka map[string]target.KafkaArgs `json:"kafka"`
MQTT map[string]target.MQTTArgs `json:"mqtt"`
MySQL map[string]target.MySQLArgs `json:"mysql"`
NATS map[string]target.NATSArgs `json:"nats"`
NSQ map[string]target.NSQArgs `json:"nsq"`
PostgreSQL map[string]target.PostgreSQLArgs `json:"postgresql"`
Redis map[string]target.RedisArgs `json:"redis"`
Webhook map[string]target.WebhookArgs `json:"webhook"`
}
// serverConfigV32 is just like version '31' with added nsq notifer.
type serverConfigV32 struct {
Version string `json:"version"`
@ -846,7 +816,7 @@ type serverConfigV32 struct {
KMS crypto.KMSConfig `json:"kms"`
// Notification queue configuration.
Notify notifier `json:"notify"`
Notify notify.Config `json:"notify"`
// Logger configuration
Logger logger.Config `json:"logger"`
@ -890,7 +860,7 @@ type serverConfigV33 struct {
KMS crypto.KMSConfig `json:"kms"`
// Notification queue configuration.
Notify notifier `json:"notify"`
Notify notify.Config `json:"notify"`
// Logger configuration
Logger logger.Config `json:"logger"`

View File

@ -0,0 +1,66 @@
/*
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package notify
import "github.com/minio/minio/pkg/event/target"
// Config - notification target configuration structure, holds
// information about various notification targets.
type Config struct {
AMQP map[string]target.AMQPArgs `json:"amqp"`
Elasticsearch map[string]target.ElasticsearchArgs `json:"elasticsearch"`
Kafka map[string]target.KafkaArgs `json:"kafka"`
MQTT map[string]target.MQTTArgs `json:"mqtt"`
MySQL map[string]target.MySQLArgs `json:"mysql"`
NATS map[string]target.NATSArgs `json:"nats"`
NSQ map[string]target.NSQArgs `json:"nsq"`
PostgreSQL map[string]target.PostgreSQLArgs `json:"postgresql"`
Redis map[string]target.RedisArgs `json:"redis"`
Webhook map[string]target.WebhookArgs `json:"webhook"`
}
const (
defaultTarget = "1"
)
// NewConfig - initialize notification config.
func NewConfig() Config {
// Make sure to initialize notification targets
cfg := Config{
NSQ: make(map[string]target.NSQArgs),
AMQP: make(map[string]target.AMQPArgs),
MQTT: make(map[string]target.MQTTArgs),
NATS: make(map[string]target.NATSArgs),
Redis: make(map[string]target.RedisArgs),
MySQL: make(map[string]target.MySQLArgs),
Kafka: make(map[string]target.KafkaArgs),
Webhook: make(map[string]target.WebhookArgs),
PostgreSQL: make(map[string]target.PostgreSQLArgs),
Elasticsearch: make(map[string]target.ElasticsearchArgs),
}
cfg.NSQ[defaultTarget] = target.NSQArgs{}
cfg.AMQP[defaultTarget] = target.AMQPArgs{}
cfg.MQTT[defaultTarget] = target.MQTTArgs{}
cfg.NATS[defaultTarget] = target.NATSArgs{}
cfg.Redis[defaultTarget] = target.RedisArgs{}
cfg.MySQL[defaultTarget] = target.MySQLArgs{}
cfg.Kafka[defaultTarget] = target.KafkaArgs{}
cfg.Webhook[defaultTarget] = target.WebhookArgs{}
cfg.PostgreSQL[defaultTarget] = target.PostgreSQLArgs{}
cfg.Elasticsearch[defaultTarget] = target.ElasticsearchArgs{}
return cfg
}

View File

@ -205,9 +205,9 @@ func LookupConfig(cfg Config, drivesPerSet int) (Config, error) {
if err != nil {
return cfg, err
}
if cfg.Standard.Parity == 0 {
cfg.Standard.Parity = drivesPerSet / 2
}
}
if cfg.Standard.Parity == 0 {
cfg.Standard.Parity = drivesPerSet / 2
}
if rrsc := env.Get(RRSEnv, cfg.RRS.String()); rrsc != "" {
@ -215,9 +215,9 @@ func LookupConfig(cfg Config, drivesPerSet int) (Config, error) {
if err != nil {
return cfg, err
}
if cfg.RRS.Parity == 0 {
cfg.RRS.Parity = defaultRRSParity
}
}
if cfg.RRS.Parity == 0 {
cfg.RRS.Parity = defaultRRSParity
}
// Validation is done after parsing both the storage classes. This is needed because we need one

View File

@ -21,7 +21,7 @@ import (
"crypto/tls"
"errors"
"io"
"sync"
"sync/atomic"
"time"
"net/url"
@ -35,12 +35,10 @@ import (
// lockRESTClient is authenticable lock REST client
type lockRESTClient struct {
lockSync sync.RWMutex
host *xnet.Host
restClient *rest.Client
serverURL *url.URL
connected bool
timer *time.Timer
connected int32
}
func toLockError(err error) error {
@ -67,42 +65,11 @@ func (client *lockRESTClient) ServiceEndpoint() string {
return client.serverURL.Path
}
// check if the host is up or if it is fine
// to make a call to the lock rest server.
func (client *lockRESTClient) isHostUp() bool {
client.lockSync.Lock()
defer client.lockSync.Unlock()
if client.connected {
return true
}
select {
case <-client.timer.C:
client.connected = true
client.timer = nil
return true
default:
}
return false
}
// Mark the host as down if there is a Network error.
func (client *lockRESTClient) markHostDown() {
client.lockSync.Lock()
defer client.lockSync.Unlock()
if !client.connected {
return
}
client.connected = false
client.timer = time.NewTimer(defaultRetryUnit * 5)
}
// Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
// permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
// after verifying format.json
func (client *lockRESTClient) call(method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
if !client.isHostUp() {
if !client.IsOnline() {
return nil, errors.New("Lock rest server node is down")
}
@ -116,7 +83,12 @@ func (client *lockRESTClient) call(method string, values url.Values, body io.Rea
}
if isNetworkError(err) {
client.markHostDown()
time.AfterFunc(defaultRetryUnit*5, func() {
// After 5 seconds, take this lock client
// online for a retry.
atomic.StoreInt32(&client.connected, 1)
})
atomic.StoreInt32(&client.connected, 0)
}
return nil, toLockError(err)
@ -129,12 +101,12 @@ func (client *lockRESTClient) String() string {
// IsOnline - returns whether REST client failed to connect or not.
func (client *lockRESTClient) IsOnline() bool {
return client.connected
return atomic.LoadInt32(&client.connected) == 1
}
// Close - marks the client as closed.
func (client *lockRESTClient) Close() error {
client.connected = false
atomic.StoreInt32(&client.connected, 0)
client.restClient.Close()
return nil
}
@ -217,8 +189,8 @@ func newlockRESTClient(peer *xnet.Host) *lockRESTClient {
if err != nil {
logger.LogIf(context.Background(), err)
return &lockRESTClient{serverURL: serverURL, host: peer, restClient: restClient, connected: false, timer: time.NewTimer(defaultRetryUnit * 5)}
return &lockRESTClient{serverURL: serverURL, host: peer, restClient: restClient, connected: 0}
}
return &lockRESTClient{serverURL: serverURL, host: peer, restClient: restClient, connected: true}
return &lockRESTClient{serverURL: serverURL, host: peer, restClient: restClient, connected: 1}
}

View File

@ -30,7 +30,7 @@ func TestLockRESTlient(t *testing.T) {
t.Fatalf("unexpected error %v", err)
}
lkClient := newlockRESTClient(host)
if lkClient.connected == false {
if lkClient.connected == 0 {
t.Fatalf("unexpected error. connection failed")
}

View File

@ -196,7 +196,7 @@ func (l *lockRESTServer) lockMaintenance(interval time.Duration) {
continue
}
c := newlockRESTClient(host)
if !c.connected {
if !c.IsOnline() {
continue
}

View File

@ -51,6 +51,30 @@ const (
defaultTarget = "_"
)
// NewConfig - initialize new logger config.
func NewConfig() Config {
cfg := Config{
// Console logging is on by default
Console: Console{
Enabled: true,
},
HTTP: make(map[string]HTTP),
Audit: make(map[string]HTTP),
}
// Create an example HTTP logger
cfg.HTTP[defaultTarget] = HTTP{
Endpoint: "https://username:password@example.com/api",
}
// Create an example Audit logger
cfg.Audit[defaultTarget] = HTTP{
Endpoint: "https://username:password@example.com/api/audit",
}
return cfg
}
// LookupConfig - lookup logger config, override with ENVs if set.
func LookupConfig(cfg Config) (Config, error) {
envs := env.List(EnvLoggerHTTPEndpoint)

View File

@ -25,6 +25,7 @@ import (
"math/rand"
"net/url"
"strconv"
"sync/atomic"
"time"
"github.com/minio/minio/cmd/http"
@ -42,14 +43,12 @@ import (
type peerRESTClient struct {
host *xnet.Host
restClient *rest.Client
connected bool
connected int32
}
// Reconnect to a peer rest server.
func (client *peerRESTClient) reConnect() error {
// correct (intelligent) retry logic will be
// implemented in subsequent PRs.
client.connected = true
atomic.StoreInt32(&client.connected, 1)
return nil
}
@ -64,7 +63,7 @@ func (client *peerRESTClient) call(method string, values url.Values, body io.Rea
// permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
// after verifying format.json
func (client *peerRESTClient) callWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
if !client.connected {
if !client.IsOnline() {
err := client.reConnect()
logger.LogIf(ctx, err)
if err != nil {
@ -82,7 +81,7 @@ func (client *peerRESTClient) callWithContext(ctx context.Context, method string
}
if isNetworkError(err) {
client.connected = false
atomic.StoreInt32(&client.connected, 0)
}
return nil, err
@ -95,12 +94,12 @@ func (client *peerRESTClient) String() string {
// IsOnline - returns whether RPC client failed to connect or not.
func (client *peerRESTClient) IsOnline() bool {
return client.connected
return atomic.LoadInt32(&client.connected) == 1
}
// Close - marks the client as closed.
func (client *peerRESTClient) Close() error {
client.connected = false
atomic.StoreInt32(&client.connected, 0)
client.restClient.Close()
return nil
}
@ -733,8 +732,8 @@ func newPeerRESTClient(peer *xnet.Host) (*peerRESTClient, error) {
restClient, err := rest.NewClient(serverURL, tlsConfig, rest.DefaultRESTTimeout, newAuthToken)
if err != nil {
return &peerRESTClient{host: peer, restClient: restClient, connected: false}, err
return &peerRESTClient{host: peer, restClient: restClient, connected: 0}, err
}
return &peerRESTClient{host: peer, restClient: restClient, connected: true}, nil
return &peerRESTClient{host: peer, restClient: restClient, connected: 1}, nil
}

View File

@ -1,5 +1,5 @@
/*
* MinIO Cloud Storage, (C) 2018 MinIO, Inc.
* MinIO Cloud Storage, (C) 2018-2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -27,6 +27,7 @@ import (
"net/url"
"path"
"strconv"
"sync/atomic"
"github.com/minio/minio/cmd/http"
"github.com/minio/minio/cmd/rest"
@ -107,7 +108,7 @@ func toStorageErr(err error) error {
type storageRESTClient struct {
endpoint Endpoint
restClient *rest.Client
connected bool
connected int32
lastError error
instanceID string // REST server's instanceID which is sent with every request for validation.
}
@ -116,7 +117,7 @@ type storageRESTClient struct {
// permanently. The only way to restore the storage connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
// after verifying format.json
func (client *storageRESTClient) call(method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
if !client.connected {
if !client.IsOnline() {
return nil, errDiskNotFound
}
if values == nil {
@ -129,7 +130,7 @@ func (client *storageRESTClient) call(method string, values url.Values, body io.
}
client.lastError = err
if isNetworkError(err) {
client.connected = false
atomic.StoreInt32(&client.connected, 0)
}
return nil, toStorageErr(err)
@ -142,7 +143,7 @@ func (client *storageRESTClient) String() string {
// IsOnline - returns whether RPC client failed to connect or not.
func (client *storageRESTClient) IsOnline() bool {
return client.connected
return atomic.LoadInt32(&client.connected) == 1
}
// LastError - returns the network error if any.
@ -460,7 +461,7 @@ func (client *storageRESTClient) VerifyFile(volume, path string, size int64, alg
// Close - marks the client as closed.
func (client *storageRESTClient) Close() error {
client.connected = false
atomic.StoreInt32(&client.connected, 0)
client.restClient.Close()
return nil
}
@ -496,7 +497,11 @@ func newStorageRESTClient(endpoint Endpoint) (*storageRESTClient, error) {
if err != nil {
return nil, err
}
client := &storageRESTClient{endpoint: endpoint, restClient: restClient, connected: true}
client.connected = client.getInstanceID() == nil
client := &storageRESTClient{endpoint: endpoint, restClient: restClient, connected: 1}
if client.getInstanceID() == nil {
client.connected = 1
} else {
client.connected = 0
}
return client, nil
}

2
go.mod
View File

@ -44,7 +44,7 @@ require (
github.com/minio/lsync v1.0.1
github.com/minio/mc v0.0.0-20190924013003-643835013047
github.com/minio/minio-go v0.0.0-20190327203652-5325257a208f
github.com/minio/minio-go/v6 v6.0.38
github.com/minio/minio-go/v6 v6.0.39
github.com/minio/parquet-go v0.0.0-20190318185229-9d767baf1679
github.com/minio/sha256-simd v0.1.1
github.com/minio/sio v0.2.0

2
go.sum
View File

@ -438,6 +438,8 @@ github.com/minio/minio-go/v6 v6.0.37 h1:rqot4cO9+mLpf56q+yumA0xZlncbkFpqa4A8jw1Y
github.com/minio/minio-go/v6 v6.0.37/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
github.com/minio/minio-go/v6 v6.0.38 h1:zd3yagckaBVAMJT+HsbpURx9ndqYQp/N/udc1UVS72E=
github.com/minio/minio-go/v6 v6.0.38/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
github.com/minio/minio-go/v6 v6.0.39 h1:9qmKCTBpQpMdGlDAbs3mbb4mmL45/lwRUvHL1VLhYUk=
github.com/minio/minio-go/v6 v6.0.39/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
github.com/minio/parquet-go v0.0.0-20190318185229-9d767baf1679 h1:OMKaN/82sBHUZPvjYNBFituHExa1OGY63eACDGtetKs=
github.com/minio/parquet-go v0.0.0-20190318185229-9d767baf1679/go.mod h1:J+goXSuzlte5imWMqb6cUWC/tbYYysUHctwmKXomYzM=
github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=