Add proper contexts with timeouts for etcd operations (#7097)

This fixes an issue of perceived hang when incorrect
unreachable URLs are specified in MINIO_ETCD_ENDPOINTS
variable.

Fixes #7096
This commit is contained in:
Harshavardhana
2019-01-18 23:06:45 +05:30
committed by kannappanr
parent 730ac5381c
commit 74c2048ea9
5 changed files with 56 additions and 33 deletions

View File

@@ -20,6 +20,7 @@ import (
"bytes"
"context"
"errors"
"fmt"
etcd "github.com/coreos/etcd/clientv3"
"github.com/minio/minio/cmd/logger"
@@ -60,8 +61,15 @@ func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) er
}
func saveConfigEtcd(ctx context.Context, client *etcd.Client, configFile string, data []byte) error {
_, err := client.Put(ctx, configFile, string(data))
return err
timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
defer cancel()
_, err := client.Put(timeoutCtx, configFile, string(data))
if err == context.DeadlineExceeded {
return fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", client.Endpoints())
} else if err != nil {
return fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, client.Endpoints())
}
return nil
}
func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data []byte) error {
@@ -75,9 +83,14 @@ func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data
}
func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) ([]byte, error) {
resp, err := client.Get(ctx, configFile)
timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
defer cancel()
resp, err := client.Get(timeoutCtx, configFile)
if err != nil {
return nil, err
if err == context.DeadlineExceeded {
return nil, fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", client.Endpoints())
}
return nil, fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, client.Endpoints())
}
if resp.Count == 0 {
return nil, errConfigNotFound
@@ -93,7 +106,9 @@ func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string)
// watchConfig - watches for changes on `configFile` on etcd and loads them.
func watchConfig(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLayer) error) {
if globalEtcdClient != nil {
for watchResp := range globalEtcdClient.Watch(context.Background(), configFile) {
ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
defer cancel()
for watchResp := range globalEtcdClient.Watch(ctx, configFile) {
for _, event := range watchResp.Events {
if event.IsModify() || event.IsCreate() {
loadCfgFn(objAPI)
@@ -104,9 +119,14 @@ func watchConfig(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLay
}
func checkConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
resp, err := globalEtcdClient.Get(ctx, configFile)
timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
defer cancel()
resp, err := client.Get(timeoutCtx, configFile)
if err != nil {
return err
if err == context.DeadlineExceeded {
return fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", client.Endpoints())
}
return fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, client.Endpoints())
}
if resp.Count == 0 {
return errConfigNotFound