simplify context timeout for readiness (#9772)

additionally also add CORS support to restrict
for specific origin, adds a new config and
updated the documentation as well

Makefile

@@ -8,8 +8,6 @@ GOOS := $(shell go env GOOS)
 VERSION ?= $(shell git describe --tags)
 TAG ?= "minio/minio:$(VERSION)"
 
-BUILD_LDFLAGS := '$(LDFLAGS)'
-
 all: build
 
 checks:
@@ -48,19 +46,19 @@ test-race: verifiers build
 # Verify minio binary
 verify:
 	@echo "Verifying build with race"
-	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
 # Verify healing of disks with minio binary
 verify-healing:
 	@echo "Verify healing build with race"
-	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-healing.sh)
 
 # Builds minio locally.
 build: checks
 	@echo "Building minio binary to './minio'"
-	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath  --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 
 docker: build
 	@docker build -t $(TAG) . -f Dockerfile.dev

cmd/config-current.go

@@ -365,13 +365,7 @@ func lookupConfigs(s config.Config) {
 		logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
 	}
 
-	apiRequestsMax := apiConfig.APIRequestsMax
+	globalAPIConfig.init(apiConfig)
-	if len(globalEndpoints.Hosts()) > 0 {
-		apiRequestsMax /= len(globalEndpoints.Hosts())
-	}
-
-	globalAPIThrottling.init(apiRequestsMax, apiConfig.APIRequestsDeadline)
-	globalReadyDeadline = apiConfig.APIReadyDeadline
 
 	if globalIsXL {
 		globalStorageClass, err = storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default],

cmd/config/api/api.go

@@ -20,16 +20,24 @@ import (
 	"encoding/json"
 	"errors"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/pkg/env"
 )
 
+// API sub-system constants
 const (
 	apiRequestsMax      = "requests_max"
 	apiRequestsDeadline = "requests_deadline"
 	apiReadyDeadline    = "ready_deadline"
+	apiCorsAllowOrigin  = "cors_allow_origin"
+
+	EnvAPIRequestsMax      = "MINIO_API_REQUESTS_MAX"
+	EnvAPIRequestsDeadline = "MINIO_API_REQUESTS_DEADLINE"
+	EnvAPIReadyDeadline    = "MINIO_API_READY_DEADLINE"
+	EnvAPICorsAllowOrigin  = "MINIO_API_CORS_ALLOW_ORIGIN"
 )
 
 // DefaultKVS - default storage class config
@@ -47,6 +55,10 @@ var (
 			Key:   apiReadyDeadline,
 			Value: "10s",
 		},
+		config.KV{
+			Key:   apiCorsAllowOrigin,
+			Value: "*",
+		},
 	}
 )
 
@@ -55,6 +67,7 @@ type Config struct {
 	APIRequestsMax      int           `json:"requests_max"`
 	APIRequestsDeadline time.Duration `json:"requests_deadline"`
 	APIReadyDeadline    time.Duration `json:"ready_deadline"`
+	APICorsAllowOrigin  []string      `json:"cors_allow_origin"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -75,7 +88,7 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 	}
 
 	// Check environment variables parameters
-	requestsMax, err := strconv.Atoi(env.Get(config.EnvAPIRequestsMax, kvs.Get(apiRequestsMax)))
+	requestsMax, err := strconv.Atoi(env.Get(EnvAPIRequestsMax, kvs.Get(apiRequestsMax)))
 	if err != nil {
 		return cfg, err
 	}
@@ -84,20 +97,21 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 		return cfg, errors.New("invalid API max requests value")
 	}
 
-	requestsDeadline, err := time.ParseDuration(env.Get(config.EnvAPIRequestsDeadline, kvs.Get(apiRequestsDeadline)))
+	requestsDeadline, err := time.ParseDuration(env.Get(EnvAPIRequestsDeadline, kvs.Get(apiRequestsDeadline)))
 	if err != nil {
 		return cfg, err
 	}
 
-	readyDeadline, err := time.ParseDuration(env.Get(config.EnvAPIReadyDeadline, kvs.Get(apiReadyDeadline)))
+	readyDeadline, err := time.ParseDuration(env.Get(EnvAPIReadyDeadline, kvs.Get(apiReadyDeadline)))
 	if err != nil {
 		return cfg, err
 	}
 
-	cfg = Config{
+	corsAllowOrigin := strings.Split(env.Get(EnvAPICorsAllowOrigin, kvs.Get(apiCorsAllowOrigin)), ",")
+	return Config{
 		APIRequestsMax:      requestsMax,
 		APIRequestsDeadline: requestsDeadline,
 		APIReadyDeadline:    readyDeadline,
-	}
+		APICorsAllowOrigin:  corsAllowOrigin,
-	return cfg, nil
+	}, nil
 }

cmd/config/api/help.go

@@ -39,5 +39,11 @@ var (
 			Optional:    true,
 			Type:        "duration",
 		},
+		config.HelpKV{
+			Key:         apiCorsAllowOrigin,
+			Description: `set comma separated list of origins allowed for CORS requests e.g. "https://example1.com,https://example2.com"`,
+			Optional:    true,
+			Type:        "csv",
+		},
 	}
 )

cmd/config/constants.go

@@ -34,11 +34,6 @@ const (
 	EnvEndpoints    = "MINIO_ENDPOINTS"
 	EnvFSOSync      = "MINIO_FS_OSYNC"
 
-	// API sub-system
-	EnvAPIRequestsMax      = "MINIO_API_REQUESTS_MAX"
-	EnvAPIRequestsDeadline = "MINIO_API_REQUESTS_DEADLINE"
-	EnvAPIReadyDeadline    = "MINIO_API_READY_DEADLINE"
-
 	EnvUpdate = "MINIO_UPDATE"
 
 	EnvWorm   = "MINIO_WORM"   // legacy

cmd/generic-handlers.go

@@ -30,6 +30,7 @@ import (
 	"github.com/minio/minio/cmd/http/stats"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/handlers"
+	"github.com/minio/minio/pkg/wildcard"
 	"github.com/rs/cors"
 )
 
@@ -410,7 +411,14 @@ func setCorsHandler(h http.Handler) http.Handler {
 	}
 
 	c := cors.New(cors.Options{
-		AllowedOrigins: []string{"*"},
+		AllowOriginFunc: func(origin string) bool {
+			for _, allowedOrigin := range globalAPIConfig.getCorsAllowOrigins() {
+				if wildcard.MatchSimple(allowedOrigin, origin) {
+					return true
+				}
+			}
+			return false
+		},
 		AllowedMethods: []string{
 			http.MethodGet,
 			http.MethodPut,

cmd/globals.go

@@ -154,9 +154,9 @@ var (
 	globalLifecycleSys       *LifecycleSys
 	globalBucketSSEConfigSys *BucketSSEConfigSys
 
-	// globalAPIThrottling controls S3 requests throttling when
+	// globalAPIConfig controls S3 API requests throttling,
-	// enabled in the config or in the shell environment.
+	// healthcheck readiness deadlines and cors settings.
-	globalAPIThrottling apiThrottling
+	globalAPIConfig apiConfig
 
 	globalStorageClass storageclass.Config
 	globalLDAPConfig   xldap.Config
@@ -275,8 +275,6 @@ var (
 	// If writes to FS backend should be O_SYNC.
 	globalFSOSync bool
 
-	// Deadline by which /minio/health/ready should respond.
-	globalReadyDeadline time.Duration
 	// Add new variable global values here.
 )
 

cmd/handler-api.go

@@ -20,34 +20,61 @@ import (
 	"net/http"
 	"sync"
 	"time"
+
+	"github.com/minio/minio/cmd/config/api"
 )
 
-type apiThrottling struct {
+type apiConfig struct {
 	mu sync.RWMutex
-	enabled bool
 
 	requestsDeadline time.Duration
 	requestsPool     chan struct{}
+	readyDeadline    time.Duration
+	corsAllowOrigins []string
 }
 
-func (t *apiThrottling) init(max int, deadline time.Duration) {
+func (t *apiConfig) init(cfg api.Config) {
-	if max <= 0 {
-		return
-	}
-
 	t.mu.Lock()
 	defer t.mu.Unlock()
 
-	t.requestsPool = make(chan struct{}, max)
+	t.readyDeadline = cfg.APIReadyDeadline
-	t.requestsDeadline = deadline
+	t.corsAllowOrigins = cfg.APICorsAllowOrigin
-	t.enabled = true
+	if cfg.APIRequestsMax <= 0 {
+		return
 	}
 
-func (t *apiThrottling) get() (chan struct{}, <-chan time.Time) {
+	apiRequestsMax := cfg.APIRequestsMax
+	if len(globalEndpoints.Hosts()) > 0 {
+		apiRequestsMax /= len(globalEndpoints.Hosts())
+	}
+
+	t.requestsPool = make(chan struct{}, apiRequestsMax)
+	t.requestsDeadline = cfg.APIRequestsDeadline
+}
+
+func (t *apiConfig) getCorsAllowOrigins() []string {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 
-	if !t.enabled {
+	return t.corsAllowOrigins
+}
+
+func (t *apiConfig) getReadyDeadline() time.Duration {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	if t.readyDeadline == 0 {
+		return 10 * time.Second
+	}
+
+	return t.readyDeadline
+}
+
+func (t *apiConfig) getRequestsPool() (chan struct{}, <-chan time.Time) {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	if t.requestsPool == nil {
 		return nil, nil
 	}
 
@@ -57,7 +84,7 @@ func (t *apiThrottling) get() (chan struct{}, <-chan time.Time) {
 // maxClients throttles the S3 API calls
 func maxClients(f http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		pool, deadlineTimer := globalAPIThrottling.get()
+		pool, deadlineTimer := globalAPIConfig.getRequestsPool()
 		if pool == nil {
 			f.ServeHTTP(w, r)
 			return

cmd/healthcheck-handler.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"net/http"
 )
 
@@ -28,7 +29,15 @@ func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
 
 	objLayer := newObjectLayerWithoutSafeModeFn()
 	// Service not initialized yet
-	if objLayer == nil || !objLayer.IsReady(ctx) {
+	if objLayer == nil {
+		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, globalAPIConfig.getReadyDeadline())
+	defer cancel()
+
+	if !objLayer.IsReady(ctx) {
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
 	}

cmd/notification.go

@@ -1164,7 +1164,7 @@ func (sys *NotificationSys) ServerInfo() []madmin.ServerProperties {
 }
 
 // GetLocalDiskIDs - return disk ids of the local disks of the peers.
-func (sys *NotificationSys) GetLocalDiskIDs() []string {
+func (sys *NotificationSys) GetLocalDiskIDs(ctx context.Context) []string {
 	var diskIDs []string
 	var mu sync.Mutex
 
@@ -1176,7 +1176,7 @@ func (sys *NotificationSys) GetLocalDiskIDs() []string {
 		wg.Add(1)
 		go func(client *peerRESTClient) {
 			defer wg.Done()
-			ids := client.GetLocalDiskIDs()
+			ids := client.GetLocalDiskIDs(ctx)
 			mu.Lock()
 			diskIDs = append(diskIDs, ids...)
 			mu.Unlock()

cmd/peer-rest-client.go

@@ -677,19 +677,7 @@ func (client *peerRESTClient) BackgroundHealStatus() (madmin.BgHealState, error)
 }
 
 // GetLocalDiskIDs - get a peer's local disks' IDs.
-func (client *peerRESTClient) GetLocalDiskIDs() []string {
+func (client *peerRESTClient) GetLocalDiskIDs(ctx context.Context) []string {
-	doneCh := make(chan struct{})
-	defer close(doneCh)
-
-	ctx, cancel := context.WithCancel(GlobalContext)
-	go func() {
-		select {
-		case <-doneCh:
-			return
-		case <-time.After(globalReadyDeadline):
-			cancel()
-		}
-	}()
 	respBody, err := client.callWithContext(ctx, peerRESTMethodGetLocalDiskIDs, nil, nil, -1)
 	if err != nil {
 		return nil

cmd/xl-v1.go

@@ -403,7 +403,7 @@ func (xl xlObjects) crawlAndGetDataUsage(ctx context.Context, buckets []BucketIn
 	return nil
 }
 
-// IsReady - No Op.
+// IsReady - shouldn't be called will panic.
 func (xl xlObjects) IsReady(ctx context.Context) bool {
 	logger.CriticalIf(ctx, NotImplemented{})
 	return true

cmd/xl-zones.go

@@ -1615,7 +1615,7 @@ func (z *xlZones) IsReady(ctx context.Context) bool {
 		erasureSetUpCount[i] = make([]int, len(z.zones[i].sets))
 	}
 
-	diskIDs := globalNotificationSys.GetLocalDiskIDs()
+	diskIDs := globalNotificationSys.GetLocalDiskIDs(ctx)
 
 	diskIDs = append(diskIDs, getLocalDiskIDs(z)...)
 

docs/config/README.md

@@ -172,7 +172,6 @@ MINIO_ETCD_COMMENT          (sentence)  optionally add a comment to this setting
 ```
 
 ### API
-
 By default, there is no limitation on the number of concurrents requests that a server/cluster processes at the same time. However, it is possible to impose such limitation using the API subsystem. Read more about throttling limitation in MinIO server [here](https://github.com/minio/minio/blob/master/docs/throttle/README.md).
 
 ```
@@ -180,16 +179,19 @@ KEY:
 api  manage global HTTP API call specific features, such as throttling, authentication types, etc.
 
 ARGS:
-requests_max       (number)     set the maximum number of concurrent requests
+requests_max       (number)    set the maximum number of concurrent requests, e.g. "1600"
-requests_deadline  (duration)   set the deadline for API requests waiting to be processed
+requests_deadline  (duration)  set the deadline for API requests waiting to be processed e.g. "1m"
+ready_deadline     (duration)  set the deadline for health check API /minio/health/ready e.g. "1m"
+cors_allow_origin  (csv)       set comma separated list of origins allowed for CORS requests e.g. "https://example1.com,https://example2.com"
 ```
 
 or environment variables
 
 ```
-MINIO_API_REQUESTS_MAX        (number)     set the maximum number of concurrent requests
+MINIO_API_REQUESTS_MAX       (number)    set the maximum number of concurrent requests, e.g. "1600"
-MINIO_API_REQUESTS_DEADLINE   (duration)   set the deadline for API requests waiting to be processed
+MINIO_API_REQUESTS_DEADLINE  (duration)  set the deadline for API requests waiting to be processed e.g. "1m"
-
+MINIO_API_READY_DEADLINE     (duration)  set the deadline for health check API /minio/health/ready e.g. "1m"
+MINIO_API_CORS_ALLOW_ORIGIN  (csv)       set comma separated list of origins allowed for CORS requests e.g. "https://example1.com,https://example2.com"
 ```
 
 #### Notifications

docs/metrics/healthcheck/README.md

@@ -21,3 +21,15 @@ Platforms like Kubernetes *do not* forward traffic to a pod until its readiness
 ### Configuration example
 
 Sample `liveness` and `readiness` probe configuration in a Kubernetes `yaml` file can be found [here](https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes/minio-standalone-deployment.yaml).
+
+### Configure readiness deadline
+Readiness checks need to respond faster in orchestrated environments, to facilitate this you can use the following environment variable before starting MinIO
+
+```
+MINIO_API_READY_DEADLINE     (duration)  set the deadline for health check API /minio/health/ready e.g. "1m"
+```
+
+Set a *5s* deadline for MinIO to ensure readiness handler responds with-in 5seconds.
+```
+export MINIO_API_READY_DEADLINE=5s
+```

go.mod

@@ -94,7 +94,7 @@ require (
 	github.com/prometheus/client_golang v0.9.3
 	github.com/rcrowley/go-metrics v0.0.0-20190704165056-9c2d0518ed81 // indirect
 	github.com/rjeczalik/notify v0.9.2
-	github.com/rs/cors v1.6.0
+	github.com/rs/cors v1.7.0
 	github.com/secure-io/sio-go v0.3.0
 	github.com/shirou/gopsutil v2.20.3-0.20200314133625-53cec6b37e6a+incompatible
 	github.com/sirupsen/logrus v1.5.0

go.sum

@@ -377,8 +377,8 @@ github.com/rjeczalik/notify v0.9.2 h1:MiTWrPj55mNDHEiIX5YUSKefw/+lCQVoAFmD6oQm5w
 github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rs/cors v1.6.0 h1:G9tHG9lebljV9mfp9SNPDL36nCDxmo3zTlAf1YgvzmI=
+github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
-github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
+github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
 github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=