readiness returns error quickly if any of the set is down (#9662)

This PR adds a new configuration parameter which allows readiness
check to respond within 10secs, this can be reduced to a lower value
if necessary using 

```
mc admin config set api ready_deadline=5s
```

 or

```
export MINIO_API_READY_DEADLINE=5s
```

cmd/config/api/api.go

@@ -29,6 +29,7 @@ import (
 const (
 	apiRequestsMax      = "requests_max"
 	apiRequestsDeadline = "requests_deadline"
+	apiReadyDeadline    = "ready_deadline"
 )
 
 // DefaultKVS - default storage class config
@@ -42,6 +43,10 @@ var (
 			Key:   apiRequestsDeadline,
 			Value: "10s",
 		},
+		config.KV{
+			Key:   apiReadyDeadline,
+			Value: "10s",
+		},
 	}
 )
 
@@ -49,6 +54,7 @@ var (
 type Config struct {
 	APIRequestsMax      int           `json:"requests_max"`
 	APIRequestsDeadline time.Duration `json:"requests_deadline"`
+	APIReadyDeadline    time.Duration `json:"ready_deadline"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -83,9 +89,15 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 		return cfg, err
 	}
 
+	readyDeadline, err := time.ParseDuration(env.Get(config.EnvAPIReadyDeadline, kvs.Get(apiReadyDeadline)))
+	if err != nil {
+		return cfg, err
+	}
+
 	cfg = Config{
 		APIRequestsMax:      requestsMax,
 		APIRequestsDeadline: requestsDeadline,
+		APIReadyDeadline:    readyDeadline,
 	}
 	return cfg, nil
 }

cmd/config/api/help.go

@@ -33,5 +33,11 @@ var (
 			Optional:    true,
 			Type:        "duration",
 		},
+		config.HelpKV{
+			Key:         apiReadyDeadline,
+			Description: `set the deadline for health check API /minio/health/ready e.g. "1m"`,
+			Optional:    true,
+			Type:        "duration",
+		},
 	}
 )