fix: readiness needs to be like liveness (#9941)

Readiness as no reasoning to be cluster scope
because that is not how the k8s networking works
for pods, all the pods to a deployment are not
sharing the network in a singleton. Instead they
are run as local scopes to themselves, with
readiness failures the pod is potentially taken
out of the network to be resolvable - this
affects the distributed setup in myriad of
different ways.

Instead readiness should behave like liveness
with local scope alone, and should be a dummy
implementation.

This PR all the startup times and overal k8s
startup time dramatically improves.

Added another handler called as `/minio/health/cluster`
to understand the cluster scope health.

cmd/config/api/help.go

@@ -33,12 +33,6 @@ var (
 			Optional:    true,
 			Type:        "duration",
 		},
-		config.HelpKV{
-			Key:         apiReadyDeadline,
-			Description: `set the deadline for health check API /minio/health/ready e.g. "1m"`,
-			Optional:    true,
-			Type:        "duration",
-		},
 		config.HelpKV{
 			Key:         apiCorsAllowOrigin,
 			Description: `set comma separated list of origins allowed for CORS requests e.g. "https://example1.com,https://example2.com"`,

cmd/endpoint.go

@@ -44,8 +44,6 @@ const (
 
 	// URLEndpointType - URL style endpoint type enum.
 	URLEndpointType
-
-	retryInterval = 5 // In Seconds.
 )
 
 // Endpoint - any type of endpoint.
@@ -302,7 +300,7 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 	resolvedList := make([]bool, len(endpoints))
 	// Mark the starting time
 	startTime := time.Now()
-	keepAliveTicker := time.NewTicker(retryInterval * time.Second)
+	keepAliveTicker := time.NewTicker(10 * time.Millisecond)
 	defer keepAliveTicker.Stop()
 	for {
 		// Break if the local endpoint is found already Or all the endpoints are resolved.

cmd/healthcheck-handler.go

@@ -21,13 +21,11 @@ import (
 	"net/http"
 )
 
-// ReadinessCheckHandler returns if the server is ready to receive requests.
-// For FS - Checks if the backend disk is available
-// For Erasure backend - Checks if all the erasure sets are writable
-func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ReadinessCheckHandler")
+// ClusterCheckHandler returns if the server is ready for requests.
+func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ClusterCheckCheckHandler")
 
-	objLayer := newObjectLayerWithoutSafeModeFn()
+	objLayer := newObjectLayerFn()
 	// Service not initialized yet
 	if objLayer == nil {
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
@@ -37,7 +35,7 @@ func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
 	ctx, cancel := context.WithTimeout(ctx, globalAPIConfig.getReadyDeadline())
 	defer cancel()
 
-	if !objLayer.IsReady(ctx) && newObjectLayerFn() == nil {
+	if !objLayer.IsReady(ctx) {
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
 	}
@@ -45,6 +43,13 @@ func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
 	writeResponse(w, http.StatusOK, nil, mimeNone)
 }
 
+// ReadinessCheckHandler Checks if the process is up. Always returns success.
+func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
+	// TODO: only implement this function to notify that this pod is
+	// busy, at a local scope in future, for now '200 OK'.
+	writeResponse(w, http.StatusOK, nil, mimeNone)
+}
+
 // LivenessCheckHandler - Checks if the process is up. Always returns success.
 func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
 	writeResponse(w, http.StatusOK, nil, mimeNone)

cmd/healthcheck-router.go

@@ -26,6 +26,7 @@ const (
 	healthCheckPath          = "/health"
 	healthCheckLivenessPath  = "/live"
 	healthCheckReadinessPath = "/ready"
+	healthCheckClusterPath   = "/cluster"
 	healthCheckPathPrefix    = minioReservedBucketPath + healthCheckPath
 )
 
@@ -35,6 +36,9 @@ func registerHealthCheckRouter(router *mux.Router) {
 	// Healthcheck router
 	healthRouter := router.PathPrefix(healthCheckPathPrefix).Subrouter()
 
+	// Cluster check handler to verify cluster is active
+	healthRouter.Methods(http.MethodGet).Path(healthCheckClusterPath).HandlerFunc(httpTraceAll(ClusterCheckHandler))
+
 	// Liveness handler
 	healthRouter.Methods(http.MethodGet).Path(healthCheckLivenessPath).HandlerFunc(httpTraceAll(LivenessCheckHandler))
 	healthRouter.Methods(http.MethodHead).Path(healthCheckLivenessPath).HandlerFunc(httpTraceAll(LivenessCheckHandler))