fix: liveness/readiness must return errors if KMS is unreachable (#16540)

This commit is contained in:
Harshavardhana 2023-02-06 08:55:56 -08:00 committed by GitHub
parent 6ac48aff46
commit 21885f9457
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -21,13 +21,14 @@ import (
"context" "context"
"net/http" "net/http"
"strconv" "strconv"
"time"
xhttp "github.com/minio/minio/internal/http" xhttp "github.com/minio/minio/internal/http"
) )
const unavailable = "offline" const unavailable = "offline"
func shouldProxy() bool { func isServerInitialized() bool {
return newObjectLayerFn() == nil return newObjectLayerFn() == nil
} }
@ -35,7 +36,7 @@ func shouldProxy() bool {
func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) { func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "ClusterCheckHandler") ctx := newContext(r, w, "ClusterCheckHandler")
if shouldProxy() { if isServerInitialized() {
w.Header().Set(xhttp.MinIOServerStatus, unavailable) w.Header().Set(xhttp.MinIOServerStatus, unavailable)
writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone) writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
return return
@ -73,7 +74,7 @@ func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
func ClusterReadCheckHandler(w http.ResponseWriter, r *http.Request) { func ClusterReadCheckHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "ClusterReadCheckHandler") ctx := newContext(r, w, "ClusterReadCheckHandler")
if shouldProxy() { if isServerInitialized() {
w.Header().Set(xhttp.MinIOServerStatus, unavailable) w.Header().Set(xhttp.MinIOServerStatus, unavailable)
writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone) writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
return return
@ -100,11 +101,28 @@ func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
// LivenessCheckHandler - Checks if the process is up. Always returns success. // LivenessCheckHandler - Checks if the process is up. Always returns success.
func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) { func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
if shouldProxy() { if isServerInitialized() {
// Service not initialized yet // Service not initialized yet
w.Header().Set(xhttp.MinIOServerStatus, unavailable) w.Header().Set(xhttp.MinIOServerStatus, unavailable)
} }
// Verify if KMS is reachable if its configured
if GlobalKMS != nil {
ctx, cancel := context.WithTimeout(r.Context(), time.Minute)
defer cancel()
if _, err := GlobalKMS.Stat(ctx); err != nil {
switch r.Method {
case http.MethodHead:
apiErr := toAPIError(r.Context(), err)
writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone)
case http.MethodGet:
writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL)
}
return
}
}
if globalEtcdClient != nil { if globalEtcdClient != nil {
// Borrowed from // Borrowed from
// https://github.com/etcd-io/etcd/blob/main/etcdctl/ctlv3/command/ep_command.go#L118 // https://github.com/etcd-io/etcd/blob/main/etcdctl/ctlv3/command/ep_command.go#L118