Lock: Use REST API instead of RPC (#7469)

In distributed mode, use REST API to acquire and manage locks instead
of RPC.

RPC has been completely removed from MinIO source.

Since we are moving from RPC to REST, we cannot use rolling upgrades as the
nodes that have not yet been upgraded cannot talk to the ones that have
been upgraded.

We expect all minio processes on all nodes to be stopped and then the
upgrade process to be completed.

Also force http1.1 for inter-node communication

cmd/generic-handlers.go

@@ -31,9 +31,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/handlers"
-	"github.com/minio/minio/pkg/sys"
 	"github.com/rs/cors"
-	"golang.org/x/time/rate"
 )
 
 // HandlerFunc - useful to chain different middleware http.Handler
@@ -737,46 +735,6 @@ func setBucketForwardingHandler(h http.Handler) http.Handler {
 	return bucketForwardingHandler{fwd, h}
 }
 
-// setRateLimitHandler middleware limits the throughput to h using a
-// rate.Limiter token bucket configured with maxOpenFileLimit and
-// burst set to 1. The request will idle for up to 1*time.Second.
-// If the limiter detects the deadline will be exceeded, the request is
-// canceled immediately.
-func setRateLimitHandler(h http.Handler) http.Handler {
-	_, maxLimit, err := sys.GetMaxOpenFileLimit()
-	logger.FatalIf(err, "Unable to get maximum open file limit")
-	// Burst value is set to 1 to allow only maxOpenFileLimit
-	// requests to happen at once.
-	l := rate.NewLimiter(rate.Limit(maxLimit), 1)
-	return rateLimit{l, h}
-}
-
-type rateLimit struct {
-	*rate.Limiter
-	handler http.Handler
-}
-
-func (l rateLimit) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	// create a new context from the request with the wait timeout
-	ctx, cancel := context.WithTimeout(r.Context(), 1*time.Second)
-	defer cancel() // always cancel the context!
-
-	// Wait errors out if the request cannot be processed within
-	// the deadline. time/rate tries to reserve a slot if possible
-	// with in the given duration if it's not possible then Wait(ctx)
-	// returns an error and we cancel the request with ErrSlowDown
-	// error message to the client. This context wait also ensures
-	// requests doomed to fail are terminated early, preventing a
-	// potential pileup on the server.
-	if err := l.Wait(ctx); err != nil {
-		// Send an S3 compatible error, SlowDown.
-		writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrSlowDown), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	l.handler.ServeHTTP(w, r)
-}
-
 // customHeaderHandler sets x-amz-request-id, x-minio-deployment-id header.
 // Previously, this value was set right before a response was sent to
 // the client. So, logger and Error response XML were not using this