add optimizations to bring performance on unversioned READS (#20128)

allow non-inlined on disk to be inlined via
an unversioned ReadVersion() call, we only
need ReadXL() to resolve objects with multiple
versions only.

The choice of this block makes it to be dynamic
and chosen by the user via `mc admin config set`

Other bonus things

- Start measuring internode TTFB performance.
- Set TCP_NODELAY, TCP_CORK for low latency

internal/rest/rpc-stats.go

@@ -22,14 +22,17 @@ import (
 	"net/http/httptrace"
 	"sync/atomic"
 	"time"
+
+	"github.com/minio/minio/internal/logger"
 )
 
 var globalStats = struct {
 	errs uint64
 
-	tcpDialErrs     uint64
-	tcpDialCount    uint64
-	tcpDialTotalDur uint64
+	tcpDialErrs                 uint64
+	tcpDialCount                uint64
+	tcpDialTotalDur             uint64
+	tcpTimeForFirstByteTotalDur uint64
 }{}
 
 // RPCStats holds information about the DHCP/TCP metrics and errors
@@ -37,6 +40,7 @@ type RPCStats struct {
 	Errs uint64
 
 	DialAvgDuration uint64
+	TTFBAvgDuration uint64
 	DialErrs        uint64
 }
 
@@ -48,6 +52,7 @@ func GetRPCStats() RPCStats {
 	}
 	if v := atomic.LoadUint64(&globalStats.tcpDialCount); v > 0 {
 		s.DialAvgDuration = atomic.LoadUint64(&globalStats.tcpDialTotalDur) / v
+		s.TTFBAvgDuration = atomic.LoadUint64(&globalStats.tcpTimeForFirstByteTotalDur) / v
 	}
 	return s
 }
@@ -55,14 +60,19 @@ func GetRPCStats() RPCStats {
 // Return a function which update the global stats related to tcp connections
 func setupReqStatsUpdate(req *http.Request) (*http.Request, func()) {
 	var dialStart, dialEnd int64
-
+	start := time.Now()
 	trace := &httptrace.ClientTrace{
+		GotFirstResponseByte: func() {
+			atomic.AddUint64(&globalStats.tcpTimeForFirstByteTotalDur, uint64(time.Since(start)))
+		},
 		ConnectStart: func(network, addr string) {
 			atomic.StoreInt64(&dialStart, time.Now().UnixNano())
 		},
 		ConnectDone: func(network, addr string, err error) {
 			if err == nil {
 				atomic.StoreInt64(&dialEnd, time.Now().UnixNano())
+			} else {
+				logger.LogOnceIf(req.Context(), logSubsys, err, req.URL.Hostname())
 			}
 		},
 	}