replace io.Discard usage to fix some NUMA copy() latencies (#18394)

replace io.Discard usage to fix NUMA copy() latencies

On NUMA systems copying from 8K buffer allocated via
io.Discard leads to large latency build-up for every

```
copy(new8kbuf, largebuf)
```

can in-cur upto 1ms worth of latencies on NUMA systems
due to memory sharding across NUMA nodes.

internal/config/lambda/target/webhook.go

@@ -23,7 +23,6 @@ import (
 	"crypto/tls"
 	"encoding/json"
 	"errors"
-	"io"
 	"net/http"
 	"strings"
 	"sync/atomic"
@@ -31,6 +30,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/internal/config/lambda/event"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/certs"
 	xnet "github.com/minio/pkg/v2/net"
@@ -133,8 +133,7 @@ func (target *WebhookTarget) isActive() (bool, error) {
 		}
 		return false, err
 	}
-	io.Copy(io.Discard, resp.Body)
-	resp.Body.Close()
+	xioutil.DiscardReader(resp.Body)
 	// No network failure i.e response from the target means its up
 	return true, nil
 }

internal/event/target/elasticsearch.go

@@ -34,6 +34,7 @@ import (
 	elasticsearch7 "github.com/elastic/go-elasticsearch/v7"
 	"github.com/minio/highwayhash"
 	"github.com/minio/minio/internal/event"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/once"
 	"github.com/minio/minio/internal/store"
@@ -474,7 +475,7 @@ func (c *esClientV7) createIndex(args ElasticsearchArgs) error {
 		if err != nil {
 			return err
 		}
-		defer DrainBody(resp.Body)
+		defer xioutil.DiscardReader(resp.Body)
 		if resp.IsError() {
 			return fmt.Errorf("Create index err: %v", res)
 		}
@@ -490,7 +491,7 @@ func (c *esClientV7) ping(ctx context.Context, _ ElasticsearchArgs) (bool, error
 	if err != nil {
 		return false, store.ErrNotConnected
 	}
-	DrainBody(resp.Body)
+	xioutil.DiscardReader(resp.Body)
 	return !resp.IsError(), nil
 }
 
@@ -503,7 +504,7 @@ func (c *esClientV7) entryExists(ctx context.Context, index string, key string)
 	if err != nil {
 		return false, err
 	}
-	DrainBody(res.Body)
+	xioutil.DiscardReader(res.Body)
 	return !res.IsError(), nil
 }
 
@@ -518,7 +519,7 @@ func (c *esClientV7) removeEntry(ctx context.Context, index string, key string)
 		if err != nil {
 			return err
 		}
-		defer DrainBody(res.Body)
+		defer xioutil.DiscardReader(res.Body)
 		if res.IsError() {
 			return fmt.Errorf("Delete err: %s", res.String())
 		}
@@ -546,7 +547,7 @@ func (c *esClientV7) updateEntry(ctx context.Context, index string, key string,
 	if err != nil {
 		return err
 	}
-	defer DrainBody(res.Body)
+	defer xioutil.DiscardReader(res.Body)
 	if res.IsError() {
 		return fmt.Errorf("Update err: %s", res.String())
 	}
@@ -572,7 +573,7 @@ func (c *esClientV7) addEntry(ctx context.Context, index string, eventData event
 	if err != nil {
 		return err
 	}
-	defer DrainBody(res.Body)
+	defer xioutil.DiscardReader(res.Body)
 	if res.IsError() {
 		return fmt.Errorf("Add err: %s", res.String())
 	}

internal/event/target/webhook.go

@@ -24,7 +24,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
 	"net"
 	"net/http"
 	"net/url"
@@ -34,8 +33,8 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/dustin/go-humanize"
 	"github.com/minio/minio/internal/event"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/once"
 	"github.com/minio/minio/internal/store"
@@ -197,7 +196,7 @@ func (target *WebhookTarget) send(eventData event.Event) error {
 	if err != nil {
 		return err
 	}
-	defer DrainBody(resp.Body)
+	defer xioutil.DiscardReader(resp.Body)
 
 	if resp.StatusCode < 200 || resp.StatusCode > 299 {
 		return fmt.Errorf("sending event failed with %v", resp.Status)
@@ -312,23 +311,3 @@ func NewWebhookTarget(ctx context.Context, id string, args WebhookArgs, loggerOn
 
 	return target, nil
 }
-
-// DrainBody close non nil response with any response Body.
-// convenient wrapper to drain any remaining data on response body.
-//
-// Subsequently this allows golang http RoundTripper
-// to re-use the same connection for future requests.
-func DrainBody(respBody io.ReadCloser) {
-	// Callers should close resp.Body when done reading from it.
-	// If resp.Body is not closed, the Client's underlying RoundTripper
-	// (typically Transport) may not be able to re-use a persistent TCP
-	// connection to the server for a subsequent "keep-alive" request.
-	if respBody != nil {
-		// Drain any remaining Body and then close the connection.
-		// Without this closing connection would disallow re-using
-		// the same connection for future uses.
-		//  - http://stackoverflow.com/a/17961593/4465767
-		defer respBody.Close()
-		io.CopyN(io.Discard, respBody, 1*humanize.MiByte)
-	}
-}

internal/http/close.go

@@ -19,6 +19,8 @@ package http
 
 import (
 	"io"
+
+	xioutil "github.com/minio/minio/internal/ioutil"
 )
 
 // DrainBody close non nil response with any response Body.
@@ -37,6 +39,6 @@ func DrainBody(respBody io.ReadCloser) {
 		// the same connection for future uses.
 		//  - http://stackoverflow.com/a/17961593/4465767
 		defer respBody.Close()
-		io.Copy(io.Discard, respBody)
+		xioutil.DiscardReader(respBody)
 	}
 }

internal/ioutil/discard.go

@@ -0,0 +1,40 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ioutil
+
+import (
+	"io"
+)
+
+// Discard is just like io.Discard without the io.ReaderFrom compatible
+// implementation which is buggy on NUMA systems, we have to use a simpler
+// io.Writer implementation alone avoids also unnecessary buffer copies,
+// and as such incurred latencies.
+var Discard io.Writer = discard{}
+
+// discard is /dev/null for Golang.
+type discard struct{}
+
+func (discard) Write(p []byte) (int, error) {
+	return len(p), nil
+}
+
+// DiscardReader discarded reader
+func DiscardReader(r io.Reader) {
+	Copy(Discard, r)
+}

internal/s3select/select.go

@@ -33,6 +33,7 @@ import (
 	"github.com/klauspost/compress/zstd"
 	gzip "github.com/klauspost/pgzip"
 	"github.com/minio/minio/internal/config"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/s3select/csv"
 	"github.com/minio/minio/internal/s3select/json"
 	"github.com/minio/minio/internal/s3select/parquet"
@@ -91,7 +92,7 @@ var bufioWriterPool = sync.Pool{
 	New: func() interface{} {
 		// io.Discard is just used to create the writer. Actual destination
 		// writer is set later by Reset() before using it.
-		return bufio.NewWriter(io.Discard)
+		return bufio.NewWriter(xioutil.Discard)
 	},
 }
 
@@ -467,7 +468,7 @@ func (s3Select *S3Select) marshal(buf *bytes.Buffer, record sql.Record) error {
 		// Use bufio Writer to prevent csv.Writer from allocating a new buffer.
 		bufioWriter := bufioWriterPool.Get().(*bufio.Writer)
 		defer func() {
-			bufioWriter.Reset(io.Discard)
+			bufioWriter.Reset(xioutil.Discard)
 			bufioWriterPool.Put(bufioWriter)
 		}()