Add encryption buffer (#8626)

Quite hard to measure difference:

```
λ warp cmp put-before.csv.zst put-after2.csv.zst
Operation: PUT
Operations: 340 -> 353
* Average: +4.11% (+22.7 MB/s) throughput, +4.11% (+0.2) obj/s
* 50% Median: +1.58% (+7.3 MB/s) throughput, +1.58% (+0.1) obj/s
```

Difference is likely bigger on Intel platforms due to higher syscall costs.

cmd/encryption-v1.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"bufio"
 	"context"
 	"crypto/hmac"
 	"crypto/rand"
@@ -225,10 +226,8 @@ func setEncryptionMetadata(r *http.Request, bucket, object string, metadata map[
 // with the client provided key. It also marks the object as client-side-encrypted
 // and sets the correct headers.
 func EncryptRequest(content io.Reader, r *http.Request, bucket, object string, metadata map[string]string) (reader io.Reader, objEncKey []byte, err error) {
+	var key []byte
 
-	var (
-		key []byte
-	)
 	if crypto.S3.IsRequested(r.Header) && crypto.SSEC.IsRequested(r.Header) {
 		return nil, objEncKey, crypto.ErrIncompatibleEncryptionMethod
 	}
@@ -238,6 +237,11 @@ func EncryptRequest(content io.Reader, r *http.Request, bucket, object string, m
 			return nil, objEncKey, err
 		}
 	}
+	if r.ContentLength > encryptBufferThreshold {
+		// The encryption reads in blocks of 64KB.
+		// We add a buffer on bigger files to reduce the number of syscalls upstream.
+		content = bufio.NewReaderSize(content, encryptBufferSize)
+	}
 	return newEncryptReader(content, key, bucket, object, metadata, crypto.S3.IsRequested(r.Header))
 }
 

cmd/object-handlers.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"bufio"
 	"context"
 	"crypto/hmac"
 	"encoding/binary"
@@ -64,6 +65,11 @@ var supportedHeadGetReqParams = map[string]string{
 const (
 	compressionAlgorithmV1 = "golang/snappy/LZ77"
 	compressionAlgorithmV2 = "klauspost/compress/s2"
+
+	// When an upload exceeds encryptBufferThreshold ...
+	encryptBufferThreshold = 1 << 20
+	// add an input buffer of this size.
+	encryptBufferSize = 1 << 20
 )
 
 // setHeadGetRespHeaders - set any requested parameters as response headers.
@@ -1975,7 +1981,13 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 			mac.Write(partIDbin[:])
 			partEncryptionKey := mac.Sum(nil)
 
-			reader, err = sio.EncryptReader(hashReader, sio.Config{Key: partEncryptionKey})
+			in := io.Reader(hashReader)
+			if size > encryptBufferThreshold {
+				// The encryption reads in blocks of 64KB.
+				// We add a buffer on bigger files to reduce the number of syscalls upstream.
+				in = bufio.NewReaderSize(hashReader, encryptBufferSize)
+			}
+			reader, err = sio.EncryptReader(in, sio.Config{Key: partEncryptionKey})
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return