diff --git a/pkg/s3select/message.go b/pkg/s3select/message.go index ec6a7e0ae..1733e905d 100644 --- a/pkg/s3select/message.go +++ b/pkg/s3select/message.go @@ -62,7 +62,12 @@ var recordsHeader = []byte{ } const ( - maxRecordMessageLength = 128 * 1024 // Chosen for compatibility with AWS JAVA SDK + // Chosen for compatibility with AWS JAVA SDK + // It has a a buffer size of 128K: + // https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/internal/eventstreaming/MessageDecoder.java#L26 + // but we must make sure there is always space to add 256 bytes: + // https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/model/SelectObjectContentEventStream.java#L197 + maxRecordMessageLength = (128 << 10) - 256 ) var ( @@ -83,7 +88,7 @@ func newRecordsMessage(payload []byte) []byte { } // payloadLenForMsgLen computes the length of the payload in a record -// message given the length of the message. +// message given the total length of the message. func payloadLenForMsgLen(messageLength int) int { headerLength := len(recordsHeader) payloadLength := messageLength - 4 - 4 - 4 - headerLength - 4