mirror of
https://github.com/minio/minio.git
synced 2025-11-20 18:06:10 -05:00
S3 Select API Support for CSV (#6127)
Add support for trivial where clause cases
This commit is contained in:
Arjun Mishra
kannappanr
460
pkg/s3select/output.go
Normal file
460
pkg/s3select/output.go
Normal file
@@ -0,0 +1,460 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
// DO NOT EDIT THIS PACKAGE DIRECTLY: This follows the protocol defined by
|
||||
// AmazonS3 found at
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
|
||||
// Consult the Spec before making direct edits.
|
||||
|
||||
package s3select
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"hash/crc32"
|
||||
)
|
||||
|
||||
// Record Headers
|
||||
// -11 -event type - 7 - 7 "Records"
|
||||
// -13 -content-type -7 -24 "application/octet-stream"
|
||||
// -13 -message-type -7 5 "event"
|
||||
// This is predefined from AMZ protocol found here:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
|
||||
var recordHeaders []byte
|
||||
|
||||
// End Headers
|
||||
// -13 -message-type -7 -5 "event"
|
||||
// -11 -:event-type -7 -3 "End"
|
||||
// This is predefined from AMZ protocol found here:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
|
||||
var endHeaders []byte
|
||||
|
||||
// Continuation Headers
|
||||
// -13 -message-type -7 -5 "event"
|
||||
// -11 -:event-type -7 -4 "Cont"
|
||||
// This is predefined from AMZ protocol found here:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
|
||||
var contHeaders []byte
|
||||
|
||||
// Stat Headers
|
||||
// -11 -event type - 7 - 5 "Stat" -20
|
||||
// -13 -content-type -7 -8 "text/xml" -25
|
||||
// -13 -message-type -7 -5 "event" -22
|
||||
// This is predefined from AMZ protocol found here:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
|
||||
var statHeaders []byte
|
||||
|
||||
// Progress Headers
|
||||
// -11 -event type - 7 - 8 "Progress" -23
|
||||
// -13 -content-type -7 -8 "text/xml" -25
|
||||
// -13 -message-type -7 -5 "event" -22
|
||||
// This is predefined from AMZ protocol found here:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
|
||||
var progressHeaders []byte
|
||||
|
||||
// The length of the nonvariable portion of the ErrHeaders
|
||||
// The below are the specifications of the header for a "error" event
|
||||
// -11 -error-code - 7 - DEFINED "DEFINED"
|
||||
// -14 -error-message -7 -DEFINED "DEFINED"
|
||||
// -13 -message-type -7 -5 "error"
|
||||
// This is predefined from AMZ protocol found here:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
|
||||
var errHdrLen int
|
||||
|
||||
func init() {
|
||||
recordHeaders = writeRecordHeader()
|
||||
endHeaders = writeEndHeader()
|
||||
contHeaders = writeContHeader()
|
||||
statHeaders = writeStatHeader()
|
||||
progressHeaders = writeProgressHeader()
|
||||
errHdrLen = 55
|
||||
|
||||
}
|
||||
|
||||
// encodeString encodes a string in a []byte, lenBytes is the number of bytes
|
||||
// used to encode the length of the string.
|
||||
func encodeHeaderStringValue(s string) []byte {
|
||||
n := uint16(len(s))
|
||||
lenSlice := make([]byte, 2)
|
||||
binary.BigEndian.PutUint16(lenSlice[0:], n)
|
||||
return append(lenSlice, []byte(s)...)
|
||||
}
|
||||
func encodeHeaderStringName(s string) []byte {
|
||||
lenSlice := make([]byte, 1)
|
||||
lenSlice[0] = byte(len(s))
|
||||
return append(lenSlice, []byte(s)...)
|
||||
}
|
||||
|
||||
// encodeNumber encodes a number in a []byte, lenBytes is the number of bytes
|
||||
// used to encode the length of the string.
|
||||
func encodeNumber(n byte, lenBytes int) []byte {
|
||||
lenSlice := make([]byte, lenBytes)
|
||||
lenSlice[0] = n
|
||||
return lenSlice
|
||||
}
|
||||
|
||||
// writePayloadSize writes the 4byte payload size portion of the protocol.
|
||||
func writePayloadSize(payloadSize int, headerLength int) []byte {
|
||||
totalByteLen := make([]byte, 4)
|
||||
totalMsgLen := uint32(payloadSize + headerLength + 16)
|
||||
binary.BigEndian.PutUint32(totalByteLen, totalMsgLen)
|
||||
return totalByteLen
|
||||
}
|
||||
|
||||
// writeHeaderSize writes the 4byte header size portion of the protocol.
|
||||
func writeHeaderSize(headerLength int) []byte {
|
||||
totalHeaderLen := make([]byte, 4)
|
||||
totalLen := uint32(headerLength)
|
||||
binary.BigEndian.PutUint32(totalHeaderLen, totalLen)
|
||||
return totalHeaderLen
|
||||
}
|
||||
|
||||
// writeCRC writes the CRC for both the prelude and and the end of the protocol.
|
||||
func writeCRC(myBuffer []byte) []byte {
|
||||
// Calculate the CRC here:
|
||||
myCRC := make([]byte, 4)
|
||||
cksum := crc32.ChecksumIEEE(myBuffer)
|
||||
binary.BigEndian.PutUint32(myCRC, cksum)
|
||||
return myCRC
|
||||
}
|
||||
|
||||
// writePayload writes the Payload for those protocols which the Payload is
|
||||
// necessary.
|
||||
func writePayload(myPayload string) []byte {
|
||||
convertedPayload := []byte(myPayload)
|
||||
payloadStore := make([]byte, len(convertedPayload))
|
||||
copy(payloadStore[0:], myPayload)
|
||||
return payloadStore
|
||||
}
|
||||
|
||||
// writeRecordHeader is a function which writes the headers for the continuation
|
||||
// Message
|
||||
func writeRecordHeader() []byte {
|
||||
// This is predefined from AMZ protocol found here:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
|
||||
var currentMessage = &bytes.Buffer{}
|
||||
// 11 -event type - 7 - 7 "Records"
|
||||
// header name
|
||||
currentMessage.Write(encodeHeaderStringName(":event-type"))
|
||||
// header type
|
||||
currentMessage.Write(encodeNumber(7, 1))
|
||||
// header value and header value length
|
||||
currentMessage.Write(encodeHeaderStringValue("Records"))
|
||||
// Creation of the Header for Content-Type // 13 -content-type -7 -24
|
||||
// "application/octet-stream"
|
||||
// header name
|
||||
currentMessage.Write(encodeHeaderStringName(":content-type"))
|
||||
// header type
|
||||
currentMessage.Write(encodeNumber(7, 1))
|
||||
// header value and header value length
|
||||
currentMessage.Write(encodeHeaderStringValue("application/octet-stream"))
|
||||
// Creation of the Header for message-type 13 -message-type -7 5 "event"
|
||||
// header name
|
||||
currentMessage.Write(encodeHeaderStringName(":message-type"))
|
||||
// header type
|
||||
currentMessage.Write(encodeNumber(7, 1))
|
||||
// header value and header value length
|
||||
currentMessage.Write(encodeHeaderStringValue("event"))
|
||||
return currentMessage.Bytes()
|
||||
}
|
||||
|
||||
// writeEndHeader is a function which writes the headers for the continuation
|
||||
// Message
|
||||
func writeEndHeader() []byte {
|
||||
// This is predefined from AMZ protocol found here:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
|
||||
var currentMessage = &bytes.Buffer{}
|
||||
// header name
|
||||
currentMessage.Write(encodeHeaderStringName(":event-type"))
|
||||
// header type
|
||||
currentMessage.Write(encodeNumber(7, 1))
|
||||
// header value and header value length
|
||||
currentMessage.Write(encodeHeaderStringValue("End"))
|
||||
|
||||
// Creation of the Header for message-type 13 -message-type -7 5 "event"
|
||||
// header name
|
||||
currentMessage.Write(encodeHeaderStringName(":message-type"))
|
||||
// header type
|
||||
currentMessage.Write(encodeNumber(7, 1))
|
||||
// header value and header value length
|
||||
currentMessage.Write(encodeHeaderStringValue("event"))
|
||||
return currentMessage.Bytes()
|
||||
}
|
||||
|
||||
// writeContHeader is a function which writes the headers for the continuation
|
||||
// Message
|
||||
func writeContHeader() []byte {
|
||||
// This is predefined from AMZ protocol found here:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
|
||||
var currentMessage = &bytes.Buffer{}
|
||||
// header name
|
||||
currentMessage.Write(encodeHeaderStringName(":event-type"))
|
||||
// header type
|
||||
currentMessage.Write(encodeNumber(7, 1))
|
||||
// header value and header value length
|
||||
currentMessage.Write(encodeHeaderStringValue("Cont"))
|
||||
|
||||
// Creation of the Header for message-type 13 -message-type -7 5 "event"
|
||||
// header name
|
||||
currentMessage.Write(encodeHeaderStringName(":message-type"))
|
||||
// header type
|
||||
currentMessage.Write(encodeNumber(7, 1))
|
||||
// header value and header value length
|
||||
currentMessage.Write(encodeHeaderStringValue("event"))
|
||||
return currentMessage.Bytes()
|
||||
|
||||
}
|
||||
|
||||
// writeStatHeader is a function which writes the headers for the Stat
|
||||
// Message
|
||||
func writeStatHeader() []byte {
|
||||
// This is predefined from AMZ protocol found here:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
|
||||
var currentMessage = &bytes.Buffer{}
|
||||
// header name
|
||||
currentMessage.Write(encodeHeaderStringName(":event-type"))
|
||||
// header type
|
||||
currentMessage.Write(encodeNumber(7, 1))
|
||||
// header value and header value length
|
||||
currentMessage.Write(encodeHeaderStringValue("Stats"))
|
||||
// Creation of the Header for Content-Type // 13 -content-type -7 -8
|
||||
// "text/xml"
|
||||
// header name
|
||||
currentMessage.Write(encodeHeaderStringName(":content-type"))
|
||||
// header type
|
||||
currentMessage.Write(encodeNumber(7, 1))
|
||||
// header value and header value length
|
||||
currentMessage.Write(encodeHeaderStringValue("text/xml"))
|
||||
|
||||
// Creation of the Header for message-type 13 -message-type -7 5 "event"
|
||||
currentMessage.Write(encodeHeaderStringName(":message-type"))
|
||||
// header type
|
||||
currentMessage.Write(encodeNumber(7, 1))
|
||||
// header value and header value length
|
||||
currentMessage.Write(encodeHeaderStringValue("event"))
|
||||
return currentMessage.Bytes()
|
||||
|
||||
}
|
||||
|
||||
// writeProgressHeader is a function which writes the headers for the Progress
|
||||
// Message
|
||||
func writeProgressHeader() []byte {
|
||||
// This is predefined from AMZ protocol found here:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
|
||||
var currentMessage = &bytes.Buffer{}
|
||||
// header name
|
||||
currentMessage.Write(encodeHeaderStringName(":event-type"))
|
||||
// header type
|
||||
currentMessage.Write(encodeNumber(7, 1))
|
||||
// header value and header value length
|
||||
currentMessage.Write(encodeHeaderStringValue("Progress"))
|
||||
// Creation of the Header for Content-Type // 13 -content-type -7 -8
|
||||
// "text/xml"
|
||||
// header name
|
||||
currentMessage.Write(encodeHeaderStringName(":content-type"))
|
||||
// header type
|
||||
currentMessage.Write(encodeNumber(7, 1))
|
||||
// header value and header value length
|
||||
currentMessage.Write(encodeHeaderStringValue("text/xml"))
|
||||
|
||||
// Creation of the Header for message-type 13 -message-type -7 5 "event"
|
||||
// header name
|
||||
currentMessage.Write(encodeHeaderStringName(":message-type"))
|
||||
// header type
|
||||
currentMessage.Write(encodeNumber(7, 1))
|
||||
// header value and header value length
|
||||
currentMessage.Write(encodeHeaderStringValue("event"))
|
||||
return currentMessage.Bytes()
|
||||
|
||||
}
|
||||
|
||||
// writeRecordMessage is the function which constructs the binary message for a
|
||||
// record message to be sent.
|
||||
func (csvOutput *Input) writeRecordMessage(payload string, currentMessage *bytes.Buffer) *bytes.Buffer {
|
||||
// The below are the specifications of the header for a "record" event
|
||||
// 11 -event type - 7 - 7 "Records"
|
||||
// 13 -content-type -7 -24 "application/octet-stream"
|
||||
// 13 -message-type -7 5 "event"
|
||||
// This is predefined from AMZ protocol found here:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
|
||||
headerLen := len(recordHeaders)
|
||||
// Writes the total size of the message.
|
||||
currentMessage.Write(writePayloadSize(len(payload), headerLen))
|
||||
// Writes the total size of the header.
|
||||
currentMessage.Write(writeHeaderSize(headerLen))
|
||||
// Writes the CRC of the Prelude
|
||||
currentMessage.Write(writeCRC(currentMessage.Bytes()))
|
||||
currentMessage.Write(recordHeaders)
|
||||
|
||||
// This part is where the payload is written, this will be only one row, since
|
||||
// we're sending one message at a types
|
||||
currentMessage.Write(writePayload(payload))
|
||||
|
||||
// Now we do a CRC check on the entire messages
|
||||
currentMessage.Write(writeCRC(currentMessage.Bytes()))
|
||||
return currentMessage
|
||||
|
||||
}
|
||||
|
||||
// writeContinuationMessage is the function which constructs the binary message
|
||||
// for a continuation message to be sent.
|
||||
func (csvOutput *Input) writeContinuationMessage(currentMessage *bytes.Buffer) *bytes.Buffer {
|
||||
// 11 -event type - 7 - 4 "Cont"
|
||||
// 13 -message-type -7 5 "event"
|
||||
// This is predefined from AMZ protocol found here:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
|
||||
headerLen := len(contHeaders)
|
||||
currentMessage.Write(writePayloadSize(0, headerLen))
|
||||
|
||||
currentMessage.Write(writeHeaderSize(headerLen))
|
||||
|
||||
// Calculate the Prelude CRC here:
|
||||
currentMessage.Write(writeCRC(currentMessage.Bytes()))
|
||||
|
||||
currentMessage.Write(contHeaders)
|
||||
|
||||
//Now we do a CRC check on the entire messages
|
||||
currentMessage.Write(writeCRC(currentMessage.Bytes()))
|
||||
return currentMessage
|
||||
|
||||
}
|
||||
|
||||
// writeEndMessage is the function which constructs the binary message
|
||||
// for a end message to be sent.
|
||||
func (csvOutput *Input) writeEndMessage(currentMessage *bytes.Buffer) *bytes.Buffer {
|
||||
// 11 -event type - 7 - 3 "End"
|
||||
// 13 -message-type -7 5 "event"
|
||||
// This is predefined from AMZ protocol found here:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
|
||||
headerLen := len(endHeaders)
|
||||
currentMessage.Write(writePayloadSize(0, headerLen))
|
||||
|
||||
currentMessage.Write(writeHeaderSize(headerLen))
|
||||
|
||||
//Calculate the Prelude CRC here:
|
||||
currentMessage.Write(writeCRC(currentMessage.Bytes()))
|
||||
|
||||
currentMessage.Write(endHeaders)
|
||||
|
||||
// Now we do a CRC check on the entire messages
|
||||
currentMessage.Write(writeCRC(currentMessage.Bytes()))
|
||||
return currentMessage
|
||||
|
||||
}
|
||||
|
||||
// writeStateMessage is the function which constructs the binary message for a
|
||||
// state message to be sent.
|
||||
func (csvOutput *Input) writeStatMessage(payload string, currentMessage *bytes.Buffer) *bytes.Buffer {
|
||||
// 11 -event type - 7 - 5 "Stat" 20
|
||||
// 13 -content-type -7 -8 "text/xml" 25
|
||||
// 13 -message-type -7 5 "event" 22
|
||||
// This is predefined from AMZ protocol found here:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
|
||||
headerLen := len(statHeaders)
|
||||
|
||||
currentMessage.Write(writePayloadSize(len(payload), headerLen))
|
||||
|
||||
currentMessage.Write(writeHeaderSize(headerLen))
|
||||
|
||||
currentMessage.Write(writeCRC(currentMessage.Bytes()))
|
||||
|
||||
currentMessage.Write(statHeaders)
|
||||
|
||||
// This part is where the payload is written, this will be only one row, since
|
||||
// we're sending one message at a types
|
||||
currentMessage.Write(writePayload(payload))
|
||||
|
||||
// Now we do a CRC check on the entire messages
|
||||
currentMessage.Write(writeCRC(currentMessage.Bytes()))
|
||||
return currentMessage
|
||||
|
||||
}
|
||||
|
||||
// writeProgressMessage is the function which constructs the binary message for
|
||||
// a progress message to be sent.
|
||||
func (csvOutput *Input) writeProgressMessage(payload string, currentMessage *bytes.Buffer) *bytes.Buffer {
|
||||
// The below are the specifications of the header for a "Progress" event
|
||||
// 11 -event type - 7 - 8 "Progress" 23
|
||||
// 13 -content-type -7 -8 "text/xml" 25
|
||||
// 13 -message-type -7 5 "event" 22
|
||||
// This is predefined from AMZ protocol found here:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
|
||||
headerLen := len(progressHeaders)
|
||||
|
||||
currentMessage.Write(writePayloadSize(len(payload), headerLen))
|
||||
|
||||
currentMessage.Write(writeHeaderSize(headerLen))
|
||||
|
||||
currentMessage.Write(writeCRC(currentMessage.Bytes()))
|
||||
|
||||
currentMessage.Write(progressHeaders)
|
||||
|
||||
// This part is where the payload is written, this will be only one row, since
|
||||
// we're sending one message at a types
|
||||
currentMessage.Write(writePayload(payload))
|
||||
|
||||
// Now we do a CRC check on the entire messages
|
||||
currentMessage.Write(writeCRC(currentMessage.Bytes()))
|
||||
return currentMessage
|
||||
|
||||
}
|
||||
|
||||
// writeErrorMessage is the function which constructs the binary message for a
|
||||
// error message to be sent.
|
||||
func (csvOutput *Input) writeErrorMessage(errorMessage error, currentMessage *bytes.Buffer) *bytes.Buffer {
|
||||
|
||||
// The below are the specifications of the header for a "error" event
|
||||
// 11 -error-code - 7 - DEFINED "DEFINED"
|
||||
// 14 -error-message -7 -DEFINED "DEFINED"
|
||||
// 13 -message-type -7 5 "error"
|
||||
// This is predefined from AMZ protocol found here:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
|
||||
sizeOfErrorCode := len(errorCodeResponse[errorMessage])
|
||||
sizeOfErrorMessage := len(errorMessage.Error())
|
||||
headerLen := errHdrLen + sizeOfErrorCode + sizeOfErrorMessage
|
||||
|
||||
currentMessage.Write(writePayloadSize(0, headerLen))
|
||||
|
||||
currentMessage.Write(writeHeaderSize(headerLen))
|
||||
|
||||
currentMessage.Write(writeCRC(currentMessage.Bytes()))
|
||||
// header name
|
||||
currentMessage.Write(encodeHeaderStringName(":error-code"))
|
||||
// header type
|
||||
currentMessage.Write(encodeNumber(7, 1))
|
||||
// header value and header value length
|
||||
currentMessage.Write(encodeHeaderStringValue(errorCodeResponse[errorMessage]))
|
||||
|
||||
// 14 -error-message -7 -DEFINED "DEFINED"
|
||||
|
||||
// header name
|
||||
currentMessage.Write(encodeHeaderStringName(":error-message"))
|
||||
// header type
|
||||
currentMessage.Write(encodeNumber(7, 1))
|
||||
// header value and header value length
|
||||
currentMessage.Write(encodeHeaderStringValue(errorMessage.Error()))
|
||||
// Creation of the Header for message-type 13 -message-type -7 5 "error"
|
||||
// header name
|
||||
currentMessage.Write(encodeHeaderStringName(":message-type"))
|
||||
// header type
|
||||
currentMessage.Write(encodeNumber(7, 1))
|
||||
// header value and header value length
|
||||
currentMessage.Write(encodeHeaderStringValue("error"))
|
||||
|
||||
// Now we do a CRC check on the entire messages
|
||||
currentMessage.Write(writeCRC(currentMessage.Bytes()))
|
||||
return currentMessage
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user