mirror of
https://github.com/minio/minio.git
synced 2025-11-10 22:10:12 -05:00
SQL select query for CSV/JSON (#6648)
select * , select column names have been implemented for CSV. select * is implemented for JSON.
This commit is contained in:
Ashish Kumar Sinha
committed by
kannappanr
kannappanr
parent
acf46cc3b5
commit
c0b4bf0a3e
334
pkg/s3select/format/csv/csv.go
Normal file
334
pkg/s3select/format/csv/csv.go
Normal file
@@ -0,0 +1,334 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package csv
|
||||
|
||||
import (
|
||||
"compress/bzip2"
|
||||
"encoding/csv"
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
gzip "github.com/klauspost/pgzip"
|
||||
"github.com/minio/minio/pkg/ioutil"
|
||||
"github.com/minio/minio/pkg/s3select/format"
|
||||
)
|
||||
|
||||
// Options options are passed to the underlying encoding/csv reader.
|
||||
type Options struct {
|
||||
// HasHeader when true, will treat the first row as a header row.
|
||||
HasHeader bool
|
||||
|
||||
// RecordDelimiter is the string that records are delimited by.
|
||||
RecordDelimiter string
|
||||
|
||||
// FieldDelimiter is the string that fields are delimited by.
|
||||
FieldDelimiter string
|
||||
|
||||
// Comments is the string the first character of a line of
|
||||
// text matches the comment character.
|
||||
Comments string
|
||||
|
||||
// Name of the table that is used for querying
|
||||
Name string
|
||||
|
||||
// ReadFrom is where the data will be read from.
|
||||
ReadFrom io.Reader
|
||||
|
||||
// If true then we need to add gzip or bzip reader.
|
||||
// to extract the csv.
|
||||
Compressed string
|
||||
|
||||
// SQL expression meant to be evaluated.
|
||||
Expression string
|
||||
|
||||
// What the outputted CSV will be delimited by .
|
||||
OutputFieldDelimiter string
|
||||
|
||||
// Size of incoming object
|
||||
StreamSize int64
|
||||
|
||||
// Whether Header is "USE" or another
|
||||
HeaderOpt bool
|
||||
|
||||
// Progress enabled, enable/disable progress messages.
|
||||
Progress bool
|
||||
}
|
||||
|
||||
// cinput represents a record producing input from a formatted object.
|
||||
type cinput struct {
|
||||
options *Options
|
||||
reader *csv.Reader
|
||||
firstRow []string
|
||||
header []string
|
||||
minOutputLength int
|
||||
stats struct {
|
||||
BytesScanned int64
|
||||
BytesReturned int64
|
||||
BytesProcessed int64
|
||||
}
|
||||
}
|
||||
|
||||
// New sets up a new Input, the first row is read when this is run.
|
||||
// If there is a problem with reading the first row, the error is returned.
|
||||
// Otherwise, the returned reader can be reliably consumed with Read().
|
||||
// until Read() return err.
|
||||
func New(opts *Options) (format.Select, error) {
|
||||
myReader := opts.ReadFrom
|
||||
var tempBytesScanned int64
|
||||
tempBytesScanned = 0
|
||||
switch opts.Compressed {
|
||||
case "GZIP":
|
||||
tempBytesScanned = opts.StreamSize
|
||||
var err error
|
||||
if myReader, err = gzip.NewReader(opts.ReadFrom); err != nil {
|
||||
return nil, format.ErrTruncatedInput
|
||||
}
|
||||
case "BZIP2":
|
||||
tempBytesScanned = opts.StreamSize
|
||||
myReader = bzip2.NewReader(opts.ReadFrom)
|
||||
}
|
||||
|
||||
// DelimitedReader treats custom record delimiter like `\r\n`,`\r`,`ab` etc and replaces it with `\n`.
|
||||
normalizedReader := ioutil.NewDelimitedReader(myReader, []rune(opts.RecordDelimiter))
|
||||
reader := &cinput{
|
||||
options: opts,
|
||||
reader: csv.NewReader(normalizedReader),
|
||||
}
|
||||
reader.stats.BytesScanned = tempBytesScanned
|
||||
reader.stats.BytesProcessed = 0
|
||||
reader.stats.BytesReturned = 0
|
||||
|
||||
reader.firstRow = nil
|
||||
|
||||
reader.reader.FieldsPerRecord = -1
|
||||
if reader.options.FieldDelimiter != "" {
|
||||
reader.reader.Comma = rune(reader.options.FieldDelimiter[0])
|
||||
}
|
||||
|
||||
if reader.options.Comments != "" {
|
||||
reader.reader.Comment = rune(reader.options.Comments[0])
|
||||
}
|
||||
|
||||
// QuoteCharacter - " (defaulted currently)
|
||||
reader.reader.LazyQuotes = true
|
||||
|
||||
if err := reader.readHeader(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
// Replace the spaces in columnnames with underscores
|
||||
func cleanHeader(columns []string) []string {
|
||||
for i := 0; i < len(columns); i++ {
|
||||
columns[i] = strings.Replace(columns[i], " ", "_", -1)
|
||||
}
|
||||
return columns
|
||||
}
|
||||
|
||||
// readHeader reads the header into the header variable if the header is present
|
||||
// as the first row of the csv
|
||||
func (reader *cinput) readHeader() error {
|
||||
var readErr error
|
||||
if reader.options.HasHeader {
|
||||
reader.firstRow, readErr = reader.reader.Read()
|
||||
if readErr != nil {
|
||||
return format.ErrCSVParsingError
|
||||
}
|
||||
reader.header = cleanHeader(reader.firstRow)
|
||||
reader.firstRow = nil
|
||||
reader.minOutputLength = len(reader.header)
|
||||
} else {
|
||||
reader.firstRow, readErr = reader.reader.Read()
|
||||
reader.header = make([]string, len(reader.firstRow))
|
||||
for i := 0; i < reader.minOutputLength; i++ {
|
||||
reader.header[i] = strconv.Itoa(i)
|
||||
}
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Progress - return true if progress was requested.
|
||||
func (reader *cinput) Progress() bool {
|
||||
return reader.options.Progress
|
||||
}
|
||||
|
||||
// UpdateBytesProcessed - populates the bytes Processed
|
||||
func (reader *cinput) UpdateBytesProcessed(record map[string]interface{}) {
|
||||
// Convert map to slice of values.
|
||||
values := []string{}
|
||||
for _, value := range record {
|
||||
values = append(values, value.(string))
|
||||
}
|
||||
|
||||
reader.stats.BytesProcessed += int64(len(values))
|
||||
|
||||
}
|
||||
|
||||
// Read the file and returns map[string]interface{}
|
||||
func (reader *cinput) Read() (map[string]interface{}, error) {
|
||||
record := make(map[string]interface{})
|
||||
dec := reader.readRecord()
|
||||
if dec != nil {
|
||||
if reader.options.HasHeader {
|
||||
columns := reader.header
|
||||
for i, value := range dec {
|
||||
record[columns[i]] = value
|
||||
}
|
||||
} else {
|
||||
for i, value := range dec {
|
||||
record["_"+strconv.Itoa(i)] = value
|
||||
}
|
||||
}
|
||||
return record, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// OutputFieldDelimiter - returns the delimiter specified in input request
|
||||
func (reader *cinput) OutputFieldDelimiter() string {
|
||||
return reader.options.OutputFieldDelimiter
|
||||
}
|
||||
|
||||
// HasHeader - returns true or false depending upon the header.
|
||||
func (reader *cinput) HasHeader() bool {
|
||||
return reader.options.HasHeader
|
||||
}
|
||||
|
||||
// Expression - return the Select Expression for
|
||||
func (reader *cinput) Expression() string {
|
||||
return reader.options.Expression
|
||||
}
|
||||
|
||||
// UpdateBytesReturned - updates the Bytes returned for
|
||||
func (reader *cinput) UpdateBytesReturned(size int64) {
|
||||
reader.stats.BytesReturned += size
|
||||
}
|
||||
|
||||
// Header returns the header of the reader. Either the first row if a header
|
||||
// set in the options, or c#, where # is the column number, starting with 0.
|
||||
func (reader *cinput) Header() []string {
|
||||
return reader.header
|
||||
}
|
||||
|
||||
// readRecord reads a single record from the stream and it always returns successfully.
|
||||
// If the record is empty, an empty []string is returned.
|
||||
// Record expand to match the current row size, adding blank fields as needed.
|
||||
// Records never return less then the number of fields in the first row.
|
||||
// Returns nil on EOF
|
||||
// In the event of a parse error due to an invalid record, it is logged, and
|
||||
// an empty []string is returned with the number of fields in the first row,
|
||||
// as if the record were empty.
|
||||
//
|
||||
// In general, this is a very tolerant of problems reader.
|
||||
func (reader *cinput) readRecord() []string {
|
||||
var row []string
|
||||
var fileErr error
|
||||
|
||||
if reader.firstRow != nil {
|
||||
row = reader.firstRow
|
||||
reader.firstRow = nil
|
||||
return row
|
||||
}
|
||||
|
||||
row, fileErr = reader.reader.Read()
|
||||
emptysToAppend := reader.minOutputLength - len(row)
|
||||
if fileErr == io.EOF || fileErr == io.ErrClosedPipe {
|
||||
return nil
|
||||
} else if _, ok := fileErr.(*csv.ParseError); ok {
|
||||
emptysToAppend = reader.minOutputLength
|
||||
}
|
||||
|
||||
if emptysToAppend > 0 {
|
||||
for counter := 0; counter < emptysToAppend; counter++ {
|
||||
row = append(row, "")
|
||||
}
|
||||
}
|
||||
|
||||
return row
|
||||
}
|
||||
|
||||
// CreateStatXML is the function which does the marshaling from the stat
|
||||
// structs into XML so that the progress and stat message can be sent
|
||||
func (reader *cinput) CreateStatXML() (string, error) {
|
||||
if reader.options.Compressed == "NONE" {
|
||||
reader.stats.BytesProcessed = reader.options.StreamSize
|
||||
reader.stats.BytesScanned = reader.stats.BytesProcessed
|
||||
}
|
||||
out, err := xml.Marshal(&format.Stats{
|
||||
BytesScanned: reader.stats.BytesScanned,
|
||||
BytesProcessed: reader.stats.BytesProcessed,
|
||||
BytesReturned: reader.stats.BytesReturned,
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return xml.Header + string(out), nil
|
||||
}
|
||||
|
||||
// CreateProgressXML is the function which does the marshaling from the progress
|
||||
// structs into XML so that the progress and stat message can be sent
|
||||
func (reader *cinput) CreateProgressXML() (string, error) {
|
||||
if reader.options.HasHeader {
|
||||
reader.stats.BytesProcessed += format.ProcessSize(reader.header)
|
||||
}
|
||||
if reader.options.Compressed == "NONE" {
|
||||
reader.stats.BytesScanned = reader.stats.BytesProcessed
|
||||
}
|
||||
out, err := xml.Marshal(&format.Progress{
|
||||
BytesScanned: reader.stats.BytesScanned,
|
||||
BytesProcessed: reader.stats.BytesProcessed,
|
||||
BytesReturned: reader.stats.BytesReturned,
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return xml.Header + string(out), nil
|
||||
}
|
||||
|
||||
// Type - return the data format type {
|
||||
func (reader *cinput) Type() format.Type {
|
||||
return format.CSV
|
||||
}
|
||||
|
||||
// ColNameErrs is a function which makes sure that the headers are requested are
|
||||
// present in the file otherwise it throws an error.
|
||||
func (reader *cinput) ColNameErrs(columnNames []string) error {
|
||||
for i := 0; i < len(columnNames); i++ {
|
||||
if columnNames[i] == "" {
|
||||
continue
|
||||
}
|
||||
if !format.IsInt(columnNames[i]) && !reader.options.HeaderOpt {
|
||||
return format.ErrInvalidColumnIndex
|
||||
}
|
||||
if format.IsInt(columnNames[i]) {
|
||||
tempInt, _ := strconv.Atoi(columnNames[i])
|
||||
if tempInt > len(reader.Header()) || tempInt == 0 {
|
||||
return format.ErrInvalidColumnIndex
|
||||
}
|
||||
} else {
|
||||
if reader.options.HeaderOpt && !format.StringInSlice(columnNames[i], reader.Header()) {
|
||||
return format.ErrParseInvalidPathComponent
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
38
pkg/s3select/format/errors.go
Normal file
38
pkg/s3select/format/errors.go
Normal file
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package format
|
||||
|
||||
import "errors"
|
||||
|
||||
// ErrTruncatedInput is an error if the object is not compressed properly and an
|
||||
// error occurs during decompression.
|
||||
var ErrTruncatedInput = errors.New("Object decompression failed. Check that the object is properly compressed using the format specified in the request")
|
||||
|
||||
// ErrCSVParsingError is an error if the CSV presents an error while being
|
||||
// parsed.
|
||||
var ErrCSVParsingError = errors.New("Encountered an Error parsing the CSV file. Check the file and try again")
|
||||
|
||||
// ErrInvalidColumnIndex is an error if you provide a column index which is not
|
||||
// valid.
|
||||
var ErrInvalidColumnIndex = errors.New("Column index in the SQL expression is invalid")
|
||||
|
||||
// ErrParseInvalidPathComponent is an error that occurs if there is an invalid
|
||||
// path component.
|
||||
var ErrParseInvalidPathComponent = errors.New("The SQL expression contains an invalid path component")
|
||||
|
||||
// ErrJSONParsingError is an error if while parsing the JSON an error arises.
|
||||
var ErrJSONParsingError = errors.New("Encountered an error parsing the JSON file. Check the file and try again")
|
||||
50
pkg/s3select/format/helpers.go
Normal file
50
pkg/s3select/format/helpers.go
Normal file
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package format
|
||||
|
||||
import "strconv"
|
||||
|
||||
// IsInt - returns a true or false, whether a string can
|
||||
// be represented as an int.
|
||||
func IsInt(s string) bool {
|
||||
_, err := strconv.Atoi(s)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// StringInSlice - this function finds whether a string is in a list
|
||||
func StringInSlice(x string, list []string) bool {
|
||||
for _, y := range list {
|
||||
if x == y {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ProcessSize - this function processes size so that we can calculate bytes BytesProcessed.
|
||||
func ProcessSize(myrecord []string) int64 {
|
||||
if len(myrecord) > 0 {
|
||||
var size int64
|
||||
size = int64(len(myrecord)-1) + 1
|
||||
for i := range myrecord {
|
||||
size += int64(len(myrecord[i]))
|
||||
}
|
||||
|
||||
return size
|
||||
}
|
||||
return 0
|
||||
}
|
||||
200
pkg/s3select/format/json/json.go
Normal file
200
pkg/s3select/format/json/json.go
Normal file
@@ -0,0 +1,200 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package json
|
||||
|
||||
import (
|
||||
"compress/bzip2"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"io"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
gzip "github.com/klauspost/pgzip"
|
||||
"github.com/minio/minio/pkg/s3select/format"
|
||||
)
|
||||
|
||||
// Options options are passed to the underlying encoding/json reader.
|
||||
type Options struct {
|
||||
|
||||
// Name of the table that is used for querying
|
||||
Name string
|
||||
|
||||
// ReadFrom is where the data will be read from.
|
||||
ReadFrom io.Reader
|
||||
|
||||
// If true then we need to add gzip or bzip reader.
|
||||
// to extract the csv.
|
||||
Compressed string
|
||||
|
||||
// SQL expression meant to be evaluated.
|
||||
Expression string
|
||||
|
||||
// What the outputted will be delimited by .
|
||||
RecordDelimiter string
|
||||
|
||||
// Size of incoming object
|
||||
StreamSize int64
|
||||
|
||||
// True if Type is DOCUMENTS
|
||||
Type bool
|
||||
|
||||
// Progress enabled, enable/disable progress messages.
|
||||
Progress bool
|
||||
}
|
||||
|
||||
// jinput represents a record producing input from a formatted file or pipe.
|
||||
type jinput struct {
|
||||
options *Options
|
||||
reader *jsoniter.Decoder
|
||||
firstRow []string
|
||||
header []string
|
||||
minOutputLength int
|
||||
stats struct {
|
||||
BytesScanned int64
|
||||
BytesReturned int64
|
||||
BytesProcessed int64
|
||||
}
|
||||
}
|
||||
|
||||
// New sets up a new, the first Json is read when this is run.
|
||||
// If there is a problem with reading the first Json, the error is returned.
|
||||
// Otherwise, the returned reader can be reliably consumed with jsonRead()
|
||||
// until jsonRead() returns nil.
|
||||
func New(opts *Options) (format.Select, error) {
|
||||
myReader := opts.ReadFrom
|
||||
var tempBytesScanned int64
|
||||
tempBytesScanned = 0
|
||||
switch opts.Compressed {
|
||||
case "GZIP":
|
||||
tempBytesScanned = opts.StreamSize
|
||||
var err error
|
||||
if myReader, err = gzip.NewReader(opts.ReadFrom); err != nil {
|
||||
return nil, format.ErrTruncatedInput
|
||||
}
|
||||
case "BZIP2":
|
||||
tempBytesScanned = opts.StreamSize
|
||||
myReader = bzip2.NewReader(opts.ReadFrom)
|
||||
}
|
||||
|
||||
reader := &jinput{
|
||||
options: opts,
|
||||
reader: jsoniter.NewDecoder(myReader),
|
||||
}
|
||||
reader.stats.BytesScanned = tempBytesScanned
|
||||
reader.stats.BytesProcessed = 0
|
||||
reader.stats.BytesReturned = 0
|
||||
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
// Progress - return true if progress was requested.
|
||||
func (reader *jinput) Progress() bool {
|
||||
return reader.options.Progress
|
||||
}
|
||||
|
||||
// UpdateBytesProcessed - populates the bytes Processed
|
||||
func (reader *jinput) UpdateBytesProcessed(record map[string]interface{}) {
|
||||
out, _ := json.Marshal(record)
|
||||
reader.stats.BytesProcessed += int64(len(out))
|
||||
}
|
||||
|
||||
// Read the file and returns map[string]interface{}
|
||||
func (reader *jinput) Read() (map[string]interface{}, error) {
|
||||
dec := reader.reader
|
||||
var record interface{}
|
||||
for {
|
||||
err := dec.Decode(&record)
|
||||
if err == io.EOF || err == io.ErrClosedPipe {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, format.ErrJSONParsingError
|
||||
}
|
||||
return record.(map[string]interface{}), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// OutputFieldDelimiter - returns the delimiter specified in input request
|
||||
func (reader *jinput) OutputFieldDelimiter() string {
|
||||
return ","
|
||||
}
|
||||
|
||||
// HasHeader - returns true or false depending upon the header.
|
||||
func (reader *jinput) HasHeader() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Expression - return the Select Expression for
|
||||
func (reader *jinput) Expression() string {
|
||||
return reader.options.Expression
|
||||
}
|
||||
|
||||
// UpdateBytesReturned - updates the Bytes returned for
|
||||
func (reader *jinput) UpdateBytesReturned(size int64) {
|
||||
reader.stats.BytesReturned += size
|
||||
}
|
||||
|
||||
// Header returns a nil in case of
|
||||
func (reader *jinput) Header() []string {
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateStatXML is the function which does the marshaling from the stat
|
||||
// structs into XML so that the progress and stat message can be sent
|
||||
func (reader *jinput) CreateStatXML() (string, error) {
|
||||
if reader.options.Compressed == "NONE" {
|
||||
reader.stats.BytesProcessed = reader.options.StreamSize
|
||||
reader.stats.BytesScanned = reader.stats.BytesProcessed
|
||||
}
|
||||
out, err := xml.Marshal(&format.Stats{
|
||||
BytesScanned: reader.stats.BytesScanned,
|
||||
BytesProcessed: reader.stats.BytesProcessed,
|
||||
BytesReturned: reader.stats.BytesReturned,
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return xml.Header + string(out), nil
|
||||
}
|
||||
|
||||
// CreateProgressXML is the function which does the marshaling from the progress
|
||||
// structs into XML so that the progress and stat message can be sent
|
||||
func (reader *jinput) CreateProgressXML() (string, error) {
|
||||
if !(reader.options.Compressed != "NONE") {
|
||||
reader.stats.BytesScanned = reader.stats.BytesProcessed
|
||||
}
|
||||
out, err := xml.Marshal(&format.Progress{
|
||||
BytesScanned: reader.stats.BytesScanned,
|
||||
BytesProcessed: reader.stats.BytesProcessed,
|
||||
BytesReturned: reader.stats.BytesReturned,
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return xml.Header + string(out), nil
|
||||
}
|
||||
|
||||
// Type - return the data format type {
|
||||
func (reader *jinput) Type() format.Type {
|
||||
return format.JSON
|
||||
}
|
||||
|
||||
// ColNameErrs - this is a dummy function for JSON input type.
|
||||
func (reader *jinput) ColNameErrs(columnNames []string) error {
|
||||
return nil
|
||||
}
|
||||
63
pkg/s3select/format/select.go
Normal file
63
pkg/s3select/format/select.go
Normal file
@@ -0,0 +1,63 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package format
|
||||
|
||||
import "encoding/xml"
|
||||
|
||||
// Select Interface helper methods, implementing features needed for
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
|
||||
type Select interface {
|
||||
Type() Type
|
||||
Read() (map[string]interface{}, error)
|
||||
Header() []string
|
||||
HasHeader() bool
|
||||
OutputFieldDelimiter() string
|
||||
UpdateBytesProcessed(record map[string]interface{})
|
||||
Expression() string
|
||||
UpdateBytesReturned(int64)
|
||||
CreateStatXML() (string, error)
|
||||
CreateProgressXML() (string, error)
|
||||
ColNameErrs(columnNames []string) error
|
||||
Progress() bool
|
||||
}
|
||||
|
||||
// Progress represents a struct that represents the format for XML of the
|
||||
// progress messages
|
||||
type Progress struct {
|
||||
XMLName xml.Name `xml:"Progress" json:"-"`
|
||||
BytesScanned int64 `xml:"BytesScanned"`
|
||||
BytesProcessed int64 `xml:"BytesProcessed"`
|
||||
BytesReturned int64 `xml:"BytesReturned"`
|
||||
}
|
||||
|
||||
// Stats represents a struct that represents the format for XML of the stat
|
||||
// messages
|
||||
type Stats struct {
|
||||
XMLName xml.Name `xml:"Stats" json:"-"`
|
||||
BytesScanned int64 `xml:"BytesScanned"`
|
||||
BytesProcessed int64 `xml:"BytesProcessed"`
|
||||
BytesReturned int64 `xml:"BytesReturned"`
|
||||
}
|
||||
|
||||
// Type different types of support data format types.
|
||||
type Type string
|
||||
|
||||
// Different data format types.
|
||||
const (
|
||||
JSON Type = "json"
|
||||
CSV Type = "csv"
|
||||
)
|
||||
Reference in New Issue
Block a user