S3 Select API Support for CSV (#6127)

Add support for trivial where clause cases
This commit is contained in:
Arjun Mishra
2018-08-15 03:30:19 -07:00
committed by kannappanr
parent 0e02328c98
commit 7c14cdb60e
59 changed files with 30860 additions and 3 deletions

View File

@@ -0,0 +1,65 @@
/*
Copyright 2017 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package bytes2
// Buffer implements a subset of the write portion of
// bytes.Buffer, but more efficiently. This is meant to
// be used in very high QPS operations, especially for
// WriteByte, and without abstracting it as a Writer.
// Function signatures contain errors for compatibility,
// but they do not return errors.
type Buffer struct {
bytes []byte
}
// NewBuffer is equivalent to bytes.NewBuffer.
func NewBuffer(b []byte) *Buffer {
return &Buffer{bytes: b}
}
// Write is equivalent to bytes.Buffer.Write.
func (buf *Buffer) Write(b []byte) (int, error) {
buf.bytes = append(buf.bytes, b...)
return len(b), nil
}
// WriteString is equivalent to bytes.Buffer.WriteString.
func (buf *Buffer) WriteString(s string) (int, error) {
buf.bytes = append(buf.bytes, s...)
return len(s), nil
}
// WriteByte is equivalent to bytes.Buffer.WriteByte.
func (buf *Buffer) WriteByte(b byte) error {
buf.bytes = append(buf.bytes, b)
return nil
}
// Bytes is equivalent to bytes.Buffer.Bytes.
func (buf *Buffer) Bytes() []byte {
return buf.bytes
}
// Strings is equivalent to bytes.Buffer.Strings.
func (buf *Buffer) String() string {
return string(buf.bytes)
}
// Len is equivalent to bytes.Buffer.Len.
func (buf *Buffer) Len() int {
return len(buf.bytes)
}

View File

@@ -0,0 +1,79 @@
/*
Copyright 2017 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package hack gives you some efficient functionality at the cost of
// breaking some Go rules.
package hack
import (
"reflect"
"unsafe"
)
// StringArena lets you consolidate allocations for a group of strings
// that have similar life length
type StringArena struct {
buf []byte
str string
}
// NewStringArena creates an arena of the specified size.
func NewStringArena(size int) *StringArena {
sa := &StringArena{buf: make([]byte, 0, size)}
pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&sa.buf))
pstring := (*reflect.StringHeader)(unsafe.Pointer(&sa.str))
pstring.Data = pbytes.Data
pstring.Len = pbytes.Cap
return sa
}
// NewString copies a byte slice into the arena and returns it as a string.
// If the arena is full, it returns a traditional go string.
func (sa *StringArena) NewString(b []byte) string {
if len(b) == 0 {
return ""
}
if len(sa.buf)+len(b) > cap(sa.buf) {
return string(b)
}
start := len(sa.buf)
sa.buf = append(sa.buf, b...)
return sa.str[start : start+len(b)]
}
// SpaceLeft returns the amount of space left in the arena.
func (sa *StringArena) SpaceLeft() int {
return cap(sa.buf) - len(sa.buf)
}
// String force casts a []byte to a string.
// USE AT YOUR OWN RISK
func String(b []byte) (s string) {
if len(b) == 0 {
return ""
}
pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b))
pstring := (*reflect.StringHeader)(unsafe.Pointer(&s))
pstring.Data = pbytes.Data
pstring.Len = pbytes.Len
return
}
// StringPointer returns &s[0], which is not allowed in go
func StringPointer(s string) unsafe.Pointer {
pstring := (*reflect.StringHeader)(unsafe.Pointer(&s))
return unsafe.Pointer(pstring.Data)
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,266 @@
/*
Copyright 2017 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sqltypes
import (
"errors"
"fmt"
"reflect"
"strconv"
"github.com/xwb1989/sqlparser/dependency/querypb"
)
// NullBindVariable is a bindvar with NULL value.
var NullBindVariable = &querypb.BindVariable{Type: querypb.Type_NULL_TYPE}
// ValueToProto converts Value to a *querypb.Value.
func ValueToProto(v Value) *querypb.Value {
return &querypb.Value{Type: v.typ, Value: v.val}
}
// ProtoToValue converts a *querypb.Value to a Value.
func ProtoToValue(v *querypb.Value) Value {
return MakeTrusted(v.Type, v.Value)
}
// BuildBindVariables builds a map[string]*querypb.BindVariable from a map[string]interface{}.
func BuildBindVariables(in map[string]interface{}) (map[string]*querypb.BindVariable, error) {
if len(in) == 0 {
return nil, nil
}
out := make(map[string]*querypb.BindVariable, len(in))
for k, v := range in {
bv, err := BuildBindVariable(v)
if err != nil {
return nil, fmt.Errorf("%s: %v", k, err)
}
out[k] = bv
}
return out, nil
}
// Int32BindVariable converts an int32 to a bind var.
func Int32BindVariable(v int32) *querypb.BindVariable {
return ValueBindVariable(NewInt32(v))
}
// Int64BindVariable converts an int64 to a bind var.
func Int64BindVariable(v int64) *querypb.BindVariable {
return ValueBindVariable(NewInt64(v))
}
// Uint64BindVariable converts a uint64 to a bind var.
func Uint64BindVariable(v uint64) *querypb.BindVariable {
return ValueBindVariable(NewUint64(v))
}
// Float64BindVariable converts a float64 to a bind var.
func Float64BindVariable(v float64) *querypb.BindVariable {
return ValueBindVariable(NewFloat64(v))
}
// StringBindVariable converts a string to a bind var.
func StringBindVariable(v string) *querypb.BindVariable {
return ValueBindVariable(NewVarChar(v))
}
// BytesBindVariable converts a []byte to a bind var.
func BytesBindVariable(v []byte) *querypb.BindVariable {
return &querypb.BindVariable{Type: VarBinary, Value: v}
}
// ValueBindVariable converts a Value to a bind var.
func ValueBindVariable(v Value) *querypb.BindVariable {
return &querypb.BindVariable{Type: v.typ, Value: v.val}
}
// BuildBindVariable builds a *querypb.BindVariable from a valid input type.
func BuildBindVariable(v interface{}) (*querypb.BindVariable, error) {
switch v := v.(type) {
case string:
return StringBindVariable(v), nil
case []byte:
return BytesBindVariable(v), nil
case int:
return &querypb.BindVariable{
Type: querypb.Type_INT64,
Value: strconv.AppendInt(nil, int64(v), 10),
}, nil
case int64:
return Int64BindVariable(v), nil
case uint64:
return Uint64BindVariable(v), nil
case float64:
return Float64BindVariable(v), nil
case nil:
return NullBindVariable, nil
case Value:
return ValueBindVariable(v), nil
case *querypb.BindVariable:
return v, nil
case []interface{}:
bv := &querypb.BindVariable{
Type: querypb.Type_TUPLE,
Values: make([]*querypb.Value, len(v)),
}
values := make([]querypb.Value, len(v))
for i, lv := range v {
lbv, err := BuildBindVariable(lv)
if err != nil {
return nil, err
}
values[i].Type = lbv.Type
values[i].Value = lbv.Value
bv.Values[i] = &values[i]
}
return bv, nil
case []string:
bv := &querypb.BindVariable{
Type: querypb.Type_TUPLE,
Values: make([]*querypb.Value, len(v)),
}
values := make([]querypb.Value, len(v))
for i, lv := range v {
values[i].Type = querypb.Type_VARCHAR
values[i].Value = []byte(lv)
bv.Values[i] = &values[i]
}
return bv, nil
case [][]byte:
bv := &querypb.BindVariable{
Type: querypb.Type_TUPLE,
Values: make([]*querypb.Value, len(v)),
}
values := make([]querypb.Value, len(v))
for i, lv := range v {
values[i].Type = querypb.Type_VARBINARY
values[i].Value = lv
bv.Values[i] = &values[i]
}
return bv, nil
case []int:
bv := &querypb.BindVariable{
Type: querypb.Type_TUPLE,
Values: make([]*querypb.Value, len(v)),
}
values := make([]querypb.Value, len(v))
for i, lv := range v {
values[i].Type = querypb.Type_INT64
values[i].Value = strconv.AppendInt(nil, int64(lv), 10)
bv.Values[i] = &values[i]
}
return bv, nil
case []int64:
bv := &querypb.BindVariable{
Type: querypb.Type_TUPLE,
Values: make([]*querypb.Value, len(v)),
}
values := make([]querypb.Value, len(v))
for i, lv := range v {
values[i].Type = querypb.Type_INT64
values[i].Value = strconv.AppendInt(nil, lv, 10)
bv.Values[i] = &values[i]
}
return bv, nil
case []uint64:
bv := &querypb.BindVariable{
Type: querypb.Type_TUPLE,
Values: make([]*querypb.Value, len(v)),
}
values := make([]querypb.Value, len(v))
for i, lv := range v {
values[i].Type = querypb.Type_UINT64
values[i].Value = strconv.AppendUint(nil, lv, 10)
bv.Values[i] = &values[i]
}
return bv, nil
case []float64:
bv := &querypb.BindVariable{
Type: querypb.Type_TUPLE,
Values: make([]*querypb.Value, len(v)),
}
values := make([]querypb.Value, len(v))
for i, lv := range v {
values[i].Type = querypb.Type_FLOAT64
values[i].Value = strconv.AppendFloat(nil, lv, 'g', -1, 64)
bv.Values[i] = &values[i]
}
return bv, nil
}
return nil, fmt.Errorf("type %T not supported as bind var: %v", v, v)
}
// ValidateBindVariables validates a map[string]*querypb.BindVariable.
func ValidateBindVariables(bv map[string]*querypb.BindVariable) error {
for k, v := range bv {
if err := ValidateBindVariable(v); err != nil {
return fmt.Errorf("%s: %v", k, err)
}
}
return nil
}
// ValidateBindVariable returns an error if the bind variable has inconsistent
// fields.
func ValidateBindVariable(bv *querypb.BindVariable) error {
if bv == nil {
return errors.New("bind variable is nil")
}
if bv.Type == querypb.Type_TUPLE {
if len(bv.Values) == 0 {
return errors.New("empty tuple is not allowed")
}
for _, val := range bv.Values {
if val.Type == querypb.Type_TUPLE {
return errors.New("tuple not allowed inside another tuple")
}
if err := ValidateBindVariable(&querypb.BindVariable{Type: val.Type, Value: val.Value}); err != nil {
return err
}
}
return nil
}
// If NewValue succeeds, the value is valid.
_, err := NewValue(bv.Type, bv.Value)
return err
}
// BindVariableToValue converts a bind var into a Value.
func BindVariableToValue(bv *querypb.BindVariable) (Value, error) {
if bv.Type == querypb.Type_TUPLE {
return NULL, errors.New("cannot convert a TUPLE bind var into a value")
}
return MakeTrusted(bv.Type, bv.Value), nil
}
// BindVariablesEqual compares two maps of bind variables.
func BindVariablesEqual(x, y map[string]*querypb.BindVariable) bool {
return reflect.DeepEqual(&querypb.BoundQuery{BindVariables: x}, &querypb.BoundQuery{BindVariables: y})
}
// CopyBindVariables returns a shallow-copy of the given bindVariables map.
func CopyBindVariables(bindVariables map[string]*querypb.BindVariable) map[string]*querypb.BindVariable {
result := make(map[string]*querypb.BindVariable, len(bindVariables))
for key, value := range bindVariables {
result[key] = value
}
return result
}

View File

@@ -0,0 +1,259 @@
/*
Copyright 2017 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sqltypes
import (
"encoding/json"
"errors"
"fmt"
"github.com/xwb1989/sqlparser/dependency/querypb"
)
// PlanValue represents a value or a list of values for
// a column that will later be resolved using bind vars and used
// to perform plan actions like generating the final query or
// deciding on a route.
//
// Plan values are typically used as a slice ([]planValue)
// where each entry is for one column. For situations where
// the required output is a list of rows (like in the case
// of multi-value inserts), the representation is pivoted.
// For example, a statement like this:
// INSERT INTO t VALUES (1, 2), (3, 4)
// will be represented as follows:
// []PlanValue{
// Values: {1, 3},
// Values: {2, 4},
// }
//
// For WHERE clause items that contain a combination of
// equality expressions and IN clauses like this:
// WHERE pk1 = 1 AND pk2 IN (2, 3, 4)
// The plan values will be represented as follows:
// []PlanValue{
// Value: 1,
// Values: {2, 3, 4},
// }
// When converted into rows, columns with single values
// are replicated as the same for all rows:
// [][]Value{
// {1, 2},
// {1, 3},
// {1, 4},
// }
type PlanValue struct {
Key string
Value Value
ListKey string
Values []PlanValue
}
// IsNull returns true if the PlanValue is NULL.
func (pv PlanValue) IsNull() bool {
return pv.Key == "" && pv.Value.IsNull() && pv.ListKey == "" && pv.Values == nil
}
// IsList returns true if the PlanValue is a list.
func (pv PlanValue) IsList() bool {
return pv.ListKey != "" || pv.Values != nil
}
// ResolveValue resolves a PlanValue as a single value based on the supplied bindvars.
func (pv PlanValue) ResolveValue(bindVars map[string]*querypb.BindVariable) (Value, error) {
switch {
case pv.Key != "":
bv, err := pv.lookupValue(bindVars)
if err != nil {
return NULL, err
}
return MakeTrusted(bv.Type, bv.Value), nil
case !pv.Value.IsNull():
return pv.Value, nil
case pv.ListKey != "" || pv.Values != nil:
// This code is unreachable because the parser does not allow
// multi-value constructs where a single value is expected.
return NULL, errors.New("a list was supplied where a single value was expected")
}
return NULL, nil
}
func (pv PlanValue) lookupValue(bindVars map[string]*querypb.BindVariable) (*querypb.BindVariable, error) {
bv, ok := bindVars[pv.Key]
if !ok {
return nil, fmt.Errorf("missing bind var %s", pv.Key)
}
if bv.Type == querypb.Type_TUPLE {
return nil, fmt.Errorf("TUPLE was supplied for single value bind var %s", pv.ListKey)
}
return bv, nil
}
// ResolveList resolves a PlanValue as a list of values based on the supplied bindvars.
func (pv PlanValue) ResolveList(bindVars map[string]*querypb.BindVariable) ([]Value, error) {
switch {
case pv.ListKey != "":
bv, err := pv.lookupList(bindVars)
if err != nil {
return nil, err
}
values := make([]Value, 0, len(bv.Values))
for _, val := range bv.Values {
values = append(values, MakeTrusted(val.Type, val.Value))
}
return values, nil
case pv.Values != nil:
values := make([]Value, 0, len(pv.Values))
for _, val := range pv.Values {
v, err := val.ResolveValue(bindVars)
if err != nil {
return nil, err
}
values = append(values, v)
}
return values, nil
}
// This code is unreachable because the parser does not allow
// single value constructs where multiple values are expected.
return nil, errors.New("a single value was supplied where a list was expected")
}
func (pv PlanValue) lookupList(bindVars map[string]*querypb.BindVariable) (*querypb.BindVariable, error) {
bv, ok := bindVars[pv.ListKey]
if !ok {
return nil, fmt.Errorf("missing bind var %s", pv.ListKey)
}
if bv.Type != querypb.Type_TUPLE {
return nil, fmt.Errorf("single value was supplied for TUPLE bind var %s", pv.ListKey)
}
return bv, nil
}
// MarshalJSON should be used only for testing.
func (pv PlanValue) MarshalJSON() ([]byte, error) {
switch {
case pv.Key != "":
return json.Marshal(":" + pv.Key)
case !pv.Value.IsNull():
if pv.Value.IsIntegral() {
return pv.Value.ToBytes(), nil
}
return json.Marshal(pv.Value.ToString())
case pv.ListKey != "":
return json.Marshal("::" + pv.ListKey)
case pv.Values != nil:
return json.Marshal(pv.Values)
}
return []byte("null"), nil
}
func rowCount(pvs []PlanValue, bindVars map[string]*querypb.BindVariable) (int, error) {
count := -1
setCount := func(l int) error {
switch count {
case -1:
count = l
return nil
case l:
return nil
default:
return errors.New("mismatch in number of column values")
}
}
for _, pv := range pvs {
switch {
case pv.Key != "" || !pv.Value.IsNull():
continue
case pv.Values != nil:
if err := setCount(len(pv.Values)); err != nil {
return 0, err
}
case pv.ListKey != "":
bv, err := pv.lookupList(bindVars)
if err != nil {
return 0, err
}
if err := setCount(len(bv.Values)); err != nil {
return 0, err
}
}
}
if count == -1 {
// If there were no lists inside, it was a single row.
// Note that count can never be 0 because there is enough
// protection at the top level: list bind vars must have
// at least one value (enforced by vtgate), and AST lists
// must have at least one value (enforced by the parser).
// Also lists created internally after vtgate validation
// ensure at least one value.
// TODO(sougou): verify and change API to enforce this.
return 1, nil
}
return count, nil
}
// ResolveRows resolves a []PlanValue as rows based on the supplied bindvars.
func ResolveRows(pvs []PlanValue, bindVars map[string]*querypb.BindVariable) ([][]Value, error) {
count, err := rowCount(pvs, bindVars)
if err != nil {
return nil, err
}
// Allocate the rows.
rows := make([][]Value, count)
for i := range rows {
rows[i] = make([]Value, len(pvs))
}
// Using j becasue we're resolving by columns.
for j, pv := range pvs {
switch {
case pv.Key != "":
bv, err := pv.lookupValue(bindVars)
if err != nil {
return nil, err
}
for i := range rows {
rows[i][j] = MakeTrusted(bv.Type, bv.Value)
}
case !pv.Value.IsNull():
for i := range rows {
rows[i][j] = pv.Value
}
case pv.ListKey != "":
bv, err := pv.lookupList(bindVars)
if err != nil {
// This code is unreachable because pvRowCount already checks this.
return nil, err
}
for i := range rows {
rows[i][j] = MakeTrusted(bv.Values[i].Type, bv.Values[i].Value)
}
case pv.Values != nil:
for i := range rows {
rows[i][j], err = pv.Values[i].ResolveValue(bindVars)
if err != nil {
return nil, err
}
}
// default case is a NULL value, which the row values are already initialized to.
}
}
return rows, nil
}

View File

@@ -0,0 +1,154 @@
/*
Copyright 2017 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sqltypes
import (
querypb "github.com/xwb1989/sqlparser/dependency/querypb"
)
// Functions in this file should only be used for testing.
// This is an experiment to see if test code bloat can be
// reduced and readability improved.
/*
// MakeTestFields builds a []*querypb.Field for testing.
// fields := sqltypes.MakeTestFields(
// "a|b",
// "int64|varchar",
// )
// The field types are as defined in querypb and are case
// insensitive. Column delimiters must be used only to sepearate
// strings and not at the beginning or the end.
func MakeTestFields(names, types string) []*querypb.Field {
n := split(names)
t := split(types)
var fields []*querypb.Field
for i := range n {
fields = append(fields, &querypb.Field{
Name: n[i],
Type: querypb.Type(querypb.Type_value[strings.ToUpper(t[i])]),
})
}
return fields
}
// MakeTestResult builds a *sqltypes.Result object for testing.
// result := sqltypes.MakeTestResult(
// fields,
// " 1|a",
// "10|abcd",
// )
// The field type values are set as the types for the rows built.
// Spaces are trimmed from row values. "null" is treated as NULL.
func MakeTestResult(fields []*querypb.Field, rows ...string) *Result {
result := &Result{
Fields: fields,
}
if len(rows) > 0 {
result.Rows = make([][]Value, len(rows))
}
for i, row := range rows {
result.Rows[i] = make([]Value, len(fields))
for j, col := range split(row) {
if col == "null" {
continue
}
result.Rows[i][j] = MakeTrusted(fields[j].Type, []byte(col))
}
}
result.RowsAffected = uint64(len(result.Rows))
return result
}
// MakeTestStreamingResults builds a list of results for streaming.
// results := sqltypes.MakeStreamingResults(
// fields,
// "1|a",
// "2|b",
// "---",
// "c|c",
// )
// The first result contains only the fields. Subsequent results
// are built using the field types. Every input that starts with a "-"
// is treated as streaming delimiter for one result. A final
// delimiter must not be supplied.
func MakeTestStreamingResults(fields []*querypb.Field, rows ...string) []*Result {
var results []*Result
results = append(results, &Result{Fields: fields})
start := 0
cur := 0
// Add a final streaming delimiter to simplify the loop below.
rows = append(rows, "-")
for cur < len(rows) {
if rows[cur][0] != '-' {
cur++
continue
}
result := MakeTestResult(fields, rows[start:cur]...)
result.Fields = nil
result.RowsAffected = 0
results = append(results, result)
start = cur + 1
cur = start
}
return results
}
*/
// TestBindVariable makes a *querypb.BindVariable from
// an interface{}.It panics on invalid input.
// This function should only be used for testing.
func TestBindVariable(v interface{}) *querypb.BindVariable {
if v == nil {
return NullBindVariable
}
bv, err := BuildBindVariable(v)
if err != nil {
panic(err)
}
return bv
}
// TestValue builds a Value from typ and val.
// This function should only be used for testing.
func TestValue(typ querypb.Type, val string) Value {
return MakeTrusted(typ, []byte(val))
}
/*
// PrintResults prints []*Results into a string.
// This function should only be used for testing.
func PrintResults(results []*Result) string {
b := new(bytes.Buffer)
for i, r := range results {
if i == 0 {
fmt.Fprintf(b, "%v", r)
continue
}
fmt.Fprintf(b, ", %v", r)
}
return b.String()
}
func split(str string) []string {
splits := strings.Split(str, "|")
for i, v := range splits {
splits[i] = strings.TrimSpace(v)
}
return splits
}
*/

View File

@@ -0,0 +1,288 @@
/*
Copyright 2017 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sqltypes
import (
"fmt"
"github.com/xwb1989/sqlparser/dependency/querypb"
)
// This file provides wrappers and support
// functions for querypb.Type.
// These bit flags can be used to query on the
// common properties of types.
const (
flagIsIntegral = int(querypb.Flag_ISINTEGRAL)
flagIsUnsigned = int(querypb.Flag_ISUNSIGNED)
flagIsFloat = int(querypb.Flag_ISFLOAT)
flagIsQuoted = int(querypb.Flag_ISQUOTED)
flagIsText = int(querypb.Flag_ISTEXT)
flagIsBinary = int(querypb.Flag_ISBINARY)
)
// IsIntegral returns true if querypb.Type is an integral
// (signed/unsigned) that can be represented using
// up to 64 binary bits.
// If you have a Value object, use its member function.
func IsIntegral(t querypb.Type) bool {
return int(t)&flagIsIntegral == flagIsIntegral
}
// IsSigned returns true if querypb.Type is a signed integral.
// If you have a Value object, use its member function.
func IsSigned(t querypb.Type) bool {
return int(t)&(flagIsIntegral|flagIsUnsigned) == flagIsIntegral
}
// IsUnsigned returns true if querypb.Type is an unsigned integral.
// Caution: this is not the same as !IsSigned.
// If you have a Value object, use its member function.
func IsUnsigned(t querypb.Type) bool {
return int(t)&(flagIsIntegral|flagIsUnsigned) == flagIsIntegral|flagIsUnsigned
}
// IsFloat returns true is querypb.Type is a floating point.
// If you have a Value object, use its member function.
func IsFloat(t querypb.Type) bool {
return int(t)&flagIsFloat == flagIsFloat
}
// IsQuoted returns true if querypb.Type is a quoted text or binary.
// If you have a Value object, use its member function.
func IsQuoted(t querypb.Type) bool {
return int(t)&flagIsQuoted == flagIsQuoted
}
// IsText returns true if querypb.Type is a text.
// If you have a Value object, use its member function.
func IsText(t querypb.Type) bool {
return int(t)&flagIsText == flagIsText
}
// IsBinary returns true if querypb.Type is a binary.
// If you have a Value object, use its member function.
func IsBinary(t querypb.Type) bool {
return int(t)&flagIsBinary == flagIsBinary
}
// isNumber returns true if the type is any type of number.
func isNumber(t querypb.Type) bool {
return IsIntegral(t) || IsFloat(t) || t == Decimal
}
// Vitess data types. These are idiomatically
// named synonyms for the querypb.Type values.
// Although these constants are interchangeable,
// they should be treated as different from querypb.Type.
// Use the synonyms only to refer to the type in Value.
// For proto variables, use the querypb.Type constants
// instead.
// The following conditions are non-overlapping
// and cover all types: IsSigned(), IsUnsigned(),
// IsFloat(), IsQuoted(), Null, Decimal, Expression.
// Also, IsIntegral() == (IsSigned()||IsUnsigned()).
// TestCategory needs to be updated accordingly if
// you add a new type.
// If IsBinary or IsText is true, then IsQuoted is
// also true. But there are IsQuoted types that are
// neither binary or text.
// querypb.Type_TUPLE is not included in this list
// because it's not a valid Value type.
// TODO(sougou): provide a categorization function
// that returns enums, which will allow for cleaner
// switch statements for those who want to cover types
// by their category.
const (
Null = querypb.Type_NULL_TYPE
Int8 = querypb.Type_INT8
Uint8 = querypb.Type_UINT8
Int16 = querypb.Type_INT16
Uint16 = querypb.Type_UINT16
Int24 = querypb.Type_INT24
Uint24 = querypb.Type_UINT24
Int32 = querypb.Type_INT32
Uint32 = querypb.Type_UINT32
Int64 = querypb.Type_INT64
Uint64 = querypb.Type_UINT64
Float32 = querypb.Type_FLOAT32
Float64 = querypb.Type_FLOAT64
Timestamp = querypb.Type_TIMESTAMP
Date = querypb.Type_DATE
Time = querypb.Type_TIME
Datetime = querypb.Type_DATETIME
Year = querypb.Type_YEAR
Decimal = querypb.Type_DECIMAL
Text = querypb.Type_TEXT
Blob = querypb.Type_BLOB
VarChar = querypb.Type_VARCHAR
VarBinary = querypb.Type_VARBINARY
Char = querypb.Type_CHAR
Binary = querypb.Type_BINARY
Bit = querypb.Type_BIT
Enum = querypb.Type_ENUM
Set = querypb.Type_SET
Geometry = querypb.Type_GEOMETRY
TypeJSON = querypb.Type_JSON
Expression = querypb.Type_EXPRESSION
)
// bit-shift the mysql flags by two byte so we
// can merge them with the mysql or vitess types.
const (
mysqlUnsigned = 32
mysqlBinary = 128
mysqlEnum = 256
mysqlSet = 2048
)
// If you add to this map, make sure you add a test case
// in tabletserver/endtoend.
var mysqlToType = map[int64]querypb.Type{
1: Int8,
2: Int16,
3: Int32,
4: Float32,
5: Float64,
6: Null,
7: Timestamp,
8: Int64,
9: Int24,
10: Date,
11: Time,
12: Datetime,
13: Year,
16: Bit,
245: TypeJSON,
246: Decimal,
249: Text,
250: Text,
251: Text,
252: Text,
253: VarChar,
254: Char,
255: Geometry,
}
// modifyType modifies the vitess type based on the
// mysql flag. The function checks specific flags based
// on the type. This allows us to ignore stray flags
// that MySQL occasionally sets.
func modifyType(typ querypb.Type, flags int64) querypb.Type {
switch typ {
case Int8:
if flags&mysqlUnsigned != 0 {
return Uint8
}
return Int8
case Int16:
if flags&mysqlUnsigned != 0 {
return Uint16
}
return Int16
case Int32:
if flags&mysqlUnsigned != 0 {
return Uint32
}
return Int32
case Int64:
if flags&mysqlUnsigned != 0 {
return Uint64
}
return Int64
case Int24:
if flags&mysqlUnsigned != 0 {
return Uint24
}
return Int24
case Text:
if flags&mysqlBinary != 0 {
return Blob
}
return Text
case VarChar:
if flags&mysqlBinary != 0 {
return VarBinary
}
return VarChar
case Char:
if flags&mysqlBinary != 0 {
return Binary
}
if flags&mysqlEnum != 0 {
return Enum
}
if flags&mysqlSet != 0 {
return Set
}
return Char
}
return typ
}
// MySQLToType computes the vitess type from mysql type and flags.
func MySQLToType(mysqlType, flags int64) (typ querypb.Type, err error) {
result, ok := mysqlToType[mysqlType]
if !ok {
return 0, fmt.Errorf("unsupported type: %d", mysqlType)
}
return modifyType(result, flags), nil
}
// typeToMySQL is the reverse of mysqlToType.
var typeToMySQL = map[querypb.Type]struct {
typ int64
flags int64
}{
Int8: {typ: 1},
Uint8: {typ: 1, flags: mysqlUnsigned},
Int16: {typ: 2},
Uint16: {typ: 2, flags: mysqlUnsigned},
Int32: {typ: 3},
Uint32: {typ: 3, flags: mysqlUnsigned},
Float32: {typ: 4},
Float64: {typ: 5},
Null: {typ: 6, flags: mysqlBinary},
Timestamp: {typ: 7},
Int64: {typ: 8},
Uint64: {typ: 8, flags: mysqlUnsigned},
Int24: {typ: 9},
Uint24: {typ: 9, flags: mysqlUnsigned},
Date: {typ: 10, flags: mysqlBinary},
Time: {typ: 11, flags: mysqlBinary},
Datetime: {typ: 12, flags: mysqlBinary},
Year: {typ: 13, flags: mysqlUnsigned},
Bit: {typ: 16, flags: mysqlUnsigned},
TypeJSON: {typ: 245},
Decimal: {typ: 246},
Text: {typ: 252},
Blob: {typ: 252, flags: mysqlBinary},
VarChar: {typ: 253},
VarBinary: {typ: 253, flags: mysqlBinary},
Char: {typ: 254},
Binary: {typ: 254, flags: mysqlBinary},
Enum: {typ: 254, flags: mysqlEnum},
Set: {typ: 254, flags: mysqlSet},
Geometry: {typ: 255},
}
// TypeToMySQL returns the equivalent mysql type and flag for a vitess type.
func TypeToMySQL(typ querypb.Type) (mysqlType, flags int64) {
val := typeToMySQL[typ]
return val.typ, val.flags
}

View File

@@ -0,0 +1,376 @@
/*
Copyright 2017 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package sqltypes implements interfaces and types that represent SQL values.
package sqltypes
import (
"encoding/base64"
"encoding/json"
"fmt"
"strconv"
"github.com/xwb1989/sqlparser/dependency/bytes2"
"github.com/xwb1989/sqlparser/dependency/hack"
"github.com/xwb1989/sqlparser/dependency/querypb"
)
var (
// NULL represents the NULL value.
NULL = Value{}
// DontEscape tells you if a character should not be escaped.
DontEscape = byte(255)
nullstr = []byte("null")
)
// BinWriter interface is used for encoding values.
// Types like bytes.Buffer conform to this interface.
// We expect the writer objects to be in-memory buffers.
// So, we don't expect the write operations to fail.
type BinWriter interface {
Write([]byte) (int, error)
}
// Value can store any SQL value. If the value represents
// an integral type, the bytes are always stored as a cannonical
// representation that matches how MySQL returns such values.
type Value struct {
typ querypb.Type
val []byte
}
// NewValue builds a Value using typ and val. If the value and typ
// don't match, it returns an error.
func NewValue(typ querypb.Type, val []byte) (v Value, err error) {
switch {
case IsSigned(typ):
if _, err := strconv.ParseInt(string(val), 0, 64); err != nil {
return NULL, err
}
return MakeTrusted(typ, val), nil
case IsUnsigned(typ):
if _, err := strconv.ParseUint(string(val), 0, 64); err != nil {
return NULL, err
}
return MakeTrusted(typ, val), nil
case IsFloat(typ) || typ == Decimal:
if _, err := strconv.ParseFloat(string(val), 64); err != nil {
return NULL, err
}
return MakeTrusted(typ, val), nil
case IsQuoted(typ) || typ == Null:
return MakeTrusted(typ, val), nil
}
// All other types are unsafe or invalid.
return NULL, fmt.Errorf("invalid type specified for MakeValue: %v", typ)
}
// MakeTrusted makes a new Value based on the type.
// This function should only be used if you know the value
// and type conform to the rules. Every place this function is
// called, a comment is needed that explains why it's justified.
// Exceptions: The current package and mysql package do not need
// comments. Other packages can also use the function to create
// VarBinary or VarChar values.
func MakeTrusted(typ querypb.Type, val []byte) Value {
if typ == Null {
return NULL
}
return Value{typ: typ, val: val}
}
// NewInt64 builds an Int64 Value.
func NewInt64(v int64) Value {
return MakeTrusted(Int64, strconv.AppendInt(nil, v, 10))
}
// NewInt32 builds an Int64 Value.
func NewInt32(v int32) Value {
return MakeTrusted(Int32, strconv.AppendInt(nil, int64(v), 10))
}
// NewUint64 builds an Uint64 Value.
func NewUint64(v uint64) Value {
return MakeTrusted(Uint64, strconv.AppendUint(nil, v, 10))
}
// NewFloat64 builds an Float64 Value.
func NewFloat64(v float64) Value {
return MakeTrusted(Float64, strconv.AppendFloat(nil, v, 'g', -1, 64))
}
// NewVarChar builds a VarChar Value.
func NewVarChar(v string) Value {
return MakeTrusted(VarChar, []byte(v))
}
// NewVarBinary builds a VarBinary Value.
// The input is a string because it's the most common use case.
func NewVarBinary(v string) Value {
return MakeTrusted(VarBinary, []byte(v))
}
// NewIntegral builds an integral type from a string representaion.
// The type will be Int64 or Uint64. Int64 will be preferred where possible.
func NewIntegral(val string) (n Value, err error) {
signed, err := strconv.ParseInt(val, 0, 64)
if err == nil {
return MakeTrusted(Int64, strconv.AppendInt(nil, signed, 10)), nil
}
unsigned, err := strconv.ParseUint(val, 0, 64)
if err != nil {
return Value{}, err
}
return MakeTrusted(Uint64, strconv.AppendUint(nil, unsigned, 10)), nil
}
// InterfaceToValue builds a value from a go type.
// Supported types are nil, int64, uint64, float64,
// string and []byte.
// This function is deprecated. Use the type-specific
// functions instead.
func InterfaceToValue(goval interface{}) (Value, error) {
switch goval := goval.(type) {
case nil:
return NULL, nil
case []byte:
return MakeTrusted(VarBinary, goval), nil
case int64:
return NewInt64(goval), nil
case uint64:
return NewUint64(goval), nil
case float64:
return NewFloat64(goval), nil
case string:
return NewVarChar(goval), nil
default:
return NULL, fmt.Errorf("unexpected type %T: %v", goval, goval)
}
}
// Type returns the type of Value.
func (v Value) Type() querypb.Type {
return v.typ
}
// Raw returns the internal represenation of the value. For newer types,
// this may not match MySQL's representation.
func (v Value) Raw() []byte {
return v.val
}
// ToBytes returns the value as MySQL would return it as []byte.
// In contrast, Raw returns the internal representation of the Value, which may not
// match MySQL's representation for newer types.
// If the value is not convertible like in the case of Expression, it returns nil.
func (v Value) ToBytes() []byte {
if v.typ == Expression {
return nil
}
return v.val
}
// Len returns the length.
func (v Value) Len() int {
return len(v.val)
}
// ToString returns the value as MySQL would return it as string.
// If the value is not convertible like in the case of Expression, it returns nil.
func (v Value) ToString() string {
if v.typ == Expression {
return ""
}
return hack.String(v.val)
}
// String returns a printable version of the value.
func (v Value) String() string {
if v.typ == Null {
return "NULL"
}
if v.IsQuoted() {
return fmt.Sprintf("%v(%q)", v.typ, v.val)
}
return fmt.Sprintf("%v(%s)", v.typ, v.val)
}
// EncodeSQL encodes the value into an SQL statement. Can be binary.
func (v Value) EncodeSQL(b BinWriter) {
switch {
case v.typ == Null:
b.Write(nullstr)
case v.IsQuoted():
encodeBytesSQL(v.val, b)
default:
b.Write(v.val)
}
}
// EncodeASCII encodes the value using 7-bit clean ascii bytes.
func (v Value) EncodeASCII(b BinWriter) {
switch {
case v.typ == Null:
b.Write(nullstr)
case v.IsQuoted():
encodeBytesASCII(v.val, b)
default:
b.Write(v.val)
}
}
// IsNull returns true if Value is null.
func (v Value) IsNull() bool {
return v.typ == Null
}
// IsIntegral returns true if Value is an integral.
func (v Value) IsIntegral() bool {
return IsIntegral(v.typ)
}
// IsSigned returns true if Value is a signed integral.
func (v Value) IsSigned() bool {
return IsSigned(v.typ)
}
// IsUnsigned returns true if Value is an unsigned integral.
func (v Value) IsUnsigned() bool {
return IsUnsigned(v.typ)
}
// IsFloat returns true if Value is a float.
func (v Value) IsFloat() bool {
return IsFloat(v.typ)
}
// IsQuoted returns true if Value must be SQL-quoted.
func (v Value) IsQuoted() bool {
return IsQuoted(v.typ)
}
// IsText returns true if Value is a collatable text.
func (v Value) IsText() bool {
return IsText(v.typ)
}
// IsBinary returns true if Value is binary.
func (v Value) IsBinary() bool {
return IsBinary(v.typ)
}
// MarshalJSON should only be used for testing.
// It's not a complete implementation.
func (v Value) MarshalJSON() ([]byte, error) {
switch {
case v.IsQuoted():
return json.Marshal(v.ToString())
case v.typ == Null:
return nullstr, nil
}
return v.val, nil
}
// UnmarshalJSON should only be used for testing.
// It's not a complete implementation.
func (v *Value) UnmarshalJSON(b []byte) error {
if len(b) == 0 {
return fmt.Errorf("error unmarshaling empty bytes")
}
var val interface{}
var err error
switch b[0] {
case '-':
var ival int64
err = json.Unmarshal(b, &ival)
val = ival
case '"':
var bval []byte
err = json.Unmarshal(b, &bval)
val = bval
case 'n': // null
err = json.Unmarshal(b, &val)
default:
var uval uint64
err = json.Unmarshal(b, &uval)
val = uval
}
if err != nil {
return err
}
*v, err = InterfaceToValue(val)
return err
}
func encodeBytesSQL(val []byte, b BinWriter) {
buf := &bytes2.Buffer{}
buf.WriteByte('\'')
for _, ch := range val {
if encodedChar := SQLEncodeMap[ch]; encodedChar == DontEscape {
buf.WriteByte(ch)
} else {
buf.WriteByte('\\')
buf.WriteByte(encodedChar)
}
}
buf.WriteByte('\'')
b.Write(buf.Bytes())
}
func encodeBytesASCII(val []byte, b BinWriter) {
buf := &bytes2.Buffer{}
buf.WriteByte('\'')
encoder := base64.NewEncoder(base64.StdEncoding, buf)
encoder.Write(val)
encoder.Close()
buf.WriteByte('\'')
b.Write(buf.Bytes())
}
// SQLEncodeMap specifies how to escape binary data with '\'.
// Complies to http://dev.mysql.com/doc/refman/5.1/en/string-syntax.html
var SQLEncodeMap [256]byte
// SQLDecodeMap is the reverse of SQLEncodeMap
var SQLDecodeMap [256]byte
var encodeRef = map[byte]byte{
'\x00': '0',
'\'': '\'',
'"': '"',
'\b': 'b',
'\n': 'n',
'\r': 'r',
'\t': 't',
26: 'Z', // ctl-Z
'\\': '\\',
}
func init() {
for i := range SQLEncodeMap {
SQLEncodeMap[i] = DontEscape
SQLDecodeMap[i] = DontEscape
}
for i := range SQLEncodeMap {
if to, ok := encodeRef[byte(i)]; ok {
SQLEncodeMap[byte(i)] = to
SQLDecodeMap[to] = byte(i)
}
}
}