Add support for Timestamp data type in SQL Select (#7185)

This change adds support for casting strings to Timestamp via CAST:
`CAST('2010T' AS TIMESTAMP)`

It also implements the following date-time functions:
  - UTCNOW()
  - DATE_ADD()
  - DATE_DIFF()
  - EXTRACT()

For values passed to these functions, date-types are automatically
inferred.

docs/select/README.md

@@ -103,6 +103,6 @@ For a more detailed SELECT SQL reference, please see [here](https://docs.aws.ama
 - All [operators](https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-operators.html) are supported.
 - All aggregation, conditional, type-conversion and string functions are supported.
 - JSON path expressions such as `FROM S3Object[*].path` are not yet evaluated.
-- Large numbers (more than 64-bit) are not yet supported.
-- Date [functions](https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-date.html) are not yet supported (EXTRACT, DATE_DIFF, etc).
+- Large numbers (outside of the signed 64-bit range) are not yet supported.
+- The Date [functions](https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-date.html) `DATE_ADD`, `DATE_DIFF`, `EXTRACT` and `UTCNOW` along with type conversion using `CAST` to the `TIMESTAMP` data type are currently supported.
 - AWS S3's [reserved keywords](https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-keyword-list.html) list is not yet respected.

pkg/s3select/json/record.go

@@ -60,6 +60,8 @@ func (r *Record) Set(name string, value *sql.Value) (err error) {
 		v = f
 	} else if i, ok := value.ToInt(); ok {
 		v = i
+	} else if t, ok := value.ToTimestamp(); ok {
+		v = sql.FormatSQLTimestamp(t)
 	} else if s, ok := value.ToString(); ok {
 		v = s
 	} else if value.IsNull() {

pkg/s3select/sql/analysis.go

@@ -201,6 +201,16 @@ func (e *FuncExpr) analyze(s *Select) (result qProp) {
 	case sqlFnExtract:
 		return e.Extract.From.analyze(s)
 
+	case sqlFnDateAdd:
+		result.combine(e.DateAdd.Quantity.analyze(s))
+		result.combine(e.DateAdd.Timestamp.analyze(s))
+		return result
+
+	case sqlFnDateDiff:
+		result.combine(e.DateDiff.Timestamp1.analyze(s))
+		result.combine(e.DateDiff.Timestamp2.analyze(s))
+		return result
+
 	// Handle aggregation function calls
 	case aggFnAvg, aggFnMax, aggFnMin, aggFnSum, aggFnCount:
 		// Initialize accumulator
@@ -283,6 +293,11 @@ func (e *FuncExpr) analyze(s *Select) (result qProp) {
 		}
 		return result
 
+	case sqlFnUTCNow:
+		if len(e.SFunc.ArgsList) != 0 {
+			result.err = fmt.Errorf("%s() takes no arguments", string(funcName))
+		}
+		return result
 	}
 
 	// TODO: implement other functions

pkg/s3select/sql/evaluate.go

@@ -305,10 +305,9 @@ func (e *UnaryTerm) evalNode(r Record) (*Value, error) {
 	}
 
 	inferTypeForArithOp(v)
-	if ival, ok := v.ToInt(); ok {
-		return FromInt(-ival), nil
-	} else if fval, ok := v.ToFloat(); ok {
-		return FromFloat(-fval), nil
+	v.negate()
+	if v.isNumeric() {
+		return v, nil
 	}
 	return nil, errArithMismatchedTypes
 }

pkg/s3select/sql/funceval.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
 )
 
 // FuncName - SQL function name.
@@ -52,21 +53,10 @@ const (
 	sqlFnUpper           FuncName = "UPPER"
 )
 
-// Allowed cast types
-const (
-	castBool      = "BOOL"
-	castInt       = "INT"
-	castInteger   = "INTEGER"
-	castString    = "STRING"
-	castFloat     = "FLOAT"
-	castDecimal   = "DECIMAL"
-	castNumeric   = "NUMERIC"
-	castTimestamp = "TIMESTAMP"
-)
-
 var (
 	errUnimplementedCast = errors.New("This cast not yet implemented")
 	errNonStringTrimArg  = errors.New("TRIM() received a non-string argument")
+	errNonTimestampArg   = errors.New("Expected a timestamp argument")
 )
 
 func (e *FuncExpr) getFunctionName() FuncName {
@@ -83,6 +73,10 @@ func (e *FuncExpr) getFunctionName() FuncName {
 		return sqlFnExtract
 	case e.Trim != nil:
 		return sqlFnTrim
+	case e.DateAdd != nil:
+		return sqlFnDateAdd
+	case e.DateDiff != nil:
+		return sqlFnDateDiff
 	default:
 		return ""
 	}
@@ -102,10 +96,17 @@ func (e *FuncExpr) evalSQLFnNode(r Record) (res *Value, err error) {
 		return handleSQLSubstring(r, e.Substring)
 
 	case sqlFnExtract:
-		return nil, errNotImplemented
+		return handleSQLExtract(r, e.Extract)
 
 	case sqlFnTrim:
 		return handleSQLTrim(r, e.Trim)
+
+	case sqlFnDateAdd:
+		return handleDateAdd(r, e.DateAdd)
+
+	case sqlFnDateDiff:
+		return handleDateDiff(r, e.DateDiff)
+
 	}
 
 	// For all simple argument functions, we evaluate the arguments here
@@ -119,30 +120,33 @@ func (e *FuncExpr) evalSQLFnNode(r Record) (res *Value, err error) {
 
 	switch e.getFunctionName() {
 	case sqlFnCoalesce:
-		return coalesce(r, argVals)
+		return coalesce(argVals)
 
 	case sqlFnNullIf:
-		return nullif(r, argVals[0], argVals[1])
+		return nullif(argVals[0], argVals[1])
 
 	case sqlFnCharLength, sqlFnCharacterLength:
-		return charlen(r, argVals[0])
+		return charlen(argVals[0])
 
 	case sqlFnLower:
-		return lowerCase(r, argVals[0])
+		return lowerCase(argVals[0])
 
 	case sqlFnUpper:
-		return upperCase(r, argVals[0])
+		return upperCase(argVals[0])
 
-	case sqlFnDateAdd, sqlFnDateDiff, sqlFnToString, sqlFnToTimestamp, sqlFnUTCNow:
+	case sqlFnUTCNow:
+		return handleUTCNow()
+
+	case sqlFnToString, sqlFnToTimestamp:
 		// TODO: implement
 		fallthrough
 
 	default:
-		return nil, errInvalidASTNode
+		return nil, errNotImplemented
 	}
 }
 
-func coalesce(r Record, args []*Value) (res *Value, err error) {
+func coalesce(args []*Value) (res *Value, err error) {
 	for _, arg := range args {
 		if arg.IsNull() {
 			continue
@@ -152,7 +156,7 @@ func coalesce(r Record, args []*Value) (res *Value, err error) {
 	return FromNull(), nil
 }
 
-func nullif(r Record, v1, v2 *Value) (res *Value, err error) {
+func nullif(v1, v2 *Value) (res *Value, err error) {
 	// Handle Null cases
 	if v1.IsNull() || v2.IsNull() {
 		return v1, nil
@@ -185,7 +189,7 @@ func nullif(r Record, v1, v2 *Value) (res *Value, err error) {
 	return v1, nil
 }
 
-func charlen(r Record, v *Value) (*Value, error) {
+func charlen(v *Value) (*Value, error) {
 	inferTypeAsString(v)
 	s, ok := v.ToString()
 	if !ok {
@@ -195,7 +199,7 @@ func charlen(r Record, v *Value) (*Value, error) {
 	return FromInt(int64(len(s))), nil
 }
 
-func lowerCase(r Record, v *Value) (*Value, error) {
+func lowerCase(v *Value) (*Value, error) {
 	inferTypeAsString(v)
 	s, ok := v.ToString()
 	if !ok {
@@ -205,7 +209,7 @@ func lowerCase(r Record, v *Value) (*Value, error) {
 	return FromString(strings.ToLower(s)), nil
 }
 
-func upperCase(r Record, v *Value) (*Value, error) {
+func upperCase(v *Value) (*Value, error) {
 	inferTypeAsString(v)
 	s, ok := v.ToString()
 	if !ok {
@@ -215,6 +219,58 @@ func upperCase(r Record, v *Value) (*Value, error) {
 	return FromString(strings.ToUpper(s)), nil
 }
 
+func handleDateAdd(r Record, d *DateAddFunc) (*Value, error) {
+	q, err := d.Quantity.evalNode(r)
+	if err != nil {
+		return nil, err
+	}
+	inferTypeForArithOp(q)
+	qty, ok := q.ToFloat()
+	if !ok {
+		return nil, fmt.Errorf("QUANTITY must be a numeric argument to %s()", sqlFnDateAdd)
+	}
+
+	ts, err := d.Timestamp.evalNode(r)
+	if err != nil {
+		return nil, err
+	}
+	inferTypeAsTimestamp(ts)
+	t, ok := ts.ToTimestamp()
+	if !ok {
+		return nil, fmt.Errorf("%s() expects a timestamp argument", sqlFnDateAdd)
+	}
+
+	return dateAdd(strings.ToUpper(d.DatePart), qty, t)
+}
+
+func handleDateDiff(r Record, d *DateDiffFunc) (*Value, error) {
+	tval1, err := d.Timestamp1.evalNode(r)
+	if err != nil {
+		return nil, err
+	}
+	inferTypeAsTimestamp(tval1)
+	ts1, ok := tval1.ToTimestamp()
+	if !ok {
+		return nil, fmt.Errorf("%s() expects two timestamp arguments", sqlFnDateDiff)
+	}
+
+	tval2, err := d.Timestamp2.evalNode(r)
+	if err != nil {
+		return nil, err
+	}
+	inferTypeAsTimestamp(tval2)
+	ts2, ok := tval2.ToTimestamp()
+	if !ok {
+		return nil, fmt.Errorf("%s() expects two timestamp arguments", sqlFnDateDiff)
+	}
+
+	return dateDiff(strings.ToUpper(d.DatePart), ts1, ts2)
+}
+
+func handleUTCNow() (*Value, error) {
+	return FromTimestamp(time.Now().UTC()), nil
+}
+
 func handleSQLSubstring(r Record, e *SubstringFunc) (val *Value, err error) {
 	// Both forms `SUBSTRING('abc' FROM 2 FOR 1)` and
 	// SUBSTRING('abc', 2, 1) are supported.
@@ -301,6 +357,22 @@ func handleSQLTrim(r Record, e *TrimFunc) (res *Value, err error) {
 	return FromString(result), nil
 }
 
+func handleSQLExtract(r Record, e *ExtractFunc) (res *Value, err error) {
+	timeVal, verr := e.From.evalNode(r)
+	if verr != nil {
+		return nil, verr
+	}
+
+	inferTypeAsTimestamp(timeVal)
+
+	t, ok := timeVal.ToTimestamp()
+	if !ok {
+		return nil, errNonTimestampArg
+	}
+
+	return extract(strings.ToUpper(e.Timeword), t)
+}
+
 func errUnsupportedCast(fromType, toType string) error {
 	return fmt.Errorf("Cannot cast from %v to %v", fromType, toType)
 }
@@ -309,12 +381,23 @@ func errCastFailure(msg string) error {
 	return fmt.Errorf("Error casting: %s", msg)
 }
 
+// Allowed cast types
+const (
+	castBool      = "BOOL"
+	castInt       = "INT"
+	castInteger   = "INTEGER"
+	castString    = "STRING"
+	castFloat     = "FLOAT"
+	castDecimal   = "DECIMAL"
+	castNumeric   = "NUMERIC"
+	castTimestamp = "TIMESTAMP"
+)
+
 func (e *Expression) castTo(r Record, castType string) (res *Value, err error) {
 	v, err := e.evalNode(r)
 	if err != nil {
 		return nil, err
 	}
-	fmt.Println("Cast to ", castType)
 
 	switch castType {
 	case castInt, castInteger:
@@ -329,7 +412,15 @@ func (e *Expression) castTo(r Record, castType string) (res *Value, err error) {
 		s, err := stringCast(v)
 		return FromString(s), err
 
-	case castBool, castDecimal, castNumeric, castTimestamp:
+	case castTimestamp:
+		t, err := timestampCast(v)
+		return FromTimestamp(t), err
+
+	case castBool:
+		b, err := boolCast(v)
+		return FromBool(b), err
+
+	case castDecimal, castNumeric:
 		fallthrough
 
 	default:
@@ -431,3 +522,44 @@ func stringCast(v *Value) (string, error) {
 	// This does not happen
 	return "", nil
 }
+
+func timestampCast(v *Value) (t time.Time, _ error) {
+	switch v.vType {
+	case typeString:
+		s, _ := v.ToString()
+		return parseSQLTimestamp(s)
+	case typeBytes:
+		b, _ := v.ToBytes()
+		return parseSQLTimestamp(string(b))
+	case typeTimestamp:
+		t, _ = v.ToTimestamp()
+		return t, nil
+	default:
+		return t, errCastFailure(fmt.Sprintf("cannot cast %v to Timestamp type", v.GetTypeString()))
+	}
+}
+
+func boolCast(v *Value) (b bool, _ error) {
+	sToB := func(s string) (bool, error) {
+		if s == "true" {
+			return true, nil
+		} else if s == "false" {
+			return false, nil
+		} else {
+			return false, errCastFailure("cannot cast to Bool")
+		}
+	}
+	switch v.vType {
+	case typeBool:
+		b, _ := v.ToBool()
+		return b, nil
+	case typeString:
+		s, _ := v.ToString()
+		return sToB(strings.ToLower(s))
+	case typeBytes:
+		b, _ := v.ToBytes()
+		return sToB(strings.ToLower(string(b)))
+	default:
+		return false, errCastFailure("cannot cast %v to Bool")
+	}
+}

pkg/s3select/sql/parser.go

@@ -247,6 +247,8 @@ type FuncExpr struct {
 	Substring *SubstringFunc `parser:"| @@"`
 	Extract   *ExtractFunc   `parser:"| @@"`
 	Trim      *TrimFunc      `parser:"| @@"`
+	DateAdd   *DateAddFunc   `parser:"| @@"`
+	DateDiff  *DateDiffFunc  `parser:"| @@"`
 
 	// Used during evaluation for aggregation funcs
 	aggregate *aggVal
@@ -255,7 +257,7 @@ type FuncExpr struct {
 // SimpleArgFunc represents functions with simple expression
 // arguments.
 type SimpleArgFunc struct {
-	FunctionName string `parser:" @(\"AVG\" | \"MAX\" | \"MIN\" | \"SUM\" |  \"COALESCE\" | \"NULLIF\" | \"DATE_ADD\" | \"DATE_DIFF\" | \"TO_STRING\" | \"TO_TIMESTAMP\" | \"UTCNOW\" | \"CHAR_LENGTH\" | \"CHARACTER_LENGTH\" | \"LOWER\" | \"UPPER\") "`
+	FunctionName string `parser:" @(\"AVG\" | \"MAX\" | \"MIN\" | \"SUM\" |  \"COALESCE\" | \"NULLIF\" | \"TO_STRING\" | \"TO_TIMESTAMP\" | \"UTCNOW\" | \"CHAR_LENGTH\" | \"CHARACTER_LENGTH\" | \"LOWER\" | \"UPPER\") "`
 
 	ArgsList []*Expression `parser:"\"(\" (@@ (\",\" @@)*)?\")\""`
 }
@@ -294,6 +296,20 @@ type TrimFunc struct {
 	TrimFrom  *PrimaryTerm `parser:"             \"FROM\" )? @@ \")\" "`
 }
 
+// DateAddFunc represents the DATE_ADD function
+type DateAddFunc struct {
+	DatePart  string       `parser:" \"DATE_ADD\" \"(\" @( \"YEAR\":Timeword | \"MONTH\":Timeword | \"DAY\":Timeword | \"HOUR\":Timeword | \"MINUTE\":Timeword | \"SECOND\":Timeword ) \",\""`
+	Quantity  *Operand     `parser:" @@ \",\""`
+	Timestamp *PrimaryTerm `parser:" @@ \")\""`
+}
+
+// DateDiffFunc represents the DATE_DIFF function
+type DateDiffFunc struct {
+	DatePart   string       `parser:" \"DATE_DIFF\" \"(\" @( \"YEAR\":Timeword | \"MONTH\":Timeword | \"DAY\":Timeword | \"HOUR\":Timeword | \"MINUTE\":Timeword | \"SECOND\":Timeword ) \",\" "`
+	Timestamp1 *PrimaryTerm `parser:" @@ \",\" "`
+	Timestamp2 *PrimaryTerm `parser:" @@ \")\" "`
+}
+
 // LitValue represents a literal value parsed from the sql
 type LitValue struct {
 	Number  *float64       `parser:"(  @Number"`

pkg/s3select/sql/statement.go

@@ -84,7 +84,6 @@ func ParseSelectStatement(s string) (stmt SelectStatement, err error) {
 	stmt.selectQProp = selectAST.Expression.analyze(&selectAST)
 	err = stmt.selectQProp.err
 	if err != nil {
-		fmt.Println("Got Analysis err:", err)
 		err = errQueryAnalysisFailure(err)
 	}
 	return

pkg/s3select/sql/timestampfuncs.go

@@ -0,0 +1,199 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sql
+
+import (
+	"time"
+)
+
+const (
+	layoutYear       = "2006T"
+	layoutMonth      = "2006-01T"
+	layoutDay        = "2006-01-02T"
+	layoutMinute     = "2006-01-02T15:04Z07:00"
+	layoutSecond     = "2006-01-02T15:04:05Z07:00"
+	layoutNanosecond = "2006-01-02T15:04:05.999999999Z07:00"
+)
+
+var (
+	tformats = []string{
+		layoutYear,
+		layoutMonth,
+		layoutDay,
+		layoutMinute,
+		layoutSecond,
+		layoutNanosecond,
+	}
+	oneNanoSecond = 1
+)
+
+func parseSQLTimestamp(s string) (t time.Time, err error) {
+	for _, f := range tformats {
+		t, err = time.Parse(f, s)
+		if err == nil {
+			break
+		}
+	}
+	return
+}
+
+// FormatSQLTimestamp - returns the a string representation of the
+// timestamp as used in S3 Select
+func FormatSQLTimestamp(t time.Time) string {
+	_, zoneOffset := t.Zone()
+	hasZone := zoneOffset != 0
+	hasFracSecond := t.Nanosecond() != 0
+	hasSecond := t.Second() != 0
+	hasTime := t.Hour() != 0 || t.Minute() != 0
+	hasDay := t.Day() != 1
+	hasMonth := t.Month() != 1
+
+	switch {
+	case hasFracSecond:
+		return t.Format(layoutNanosecond)
+	case hasSecond:
+		return t.Format(layoutSecond)
+	case hasTime || hasZone:
+		return t.Format(layoutMinute)
+	case hasDay:
+		return t.Format(layoutDay)
+	case hasMonth:
+		return t.Format(layoutMonth)
+	default:
+		return t.Format(layoutYear)
+	}
+}
+
+const (
+	timePartYear           = "YEAR"
+	timePartMonth          = "MONTH"
+	timePartDay            = "DAY"
+	timePartHour           = "HOUR"
+	timePartMinute         = "MINUTE"
+	timePartSecond         = "SECOND"
+	timePartTimezoneHour   = "TIMEZONE_HOUR"
+	timePartTimezoneMinute = "TIMEZONE_MINUTE"
+)
+
+func extract(what string, t time.Time) (v *Value, err error) {
+	switch what {
+	case timePartYear:
+		return FromInt(int64(t.Year())), nil
+	case timePartMonth:
+		return FromInt(int64(t.Month())), nil
+	case timePartDay:
+		return FromInt(int64(t.Day())), nil
+	case timePartHour:
+		return FromInt(int64(t.Hour())), nil
+	case timePartMinute:
+		return FromInt(int64(t.Minute())), nil
+	case timePartSecond:
+		return FromInt(int64(t.Second())), nil
+	case timePartTimezoneHour:
+		_, zoneOffset := t.Zone()
+		return FromInt(int64(zoneOffset / 3600)), nil
+	case timePartTimezoneMinute:
+		_, zoneOffset := t.Zone()
+		return FromInt(int64((zoneOffset % 3600) / 60)), nil
+	default:
+		// This does not happen
+		return nil, errNotImplemented
+	}
+}
+
+func dateAdd(timePart string, qty float64, t time.Time) (*Value, error) {
+	var duration time.Duration
+	switch timePart {
+	case timePartYear:
+		return FromTimestamp(t.AddDate(int(qty), 0, 0)), nil
+	case timePartMonth:
+		return FromTimestamp(t.AddDate(0, int(qty), 0)), nil
+	case timePartDay:
+		return FromTimestamp(t.AddDate(0, 0, int(qty))), nil
+	case timePartHour:
+		duration = time.Duration(qty) * time.Hour
+	case timePartMinute:
+		duration = time.Duration(qty) * time.Minute
+	case timePartSecond:
+		duration = time.Duration(qty) * time.Second
+	default:
+		return nil, errNotImplemented
+	}
+	return FromTimestamp(t.Add(duration)), nil
+}
+
+const (
+	dayInNanoseconds = time.Hour * 24
+)
+
+// dateDiff computes the difference between two times in terms of the
+// `timePart` which can be years, months, days, hours, minutes or
+// seconds. For difference in years, months or days, the time part,
+// including timezone is ignored.
+func dateDiff(timePart string, ts1, ts2 time.Time) (*Value, error) {
+	if ts2.Before(ts1) {
+		v, err := dateDiff(timePart, ts2, ts1)
+		v.negate()
+		return v, err
+	}
+
+	duration := ts2.Sub(ts1)
+	y1, m1, d1 := ts1.Date()
+	y2, m2, d2 := ts2.Date()
+	dy, dm := int64(y2-y1), int64(m2-m1)
+
+	switch timePart {
+	case timePartYear:
+		if m2 > m1 || (m2 == m1 && d2 >= d1) {
+			return FromInt(dy), nil
+		}
+		return FromInt(dy - 1), nil
+	case timePartMonth:
+		months := 12 * dy
+		if m2 >= m1 {
+			months += dm
+		} else {
+			months += 12 + dm
+		}
+		if d2 < d1 {
+			months--
+		}
+		return FromInt(months), nil
+	case timePartDay:
+		// To compute the number of days between two times
+		// using the time package, zero out the time portions
+		// of the timestamps, compute the difference duration
+		// and then divide by the length of a day.
+		d1 := time.Date(y1, m1, d1, 0, 0, 0, 0, time.UTC)
+		d2 := time.Date(y2, m2, d2, 0, 0, 0, 0, time.UTC)
+		diff := d2.Sub(d1)
+		days := diff / dayInNanoseconds
+		return FromInt(int64(days)), nil
+	case timePartHour:
+		hours := duration / time.Hour
+		return FromInt(int64(hours)), nil
+	case timePartMinute:
+		minutes := duration / time.Minute
+		return FromInt(int64(minutes)), nil
+	case timePartSecond:
+		seconds := duration / time.Second
+		return FromInt(int64(seconds)), nil
+	default:
+
+	}
+	return nil, errNotImplemented
+}

pkg/s3select/sql/timestampfuncs_test.go

@@ -0,0 +1,60 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sql
+
+import (
+	"testing"
+	"time"
+)
+
+func TestParseAndDisplaySQLTimestamp(t *testing.T) {
+	beijing := time.FixedZone("", int((8 * time.Hour).Seconds()))
+	fakeLosAngeles := time.FixedZone("", -int((8 * time.Hour).Seconds()))
+	cases := []struct {
+		s string
+		t time.Time
+	}{
+		{"2010T", time.Date(2010, 1, 1, 0, 0, 0, 0, time.UTC)},
+		{"2010-02T", time.Date(2010, 2, 1, 0, 0, 0, 0, time.UTC)},
+		{"2010-02-03T", time.Date(2010, 2, 3, 0, 0, 0, 0, time.UTC)},
+		{"2010-02-03T04:11Z", time.Date(2010, 2, 3, 4, 11, 0, 0, time.UTC)},
+		{"2010-02-03T04:11:30Z", time.Date(2010, 2, 3, 4, 11, 30, 0, time.UTC)},
+		{"2010-02-03T04:11:30.23Z", time.Date(2010, 2, 3, 4, 11, 30, 230000000, time.UTC)},
+		{"2010-02-03T04:11+08:00", time.Date(2010, 2, 3, 4, 11, 0, 0, beijing)},
+		{"2010-02-03T04:11:30+08:00", time.Date(2010, 2, 3, 4, 11, 30, 0, beijing)},
+		{"2010-02-03T04:11:30.23+08:00", time.Date(2010, 2, 3, 4, 11, 30, 230000000, beijing)},
+		{"2010-02-03T04:11:30-08:00", time.Date(2010, 2, 3, 4, 11, 30, 0, fakeLosAngeles)},
+		{"2010-02-03T04:11:30.23-08:00", time.Date(2010, 2, 3, 4, 11, 30, 230000000, fakeLosAngeles)},
+	}
+	for i, tc := range cases {
+		tval, err := parseSQLTimestamp(tc.s)
+		if err != nil {
+			t.Errorf("Case %d: Unexpected error: %v", i+1, err)
+			continue
+		}
+		if !tval.Equal(tc.t) {
+			t.Errorf("Case %d: Expected %v got %v", i+1, tc.t, tval)
+			continue
+		}
+
+		tstr := FormatSQLTimestamp(tc.t)
+		if tstr != tc.s {
+			t.Errorf("Case %d: Expected %s got %s", i+1, tc.s, tstr)
+			continue
+		}
+	}
+}

pkg/s3select/sql/value.go

@@ -22,6 +22,7 @@ import (
 	"math"
 	"strconv"
 	"strings"
+	"time"
 )
 
 var (
@@ -48,6 +49,9 @@ const (
 	// 64-bit floating point
 	typeFloat
 
+	// timestamp type
+	typeTimestamp
+
 	// This type refers to untyped values, e.g. as read from CSV
 	typeBytes
 )
@@ -77,6 +81,8 @@ func (v *Value) GetTypeString() string {
 		return "INT"
 	case typeFloat:
 		return "FLOAT"
+	case typeTimestamp:
+		return "TIMESTAMP"
 	case typeBytes:
 		return "BYTES"
 	}
@@ -90,6 +96,8 @@ func (v *Value) Repr() string {
 		return ":NULL"
 	case typeBool, typeInt, typeFloat:
 		return fmt.Sprintf("%v:%s", v.value, v.GetTypeString())
+	case typeTimestamp:
+		return fmt.Sprintf("%s:TIMESTAMP", v.value.(*time.Time))
 	case typeString:
 		return fmt.Sprintf("\"%s\":%s", v.value.(string), v.GetTypeString())
 	case typeBytes:
@@ -119,6 +127,11 @@ func FromBool(b bool) *Value {
 	return &Value{value: b, vType: typeBool}
 }
 
+// FromTimestamp creates a Value from a timestamp
+func FromTimestamp(t time.Time) *Value {
+	return &Value{value: t, vType: typeTimestamp}
+}
+
 // FromNull creates a Value with Null value
 func FromNull() *Value {
 	return &Value{vType: typeNull}
@@ -173,6 +186,15 @@ func (v *Value) ToBool() (val bool, ok bool) {
 	return false, false
 }
 
+// ToTimestamp returns the timestamp value if present.
+func (v *Value) ToTimestamp() (t time.Time, ok bool) {
+	switch v.vType {
+	case typeTimestamp:
+		return v.value.(time.Time), true
+	}
+	return t, false
+}
+
 // ToBytes converts Value to byte-slice.
 func (v *Value) ToBytes() ([]byte, bool) {
 	switch v.vType {
@@ -213,6 +235,11 @@ func (v *Value) setBool(b bool) {
 	v.value = b
 }
 
+func (v *Value) setTimestamp(t time.Time) {
+	v.vType = typeTimestamp
+	v.value = t
+}
+
 // CSVString - convert to string for CSV serialization
 func (v *Value) CSVString() string {
 	switch v.vType {
@@ -226,6 +253,8 @@ func (v *Value) CSVString() string {
 		return fmt.Sprintf("%v", v.value.(int64))
 	case typeFloat:
 		return fmt.Sprintf("%v", v.value.(float64))
+	case typeTimestamp:
+		return FormatSQLTimestamp(v.value.(time.Time))
 	case typeBytes:
 		return fmt.Sprintf("%v", string(v.value.([]byte)))
 	default:
@@ -242,6 +271,16 @@ func floatToValue(f float64) *Value {
 	return FromFloat(f)
 }
 
+// negate negates a numeric value
+func (v *Value) negate() {
+	switch v.vType {
+	case typeFloat:
+		v.value = -(v.value.(float64))
+	case typeInt:
+		v.value = -(v.value.(int64))
+	}
+}
+
 // Value comparison functions: we do not expose them outside the
 // module. Logical operators "<", ">", ">=", "<=" work on strings and
 // numbers. Equality operators "=", "!=" work on strings,
@@ -571,6 +610,24 @@ func (v *Value) minmax(a *Value, isMax, isFirstRow bool) error {
 	return nil
 }
 
+func inferTypeAsTimestamp(v *Value) {
+	if s, ok := v.ToString(); ok {
+		t, err := parseSQLTimestamp(s)
+		if err != nil {
+			return
+		}
+		v.setTimestamp(t)
+	} else if b, ok := v.ToBytes(); ok {
+		s := string(b)
+		t, err := parseSQLTimestamp(s)
+		if err != nil {
+			return
+		}
+		v.setTimestamp(t)
+	}
+	return
+}
+
 // inferTypeAsString is used to convert untyped values to string - it
 // is called when the caller requires a string context to proceed.
 func inferTypeAsString(v *Value) {