feat: time to bring back http2.0 support (#10230)

Bonus move our CI/CD to go1.14

.github/workflows/go.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.13.x]
+        go-version: [1.14.x]
         os: [ubuntu-latest, windows-latest]
     steps:
       - uses: actions/checkout@v2

cmd/http-tracer.go

@@ -129,6 +129,7 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ
 
 	rq := trace.RequestInfo{
 		Time:     time.Now().UTC(),
+		Proto:    r.Proto,
 		Method:   r.Method,
 		Path:     r.URL.Path,
 		RawQuery: r.URL.RawQuery,

cmd/http/server.go

@@ -191,14 +191,7 @@ func NewServer(addrs []string, handler http.Handler, getCert certs.GetCertificat
 			// TLS hardening
 			PreferServerCipherSuites: true,
 			MinVersion:               tls.VersionTLS12,
-			// Do not edit the next line, protos priority is kept
+			NextProtos:               []string{"h2", "http/1.1"},
-			// on purpose in this manner for HTTP 2.0, we would
-			// still like HTTP 2.0 clients to negotiate connection
-			// to server if needed but by default HTTP 1.1 is
-			// expected. We need to change this in future
-			// when we wish to go back to HTTP 2.0 as default
-			// priority for HTTP protocol negotiation.
-			NextProtos: []string{"http/1.1", "h2"},
 		}
 		tlsConfig.GetCertificate = getCert
 	}

cmd/lock-rest-client.go

@@ -149,7 +149,6 @@ func newlockRESTClient(endpoint Endpoint) *lockRESTClient {
 		tlsConfig = &tls.Config{
 			ServerName: endpoint.Hostname(),
 			RootCAs:    globalRootCAs,
-			NextProtos: []string{"http/1.1"}, // Force http1.1
 		}
 	}
 

cmd/prepare-storage.go

@@ -185,7 +185,6 @@ func IsServerResolvable(endpoint Endpoint) error {
 		tlsConfig = &tls.Config{
 			ServerName: endpoint.Hostname(),
 			RootCAs:    globalRootCAs,
-			NextProtos: []string{"http/1.1"}, // Force http1.1
 		}
 	}
 

cmd/signals.go

@@ -18,6 +18,8 @@ package cmd
 
 import (
 	"context"
+	"errors"
+	"net/http"
 	"os"
 	"strings"
 
@@ -73,7 +75,7 @@ func handleSignals() {
 			if objAPI := newObjectLayerWithoutSafeModeFn(); objAPI != nil {
 				objAPI.Shutdown(context.Background())
 			}
-			if err != nil {
+			if err != nil && !errors.Is(err, http.ErrServerClosed) {
 				logger.Fatal(err, "Unable to start MinIO server")
 			}
 			exit(true)

cmd/storage-rest-client.go

@@ -663,7 +663,6 @@ func newStorageRESTClient(endpoint Endpoint) *storageRESTClient {
 		tlsConfig = &tls.Config{
 			ServerName: endpoint.Hostname(),
 			RootCAs:    globalRootCAs,
-			NextProtos: []string{"http/1.1"}, // Force http1.1
 		}
 	}
 

cmd/utils.go

@@ -39,13 +39,13 @@ import (
 	"sync"
 	"time"
 
+	humanize "github.com/dustin/go-humanize"
+	"github.com/gorilla/mux"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/madmin"
-
+	"golang.org/x/net/http2"
-	humanize "github.com/dustin/go-humanize"
-	"github.com/gorilla/mux"
 )
 
 const (
@@ -467,6 +467,11 @@ func newInternodeHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration)
 		// in raw stream.
 		DisableCompression: true,
 	}
+
+	if tlsConfig != nil {
+		http2.ConfigureTransport(tr)
+	}
+
 	return func() *http.Transport {
 		return tr
 	}
@@ -490,6 +495,11 @@ func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration) fu
 		// in raw stream.
 		DisableCompression: true,
 	}
+
+	if tlsConfig != nil {
+		http2.ConfigureTransport(tr)
+	}
+
 	return func() *http.Transport {
 		return tr
 	}

go.mod

@@ -50,7 +50,7 @@ require (
 	github.com/minio/minio-go/v7 v7.0.3
 	github.com/minio/selfupdate v0.3.1
 	github.com/minio/sha256-simd v0.1.1
-	github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37
+	github.com/minio/simdjson-go v0.1.5
 	github.com/minio/sio v0.2.0
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mmcloughlin/avo v0.0.0-20200523190732-4439b6b2c061 // indirect

go.sum

@@ -297,8 +297,8 @@ github.com/minio/selfupdate v0.3.1 h1:BWEFSNnrZVMUWXbXIgLDNDjbejkmpAmZvy/nCz1HlE
 github.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM=
 github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
-github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37 h1:pDeao6M5AEd8hwTtGmE0pVKomlL56JFRa5SiXDZAuJE=
+github.com/minio/simdjson-go v0.1.5 h1:6T5mHh7r3kUvgwhmFWQAjoPV5Yt5oD/VPjAI9ViH1kM=
-github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37/go.mod h1:oKURrZZEBtqObgJrSjN1Ln2n9MJj2icuBTkeJzZnvSI=
+github.com/minio/simdjson-go v0.1.5/go.mod h1:oKURrZZEBtqObgJrSjN1Ln2n9MJj2icuBTkeJzZnvSI=
 github.com/minio/sio v0.2.0 h1:NCRCFLx0r5pRbXf65LVNjxbCGZgNQvNFQkgX3XF4BoA=
 github.com/minio/sio v0.2.0/go.mod h1:nKM5GIWSrqbOZp0uhyj6M1iA0X6xQzSGtYSaTKSCut0=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=

mint/run/core/s3select/csv.py

@@ -15,23 +15,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
 import io
+import os
 
 from minio import Minio
-from minio.select.options import (SelectObjectOptions, CSVInput,
+from minio.select.options import (CSVInput, CSVOutput, InputSerialization,
-                                  RequestProgress, InputSerialization,
+                                  JsonOutput, OutputSerialization,
-                                  OutputSerialization, CSVOutput, JsonOutput)
+                                  RequestProgress, SelectObjectOptions)
 
 from utils import *
 
+
 def test_sql_api(test_name, client, bucket_name, input_data, sql_opts, expected_output):
     """ Test if the passed SQL request has the output equal to the passed execpted one"""
     object_name = generate_object_name()
     got_output = b''
     try:
         bytes_content = io.BytesIO(input_data)
-        client.put_object(bucket_name, object_name, io.BytesIO(input_data), len(input_data))
+        client.put_object(bucket_name, object_name,
+                          io.BytesIO(input_data), len(input_data))
         data = client.select_object_content(bucket_name, object_name, sql_opts)
         # Get the records
         records = io.BytesIO()
@@ -40,12 +42,15 @@ def test_sql_api(test_name, client, bucket_name, input_data, sql_opts, expected_
             got_output = records.getvalue()
     except Exception as select_err:
         if not isinstance(expected_output, Exception):
-            raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err))
+            raise ValueError(
+                'Test {} unexpectedly failed with: {}'.format(test_name, select_err))
     else:
         if isinstance(expected_output, Exception):
-            raise ValueError('Test {}: expected an exception, got {}'.format(test_name, got_output))
+            raise ValueError(
+                'Test {}: expected an exception, got {}'.format(test_name, got_output))
         if got_output != expected_output:
-            raise ValueError('Test {}: data mismatch. Expected : {}, Received {}'.format(test_name, expected_output, got_output))
+            raise ValueError('Test {}: data mismatch. Expected : {}, Received {}'.format(
+                test_name, expected_output, got_output))
     finally:
         client.remove_object(bucket_name, object_name)
 
@@ -58,14 +63,20 @@ def test_csv_input_custom_quote_char(client, log_output):
         # Invalid quote character, should fail
         ('""', '"', b'col1,col2,col3\n', Exception()),
         # UTF-8 quote character
-            ('ع', '"', 'عcol1ع,عcol2ع,عcol3ع\n'.encode(), b'{"_1":"col1","_2":"col2","_3":"col3"}\n'),
+        ('ع', '"', 'عcol1ع,عcol2ع,عcol3ع\n'.encode(),
+         b'{"_1":"col1","_2":"col2","_3":"col3"}\n'),
         # Only one field is quoted
-            ('"', '"', b'"col1",col2,col3\n', b'{"_1":"col1","_2":"col2","_3":"col3"}\n'),
+        ('"', '"', b'"col1",col2,col3\n',
+         b'{"_1":"col1","_2":"col2","_3":"col3"}\n'),
         ('"', '"', b'"col1,col2,col3"\n', b'{"_1":"col1,col2,col3"}\n'),
-            ('\'', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
+        ('\'', '"', b'"col1",col2,col3\n',
-            ('', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
+         b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
-            ('', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
+        ('', '"', b'"col1",col2,col3\n',
-            ('', '"', b'"col1","col2","col3"\n', b'{"_1":"\\"col1\\"","_2":"\\"col2\\"","_3":"\\"col3\\""}\n'),
+         b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
+        ('', '"', b'"col1",col2,col3\n',
+         b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
+        ('', '"', b'"col1","col2","col3"\n',
+         b'{"_1":"\\"col1\\"","_2":"\\"col2\\"","_3":"\\"col3\\""}\n'),
         ('"', '"', b'""""""\n', b'{"_1":"\\"\\""}\n'),
         ('"', '"', b'A",B\n', b'{"_1":"A\\"","_2":"B"}\n'),
         ('"', '"', b'A"",B\n', b'{"_1":"A\\"\\"","_2":"B"}\n'),
@@ -105,13 +116,15 @@ def test_csv_input_custom_quote_char(client, log_output):
                 )
             )
 
-            test_sql_api(f'test_{idx}', client, bucket_name, data, sql_opts, expected_output)
+            test_sql_api(f'test_{idx}', client, bucket_name,
+                         data, sql_opts, expected_output)
     finally:
         client.remove_bucket(bucket_name)
 
     # Test passes
     print(log_output.json_report())
 
+
 def test_csv_output_custom_quote_char(client, log_output):
     # Get a unique bucket_name and object_name
     log_output.args['bucket_name'] = bucket_name = generate_bucket_name()
@@ -162,11 +175,10 @@ def test_csv_output_custom_quote_char(client, log_output):
                 )
             )
 
-            test_sql_api(f'test_{idx}', client, bucket_name, input_data, sql_opts, expected_output)
+            test_sql_api(f'test_{idx}', client, bucket_name,
+                         input_data, sql_opts, expected_output)
     finally:
         client.remove_bucket(bucket_name)
 
     # Test passes
     print(log_output.json_report())
-
-

mint/run/core/s3select/sql_ops.py

@@ -18,14 +18,15 @@
 import io
 from datetime import datetime
 
-from minio import Minio
+from minio.select.options import (CSVInput, CSVOutput, InputSerialization,
-from minio.select.options import (SelectObjectOptions, CSVInput, JSONInput,
+                                  JSONInput, OutputSerialization,
-                                  RequestProgress, InputSerialization,
+                                  RequestProgress, SelectObjectOptions)
-                                  OutputSerialization, CSVOutput, JsonOutput)
 
-from utils import *
+from utils import generate_bucket_name, generate_object_name
 
-def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql_output, tests, log_output):
+
+def test_sql_expressions_custom_input_output(client, input_bytes, sql_input,
+                                             sql_output, tests, log_output):
     bucket_name = generate_bucket_name()
     object_name = generate_object_name()
 
@@ -51,7 +52,8 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql
                     )
                 )
 
-                data = client.select_object_content(bucket_name, object_name, options)
+                data = client.select_object_content(
+                    bucket_name, object_name, options)
 
                 # Get the records
                 records = io.BytesIO()
@@ -62,13 +64,15 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql
                 if got_output != expected_output:
                     if type(expected_output) == datetime:
                         # Attempt to parse the date which will throw an exception for any issue
-                        datetime.strptime(got_output.decode("utf-8").strip(), '%Y-%m-%dT%H:%M:%S.%f%z')
+                        datetime.strptime(got_output.decode(
+                            "utf-8").strip(), '%Y-%m-%dT%H:%M:%S.%f%z')
                     else:
-                        raise ValueError('Test {}: data mismatch. Expected : {}. Received: {}.'.format(idx+1, expected_output, got_output))
+                        raise ValueError('Test {}: data mismatch. Expected : {}. Received: {}.'.format(
+                            idx+1, expected_output, got_output))
 
                 log_output.args['total_success'] += 1
             except Exception as err:
-                continue ## TODO, raise instead
+                continue  # TODO, raise instead
                 # raise Exception(err)
     finally:
         client.remove_object(bucket_name, object_name)
@@ -100,24 +104,31 @@ def test_sql_operators(client, log_output):
     tests = [
         # Logical operators
         ("AND", "select * from S3Object s where s.id = 1 AND s.name = 'John'", b'1,John,3\n'),
-       ("NOT", "select * from S3Object s where NOT s.id = 1", b'2,Elliot,4\n3,Yves,5\n4,,0\n'),
+        ("NOT", "select * from S3Object s where NOT s.id = 1",
-       ("OR", "select * from S3Object s where s.id = 1 OR s.id = 3", b'1,John,3\n3,Yves,5\n'),
+            b'2,Elliot,4\n3,Yves,5\n4,,0\n'),
+        ("OR", "select * from S3Object s where s.id = 1 OR s.id = 3",
+            b'1,John,3\n3,Yves,5\n'),
         # Comparison Operators
         ("<", "select * from S3Object s where s.age < 4", b'1,John,3\n4,,0\n'),
         (">", "select * from S3Object s where s.age > 4", b'3,Yves,5\n'),
-       ("<=", "select * from S3Object s where s.age <= 4", b'1,John,3\n2,Elliot,4\n4,,0\n'),
+        ("<=", "select * from S3Object s where s.age <= 4",
+            b'1,John,3\n2,Elliot,4\n4,,0\n'),
         (">=", "select * from S3Object s where s.age >= 4", b'2,Elliot,4\n3,Yves,5\n'),
         ("=", "select * from S3Object s where s.age = 4", b'2,Elliot,4\n'),
-       ("<>", "select * from S3Object s where s.age <> 4", b'1,John,3\n3,Yves,5\n4,,0\n'),
+        ("<>", "select * from S3Object s where s.age <> 4",
-       ("!=", "select * from S3Object s where s.age != 4", b'1,John,3\n3,Yves,5\n4,,0\n'),
+            b'1,John,3\n3,Yves,5\n4,,0\n'),
-       ("BETWEEN", "select * from S3Object s where s.age BETWEEN 4 AND 5", b'2,Elliot,4\n3,Yves,5\n'),
+        ("!=", "select * from S3Object s where s.age != 4",
+            b'1,John,3\n3,Yves,5\n4,,0\n'),
+        ("BETWEEN", "select * from S3Object s where s.age BETWEEN 4 AND 5",
+            b'2,Elliot,4\n3,Yves,5\n'),
         ("IN", "select * from S3Object s where s.age IN (3,5)", b'1,John,3\n3,Yves,5\n'),
         # Pattern Matching Operators
         ("LIKE_", "select * from S3Object s where s.name LIKE '_ves'", b'3,Yves,5\n'),
         ("LIKE%", "select * from S3Object s where s.name LIKE 'Ell%t'", b'2,Elliot,4\n'),
         # Unitary Operators
         ("NULL", "select * from S3Object s where s.name IS NULL", b'4,,0\n'),
-       ("NOT_NULL", "select * from S3Object s where s.age IS NOT NULL", b'1,John,3\n2,Elliot,4\n3,Yves,5\n4,,0\n'),
+        ("NOT_NULL", "select * from S3Object s where s.age IS NOT NULL",
+            b'1,John,3\n2,Elliot,4\n3,Yves,5\n4,,0\n'),
         # Math Operators
         ("+", "select * from S3Object s where s.age = 1+3 ", b'2,Elliot,4\n'),
         ("-", "select * from S3Object s where s.age = 5-1 ", b'2,Elliot,4\n'),
@@ -168,7 +179,6 @@ def test_sql_operators_precedence(client, log_output):
     print(log_output.json_report())
 
 
-
 def test_sql_functions_agg_cond_conv(client, log_output):
 
     json_testfile = """{"id": 1, "name": "John", "age": 3}
@@ -187,8 +197,9 @@ def test_sql_functions_agg_cond_conv(client, log_output):
         # Conditional functions
         ("COALESCE", "SELECT COALESCE(s.age, 99) FROM S3Object s", b'3\n4\n5\n99\n0\n'),
         ("NULLIF", "SELECT NULLIF(s.age, 0) FROM S3Object s", b'3\n4\n5\n\n\n'),
-       ## Conversion functions
+        # Conversion functions
-       ("CAST", "SELECT CAST(s.age AS FLOAT) FROM S3Object s", b'3.0\n4.0\n5.0\n\n0.0\n'),
+        ("CAST", "SELECT CAST(s.age AS FLOAT) FROM S3Object s",
+            b'3.0\n4.0\n5.0\n\n0.0\n'),
 
     ]
 
@@ -211,12 +222,18 @@ def test_sql_functions_date(client, log_output):
 
     tests = [
         # DATE_ADD
-       ("DATE_ADD_1", "select DATE_ADD(year, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2022-01-02T03:04:05.006+07:30\n'),
+        ("DATE_ADD_1", "select DATE_ADD(year, 5, TO_TIMESTAMP(s.datez)) from S3Object as s",
-       ("DATE_ADD_2", "select DATE_ADD(month, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-02-02T03:04:05.006+07:30\n'),
+         b'2022-01-02T03:04:05.006+07:30\n'),
-       ("DATE_ADD_3", "select DATE_ADD(day, -1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-01T03:04:05.006+07:30\n'),
+        ("DATE_ADD_2", "select DATE_ADD(month, 1, TO_TIMESTAMP(s.datez)) from S3Object as s",
-       ("DATE_ADD_4", "select DATE_ADD(hour, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T04:04:05.006+07:30\n'),
+            b'2017-02-02T03:04:05.006+07:30\n'),
-       ("DATE_ADD_5", "select DATE_ADD(minute, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T03:09:05.006+07:30\n'),
+        ("DATE_ADD_3", "select DATE_ADD(day, -1, TO_TIMESTAMP(s.datez)) from S3Object as s",
-       ("DATE_ADD_6", "select DATE_ADD(second, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T03:04:10.006+07:30\n'),
+            b'2017-01-01T03:04:05.006+07:30\n'),
+        ("DATE_ADD_4", "select DATE_ADD(hour, 1, TO_TIMESTAMP(s.datez)) from S3Object as s",
+            b'2017-01-02T04:04:05.006+07:30\n'),
+        ("DATE_ADD_5", "select DATE_ADD(minute, 5, TO_TIMESTAMP(s.datez)) from S3Object as s",
+            b'2017-01-02T03:09:05.006+07:30\n'),
+        ("DATE_ADD_6", "select DATE_ADD(second, 5, TO_TIMESTAMP(s.datez)) from S3Object as s",
+            b'2017-01-02T03:04:10.006+07:30\n'),
         # DATE_DIFF
         ("DATE_DIFF_1", "select DATE_DIFF(year, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011-01-01T')) from S3Object as s", b'-6\n'),
         ("DATE_DIFF_2", "select DATE_DIFF(month, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011T')) from S3Object as s", b'-72\n'),
@@ -229,16 +246,23 @@ def test_sql_functions_date(client, log_output):
         ("EXTRACT_5", "select EXTRACT(timezone_hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'7\n'),
         ("EXTRACT_6", "select EXTRACT(timezone_minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'30\n'),
         # TO_STRING
-       ("TO_STRING_1", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y') from S3Object as s", b'"January 2, 2017"\n'),
+        ("TO_STRING_1", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y') from S3Object as s",
+            b'"January 2, 2017"\n'),
         ("TO_STRING_2", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMM d, yyyy') from S3Object as s", b'"Jan 2, 2017"\n'),
         ("TO_STRING_3", "select TO_STRING(TO_TIMESTAMP(s.datez), 'M-d-yy') from S3Object as s", b'1-2-17\n'),
         ("TO_STRING_4", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MM-d-y') from S3Object as s", b'01-2-2017\n'),
-       ("TO_STRING_5", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y h:m a') from S3Object as s", b'"January 2, 2017 3:4 AM"\n'),
+        ("TO_STRING_5", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y h:m a') from S3Object as s",
-       ("TO_STRING_6", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'),
+            b'"January 2, 2017 3:4 AM"\n'),
-       ("TO_STRING_7", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'),
+        ("TO_STRING_6", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s",
-       ("TO_STRING_8", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'),
+            b'2017-01-02T3:4:05+0730\n'),
-       ("TO_STRING_9", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXXX') from S3Object as s", b'2017-01-02T3:4:05+07:30\n'),
+        ("TO_STRING_7", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s",
-       ("TO_TIMESTAMP", "select TO_TIMESTAMP(s.datez) from S3Object as s", b'2017-01-02T03:04:05.006+07:30\n'),
+            b'2017-01-02T3:4:05+0730\n'),
+        ("TO_STRING_8", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXX') from S3Object as s",
+            b'2017-01-02T3:4:05+0730\n'),
+        ("TO_STRING_9", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXXX') from S3Object as s",
+            b'2017-01-02T3:4:05+07:30\n'),
+        ("TO_TIMESTAMP", "select TO_TIMESTAMP(s.datez) from S3Object as s",
+            b'2017-01-02T03:04:05.006+07:30\n'),
         ("UTCNOW", "select UTCNOW() from S3Object", datetime(1, 1, 1)),
 
     ]
@@ -253,6 +277,7 @@ def test_sql_functions_date(client, log_output):
     # Test passes
     print(log_output.json_report())
 
+
 def test_sql_functions_string(client, log_output):
 
     json_testfile = """
@@ -264,7 +289,8 @@ def test_sql_functions_string(client, log_output):
     tests = [
         # CHAR_LENGTH
         ("CHAR_LENGTH", "select CHAR_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'),
-       ("CHARACTER_LENGTH", "select CHARACTER_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'),
+        ("CHARACTER_LENGTH",
+            "select CHARACTER_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'),
         # LOWER
         ("LOWER", "select LOWER(s.name) from S3Object as s where s.id= 1", b'john\n'),
         # SUBSTRING
@@ -273,8 +299,10 @@ def test_sql_functions_string(client, log_output):
         ("SUBSTRING_3", "select SUBSTRING(s.name FROM -1 FOR 2) from S3Object as s where s.id = 1", b'\n'),
         # TRIM
         ("TRIM_1", "select TRIM(s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'),
-       ("TRIM_2", "select TRIM(LEADING FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t         \n'),
+        ("TRIM_2", "select TRIM(LEADING FROM s.name) from S3Object as s where s.id = 2",
-       ("TRIM_3", "select TRIM(TRAILING FROM s.name) from S3Object as s where s.id = 2", b'       \tfoobar\t\n'),
+            b'\tfoobar\t         \n'),
+        ("TRIM_3", "select TRIM(TRAILING FROM s.name) from S3Object as s where s.id = 2",
+            b'       \tfoobar\t\n'),
         ("TRIM_4", "select TRIM(BOTH FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'),
         ("TRIM_5", "select TRIM(BOTH '12' FROM s.name) from S3Object as s where s.id = 3", b'foobar\n'),
         # UPPER
@@ -304,7 +332,8 @@ def test_sql_datatypes(client, log_output):
         ("float", "select CAST('13.3' AS FLOAT) from S3Object", b'13.3\n'),
         ("decimal", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'),
         ("numeric", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'),
-       ("timestamp", "select CAST('2007-04-05T14:30Z' AS TIMESTAMP) from S3Object", b'2007-04-05T14:30Z\n'),
+        ("timestamp", "select CAST('2007-04-05T14:30Z' AS TIMESTAMP) from S3Object",
+            b'2007-04-05T14:30Z\n'),
     ]
 
     try:
@@ -323,9 +352,12 @@ def test_sql_select(client, log_output):
     json_testfile = """{"id": 1, "created": "June 27", "modified": "July 6" }
 {"id": 2, "Created": "June 28", "Modified": "July 7", "Cast": "Random Date" }"""
     tests = [
-       ("select_1", "select * from S3Object", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
+        ("select_1", "select * from S3Object",
-       ("select_2", "select * from S3Object s", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
+         b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
-       ("select_3", "select * from S3Object as s", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
+        ("select_2", "select * from S3Object s",
+            b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
+        ("select_3", "select * from S3Object as s",
+            b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
         ("select_4", "select s.line from S3Object as s", b'\n\n'),
         ("select_5", 'select s."Created" from S3Object as s', b'\nJune 28\n'),
         ("select_5", 'select s."Cast" from S3Object as s', b'\nRandom Date\n'),
@@ -343,16 +375,22 @@ def test_sql_select(client, log_output):
     # Test passes
     print(log_output.json_report())
 
+
 def test_sql_select_json(client, log_output):
     json_testcontent = """{ "Rules": [ {"id": "1"}, {"expr": "y > x"}, {"id": "2", "expr": "z = DEBUG"} ]}
 { "created": "June 27", "modified": "July 6" }
 """
     tests = [
-       ("select_1", "SELECT id FROM S3Object[*].Rules[*].id", b'{"id":"1"}\n{}\n{"id":"2"}\n{}\n'),
+        ("select_1", "SELECT id FROM S3Object[*].Rules[*].id",
-       ("select_2", "SELECT id FROM S3Object[*].Rules[*].id WHERE id IS NOT MISSING", b'{"id":"1"}\n{"id":"2"}\n'),
+         b'{"id":"1"}\n{}\n{"id":"2"}\n{}\n'),
-       ("select_3", "SELECT d.created, d.modified FROM S3Object[*] d", b'{}\n{"created":"June 27","modified":"July 6"}\n'),
+        ("select_2",
-       ("select_4", "SELECT _1.created, _1.modified FROM S3Object[*]", b'{}\n{"created":"June 27","modified":"July 6"}\n'),
+            "SELECT id FROM S3Object[*].Rules[*].id WHERE id IS NOT MISSING", b'{"id":"1"}\n{"id":"2"}\n'),
-       ("select_5", "Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'),
+        ("select_3", "SELECT d.created, d.modified FROM S3Object[*] d",
+            b'{}\n{"created":"June 27","modified":"July 6"}\n'),
+        ("select_4", "SELECT _1.created, _1.modified FROM S3Object[*]",
+            b'{}\n{"created":"June 27","modified":"July 6"}\n'),
+        ("select_5",
+            "Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'),
     ]
 
     input_serialization = InputSerialization(json=JSONInput(Type="DOCUMENT"))
@@ -395,5 +433,3 @@ val4,val5,val6
 
     # Test passes
     print(log_output.json_report())
-
-

mint/run/core/s3select/tests.py

@@ -16,12 +16,19 @@
 # limitations under the License.
 
 import os
-from sys import exit
+import sys
+from csv import (test_csv_input_custom_quote_char,
+                 test_csv_output_custom_quote_char)
+
 from minio import Minio
 
+from sql_ops import (test_sql_datatypes, test_sql_functions_agg_cond_conv,
+                     test_sql_functions_date, test_sql_functions_string,
+                     test_sql_operators, test_sql_operators_precedence,
+                     test_sql_select, test_sql_select_csv_no_header,
+                     test_sql_select_json)
 from utils import LogOutput
-from sql_ops import *
+
-from csv import *
 
 def main():
     """
@@ -39,48 +46,56 @@ def main():
             secret_key = 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG'
             secure = True
 
-        client = Minio(server_endpoint, access_key, secret_key, secure=False)
+        client = Minio(server_endpoint, access_key, secret_key, secure=secure)
 
-        log_output = LogOutput(client.select_object_content, 'test_csv_input_quote_char')
+        log_output = LogOutput(client.select_object_content,
+                               'test_csv_input_quote_char')
         test_csv_input_custom_quote_char(client, log_output)
 
-        log_output = LogOutput(client.select_object_content, 'test_csv_output_quote_char')
+        log_output = LogOutput(client.select_object_content,
+                               'test_csv_output_quote_char')
         test_csv_output_custom_quote_char(client, log_output)
 
-        log_output = LogOutput(client.select_object_content, 'test_sql_operators')
+        log_output = LogOutput(
+            client.select_object_content, 'test_sql_operators')
         test_sql_operators(client, log_output)
 
-        log_output = LogOutput(client.select_object_content, 'test_sql_operators_precedence')
+        log_output = LogOutput(client.select_object_content,
+                               'test_sql_operators_precedence')
         test_sql_operators_precedence(client, log_output)
 
-        log_output = LogOutput(client.select_object_content, 'test_sql_functions_agg_cond_conv')
+        log_output = LogOutput(client.select_object_content,
+                               'test_sql_functions_agg_cond_conv')
         test_sql_functions_agg_cond_conv(client, log_output)
 
-        log_output = LogOutput(client.select_object_content, 'test_sql_functions_date')
+        log_output = LogOutput(
+            client.select_object_content, 'test_sql_functions_date')
         test_sql_functions_date(client, log_output)
 
-        log_output = LogOutput(client.select_object_content, 'test_sql_functions_string')
+        log_output = LogOutput(client.select_object_content,
+                               'test_sql_functions_string')
         test_sql_functions_string(client, log_output)
 
-        log_output = LogOutput(client.select_object_content, 'test_sql_datatypes')
+        log_output = LogOutput(
+            client.select_object_content, 'test_sql_datatypes')
         test_sql_datatypes(client, log_output)
 
         log_output = LogOutput(client.select_object_content, 'test_sql_select')
         test_sql_select(client, log_output)
 
-        log_output = LogOutput(client.select_object_content, 'test_sql_select_json')
+        log_output = LogOutput(
+            client.select_object_content, 'test_sql_select_json')
         test_sql_select_json(client, log_output)
 
-        log_output = LogOutput(client.select_object_content, 'test_sql_select_csv')
+        log_output = LogOutput(
+            client.select_object_content, 'test_sql_select_csv')
         test_sql_select_csv_no_header(client, log_output)
 
     except Exception as err:
         print(log_output.json_report(err))
-        exit(1)
+        sys.exit(1)
+
 
 if __name__ == "__main__":
     # Execute only if run as a script
     main()
-
-
-

mint/run/core/s3select/utils.py

@@ -15,11 +15,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import uuid
 import inspect
 import json
 import time
 import traceback
+import uuid
+
 
 class LogOutput(object):
     """
@@ -100,7 +101,6 @@ class LogOutput(object):
 def generate_bucket_name():
     return "s3select-test-" + str(uuid.uuid4())
 
+
 def generate_object_name():
     return str(uuid.uuid4())
-
-

pkg/trace/trace.go

@@ -42,6 +42,7 @@ type CallStats struct {
 // RequestInfo represents trace of http request
 type RequestInfo struct {
 	Time     time.Time   `json:"time"`
+	Proto    string      `json:"proto"`
 	Method   string      `json:"method"`
 	Path     string      `json:"path,omitempty"`
 	RawQuery string      `json:"rawquery,omitempty"`