MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-02-04 10:26:01 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: s3select tests with new minio-py SDK (#10995)

Browse Source
This commit is contained in:
Harshavardhana 2020-11-29 13:05:37 -08:00 committed by GitHub
parent bdd094bc39
commit ae4ded7fd1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 64 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mint/run/core/minio-go/go.mod
View File

@ -1,3 +1,5 @@
module mint.minio.io/minio-go module mint.minio.io/minio-go
go 1.14 go 1.14
require github.com/minio/minio-go/v7 v7.0.6 // indirect

82
mint/run/core/s3select/csv.py
View File

@ -19,9 +19,13 @@ import io
import os import os
from minio import Minio from minio import Minio
from minio.select.options import (CSVInput, CSVOutput, InputSerialization, from minio.selectrequest import (COMPRESSION_TYPE_NONE, FILE_HEADER_INFO_NONE,
JSONOutput, OutputSerialization, JSON_TYPE_DOCUMENT, QUOTE_FIELDS_ALWAYS,
RequestProgress, SelectObjectOptions) QUOTE_FIELDS_ASNEEDED, CSVInputSerialization,
CSVOutputSerialization,
JSONInputSerialization,
JSONOutputSerialization, SelectRequest)
from utils import * from utils import *
@ -93,26 +97,22 @@ def test_csv_input_custom_quote_char(client, log_output):
try: try:
for idx, (quote_char, escape_char, data, expected_output) in enumerate(tests): for idx, (quote_char, escape_char, data, expected_output) in enumerate(tests):
sql_opts = SelectObjectOptions( sql_opts = SelectRequest(
expression="select * from s3object", "select * from s3object",
input_serialization=InputSerialization( CSVInputSerialization(
compression_type="NONE", compression_type=COMPRESSION_TYPE_NONE,
csv=CSVInput(file_header_info="NONE", file_header_info=FILE_HEADER_INFO_NONE,
record_delimiter="\n", record_delimiter="\n",
field_delimiter=",", field_delimiter=",",
quote_character=quote_char, quote_character=quote_char,
quote_escape_character=escape_char, quote_escape_character=escape_char,
comments="#", comments="#",
allow_quoted_record_delimiter="FALSE",), allow_quoted_record_delimiter="FALSE",
), ),
output_serialization=OutputSerialization( JSONOutputSerialization(
json=JSONOutput( record_delimiter="\n",
record_delimiter="\n",
)
), ),
request_progress=RequestProgress( request_progress=False,
enabled="False"
)
) )
test_sql_api(f'test_{idx}', client, bucket_name, test_sql_api(f'test_{idx}', client, bucket_name,
@ -150,30 +150,26 @@ def test_csv_output_custom_quote_char(client, log_output):
try: try:
for idx, (quote_char, escape_char, input_data, expected_output) in enumerate(tests): for idx, (quote_char, escape_char, input_data, expected_output) in enumerate(tests):
sql_opts = SelectObjectOptions( sql_opts = SelectRequest(
expression="select * from s3object", "select * from s3object",
input_serialization=InputSerialization( CSVInputSerialization(
compression_type="NONE", compression_type=COMPRESSION_TYPE_NONE,
csv=CSVInput(file_header_info="NONE", file_header_info=FILE_HEADER_INFO_NONE,
record_delimiter="\n", record_delimiter="\n",
field_delimiter=",", field_delimiter=",",
quote_character='"', quote_character='"',
quote_escape_character='"', quote_escape_character='"',
comments="#", comments="#",
allow_quoted_record_delimiter="FALSE", allow_quoted_record_delimiter="FALSE",
),
), ),
output_serialization=OutputSerialization( CSVOutputSerialization(
csv=CSVOutput(quote_fields="ALWAYS", quote_fields=QUOTE_FIELDS_ALWAYS,
record_delimiter="\n", record_delimiter="\n",
field_delimiter=",", field_delimiter=",",
quote_character=quote_char, quote_character=quote_char,
quote_escape_character=escape_char, quote_escape_character=escape_char,
)
), ),
request_progress=RequestProgress( request_progress=False,
enabled="False"
)
) )
test_sql_api(f'test_{idx}', client, bucket_name, test_sql_api(f'test_{idx}', client, bucket_name,

47
mint/run/core/s3select/sql_ops.py
View File

@ -18,9 +18,12 @@
import io import io
from datetime import datetime from datetime import datetime
from minio.select.options import (CSVInput, CSVOutput, InputSerialization, from minio.selectrequest import (FILE_HEADER_INFO_NONE, JSON_TYPE_DOCUMENT,
JSONInput, JSONOutput, OutputSerialization, QUOTE_FIELDS_ASNEEDED, CSVInputSerialization,
RequestProgress, SelectObjectOptions) CSVOutputSerialization,
JSONInputSerialization,
JSONOutputSerialization, SelectRequest)
from utils import generate_bucket_name, generate_object_name from utils import generate_bucket_name, generate_object_name
@ -42,17 +45,15 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input,
continue continue
try: try:
log_output.args['total_tests'] += 1 log_output.args['total_tests'] += 1
options = SelectObjectOptions( sreq = SelectRequest(
expression=select_expression, select_expression,
input_serialization=sql_input, sql_input,
output_serialization=sql_output, sql_output,
request_progress=RequestProgress( request_progress=False
enabled="False"
)
) )
data = client.select_object_content( data = client.select_object_content(
bucket_name, object_name, options) bucket_name, object_name, sreq)
# Get the records # Get the records
records = io.BytesIO() records = io.BytesIO()
@ -79,14 +80,13 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input,
def test_sql_expressions(client, input_json_bytes, tests, log_output): def test_sql_expressions(client, input_json_bytes, tests, log_output):
input_serialization = InputSerialization( input_serialization = JSONInputSerialization(
compression_type="NONE", compression_type="NONE",
json=JSONInput(json_type="DOCUMENT"), json_type=JSON_TYPE_DOCUMENT,
) )
output_serialization = OutputSerialization( output_serialization = CSVOutputSerialization(
csv=CSVOutput(quote_fields="ASNEEDED") quote_fields=QUOTE_FIELDS_ASNEEDED)
)
test_sql_expressions_custom_input_output(client, input_json_bytes, test_sql_expressions_custom_input_output(client, input_json_bytes,
input_serialization, output_serialization, tests, log_output) input_serialization, output_serialization, tests, log_output)
@ -392,8 +392,8 @@ def test_sql_select_json(client, log_output):
"Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'), "Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'),
] ]
input_serialization = InputSerialization(json=JSONInput(json_type="DOCUMENT")) input_serialization = JSONInputSerialization(json_type=JSON_TYPE_DOCUMENT)
output_serialization = OutputSerialization(json=JSONOutput()) output_serialization = JSONOutputSerialization()
try: try:
test_sql_expressions_custom_input_output(client, json_testcontent, test_sql_expressions_custom_input_output(client, json_testcontent,
input_serialization, output_serialization, tests, log_output) input_serialization, output_serialization, tests, log_output)
@ -414,14 +414,11 @@ val4,val5,val6
("select_1", "SELECT s._2 FROM S3Object as s", b'val2\nval5\n'), ("select_1", "SELECT s._2 FROM S3Object as s", b'val2\nval5\n'),
] ]
input_serialization = InputSerialization( input_serialization = CSVInputSerialization(
csv=CSVInput( file_header_info=FILE_HEADER_INFO_NONE,
file_header_info="NONE", allow_quoted_record_delimiter="FALSE",
allow_quoted_record_delimiter="FALSE",
),
) )
output_serialization = CSVOutputSerialization()
output_serialization = OutputSerialization(csv=CSVOutput())
try: try:
test_sql_expressions_custom_input_output(client, json_testcontent, test_sql_expressions_custom_input_output(client, json_testcontent,
input_serialization, output_serialization, tests, log_output) input_serialization, output_serialization, tests, log_output)

1
mint/run/core/s3select/tests.py
View File

@ -21,6 +21,7 @@ from csv import (test_csv_input_custom_quote_char,
test_csv_output_custom_quote_char) test_csv_output_custom_quote_char)
from minio import Minio from minio import Minio
from sql_ops import (test_sql_datatypes, test_sql_functions_agg_cond_conv, from sql_ops import (test_sql_datatypes, test_sql_functions_agg_cond_conv,
test_sql_functions_date, test_sql_functions_string, test_sql_functions_date, test_sql_functions_string,
test_sql_operators, test_sql_operators_precedence, test_sql_operators, test_sql_operators_precedence,