mirror of
https://github.com/minio/minio.git
synced 2025-02-04 02:15:59 -05:00
fix: s3select tests with new minio-py SDK (#10995)
This commit is contained in:
Harshavardhana
GitHub
parent
bdd094bc39
commit
ae4ded7fd1
@ -1,3 +1,5 @@
|
||||
module mint.minio.io/minio-go
|
||||
|
||||
go 1.14
|
||||
|
||||
require github.com/minio/minio-go/v7 v7.0.6 // indirect
|
||||
|
||||
@ -19,9 +19,13 @@ import io
|
||||
import os
|
||||
|
||||
from minio import Minio
|
||||
from minio.select.options import (CSVInput, CSVOutput, InputSerialization,
|
||||
JSONOutput, OutputSerialization,
|
||||
RequestProgress, SelectObjectOptions)
|
||||
from minio.selectrequest import (COMPRESSION_TYPE_NONE, FILE_HEADER_INFO_NONE,
|
||||
JSON_TYPE_DOCUMENT, QUOTE_FIELDS_ALWAYS,
|
||||
QUOTE_FIELDS_ASNEEDED, CSVInputSerialization,
|
||||
CSVOutputSerialization,
|
||||
JSONInputSerialization,
|
||||
JSONOutputSerialization, SelectRequest)
|
||||
|
||||
from utils import *
|
||||
|
||||
|
||||
@ -93,26 +97,22 @@ def test_csv_input_custom_quote_char(client, log_output):
|
||||
|
||||
try:
|
||||
for idx, (quote_char, escape_char, data, expected_output) in enumerate(tests):
|
||||
sql_opts = SelectObjectOptions(
|
||||
expression="select * from s3object",
|
||||
input_serialization=InputSerialization(
|
||||
compression_type="NONE",
|
||||
csv=CSVInput(file_header_info="NONE",
|
||||
record_delimiter="\n",
|
||||
field_delimiter=",",
|
||||
quote_character=quote_char,
|
||||
quote_escape_character=escape_char,
|
||||
comments="#",
|
||||
allow_quoted_record_delimiter="FALSE",),
|
||||
sql_opts = SelectRequest(
|
||||
"select * from s3object",
|
||||
CSVInputSerialization(
|
||||
compression_type=COMPRESSION_TYPE_NONE,
|
||||
file_header_info=FILE_HEADER_INFO_NONE,
|
||||
record_delimiter="\n",
|
||||
field_delimiter=",",
|
||||
quote_character=quote_char,
|
||||
quote_escape_character=escape_char,
|
||||
comments="#",
|
||||
allow_quoted_record_delimiter="FALSE",
|
||||
),
|
||||
output_serialization=OutputSerialization(
|
||||
json=JSONOutput(
|
||||
record_delimiter="\n",
|
||||
)
|
||||
JSONOutputSerialization(
|
||||
record_delimiter="\n",
|
||||
),
|
||||
request_progress=RequestProgress(
|
||||
enabled="False"
|
||||
)
|
||||
request_progress=False,
|
||||
)
|
||||
|
||||
test_sql_api(f'test_{idx}', client, bucket_name,
|
||||
@ -150,30 +150,26 @@ def test_csv_output_custom_quote_char(client, log_output):
|
||||
|
||||
try:
|
||||
for idx, (quote_char, escape_char, input_data, expected_output) in enumerate(tests):
|
||||
sql_opts = SelectObjectOptions(
|
||||
expression="select * from s3object",
|
||||
input_serialization=InputSerialization(
|
||||
compression_type="NONE",
|
||||
csv=CSVInput(file_header_info="NONE",
|
||||
record_delimiter="\n",
|
||||
field_delimiter=",",
|
||||
quote_character='"',
|
||||
quote_escape_character='"',
|
||||
comments="#",
|
||||
allow_quoted_record_delimiter="FALSE",
|
||||
),
|
||||
sql_opts = SelectRequest(
|
||||
"select * from s3object",
|
||||
CSVInputSerialization(
|
||||
compression_type=COMPRESSION_TYPE_NONE,
|
||||
file_header_info=FILE_HEADER_INFO_NONE,
|
||||
record_delimiter="\n",
|
||||
field_delimiter=",",
|
||||
quote_character='"',
|
||||
quote_escape_character='"',
|
||||
comments="#",
|
||||
allow_quoted_record_delimiter="FALSE",
|
||||
),
|
||||
output_serialization=OutputSerialization(
|
||||
csv=CSVOutput(quote_fields="ALWAYS",
|
||||
record_delimiter="\n",
|
||||
field_delimiter=",",
|
||||
quote_character=quote_char,
|
||||
quote_escape_character=escape_char,
|
||||
)
|
||||
CSVOutputSerialization(
|
||||
quote_fields=QUOTE_FIELDS_ALWAYS,
|
||||
record_delimiter="\n",
|
||||
field_delimiter=",",
|
||||
quote_character=quote_char,
|
||||
quote_escape_character=escape_char,
|
||||
),
|
||||
request_progress=RequestProgress(
|
||||
enabled="False"
|
||||
)
|
||||
request_progress=False,
|
||||
)
|
||||
|
||||
test_sql_api(f'test_{idx}', client, bucket_name,
|
||||
|
||||
@ -18,9 +18,12 @@
|
||||
import io
|
||||
from datetime import datetime
|
||||
|
||||
from minio.select.options import (CSVInput, CSVOutput, InputSerialization,
|
||||
JSONInput, JSONOutput, OutputSerialization,
|
||||
RequestProgress, SelectObjectOptions)
|
||||
from minio.selectrequest import (FILE_HEADER_INFO_NONE, JSON_TYPE_DOCUMENT,
|
||||
QUOTE_FIELDS_ASNEEDED, CSVInputSerialization,
|
||||
CSVOutputSerialization,
|
||||
JSONInputSerialization,
|
||||
JSONOutputSerialization, SelectRequest)
|
||||
|
||||
from utils import generate_bucket_name, generate_object_name
|
||||
|
||||
|
||||
@ -42,17 +45,15 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input,
|
||||
continue
|
||||
try:
|
||||
log_output.args['total_tests'] += 1
|
||||
options = SelectObjectOptions(
|
||||
expression=select_expression,
|
||||
input_serialization=sql_input,
|
||||
output_serialization=sql_output,
|
||||
request_progress=RequestProgress(
|
||||
enabled="False"
|
||||
)
|
||||
sreq = SelectRequest(
|
||||
select_expression,
|
||||
sql_input,
|
||||
sql_output,
|
||||
request_progress=False
|
||||
)
|
||||
|
||||
data = client.select_object_content(
|
||||
bucket_name, object_name, options)
|
||||
bucket_name, object_name, sreq)
|
||||
|
||||
# Get the records
|
||||
records = io.BytesIO()
|
||||
@ -79,14 +80,13 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input,
|
||||
|
||||
|
||||
def test_sql_expressions(client, input_json_bytes, tests, log_output):
|
||||
input_serialization = InputSerialization(
|
||||
input_serialization = JSONInputSerialization(
|
||||
compression_type="NONE",
|
||||
json=JSONInput(json_type="DOCUMENT"),
|
||||
json_type=JSON_TYPE_DOCUMENT,
|
||||
)
|
||||
|
||||
output_serialization = OutputSerialization(
|
||||
csv=CSVOutput(quote_fields="ASNEEDED")
|
||||
)
|
||||
output_serialization = CSVOutputSerialization(
|
||||
quote_fields=QUOTE_FIELDS_ASNEEDED)
|
||||
|
||||
test_sql_expressions_custom_input_output(client, input_json_bytes,
|
||||
input_serialization, output_serialization, tests, log_output)
|
||||
@ -392,8 +392,8 @@ def test_sql_select_json(client, log_output):
|
||||
"Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'),
|
||||
]
|
||||
|
||||
input_serialization = InputSerialization(json=JSONInput(json_type="DOCUMENT"))
|
||||
output_serialization = OutputSerialization(json=JSONOutput())
|
||||
input_serialization = JSONInputSerialization(json_type=JSON_TYPE_DOCUMENT)
|
||||
output_serialization = JSONOutputSerialization()
|
||||
try:
|
||||
test_sql_expressions_custom_input_output(client, json_testcontent,
|
||||
input_serialization, output_serialization, tests, log_output)
|
||||
@ -414,14 +414,11 @@ val4,val5,val6
|
||||
("select_1", "SELECT s._2 FROM S3Object as s", b'val2\nval5\n'),
|
||||
]
|
||||
|
||||
input_serialization = InputSerialization(
|
||||
csv=CSVInput(
|
||||
file_header_info="NONE",
|
||||
allow_quoted_record_delimiter="FALSE",
|
||||
),
|
||||
input_serialization = CSVInputSerialization(
|
||||
file_header_info=FILE_HEADER_INFO_NONE,
|
||||
allow_quoted_record_delimiter="FALSE",
|
||||
)
|
||||
|
||||
output_serialization = OutputSerialization(csv=CSVOutput())
|
||||
output_serialization = CSVOutputSerialization()
|
||||
try:
|
||||
test_sql_expressions_custom_input_output(client, json_testcontent,
|
||||
input_serialization, output_serialization, tests, log_output)
|
||||
|
||||
@ -21,6 +21,7 @@ from csv import (test_csv_input_custom_quote_char,
|
||||
test_csv_output_custom_quote_char)
|
||||
|
||||
from minio import Minio
|
||||
|
||||
from sql_ops import (test_sql_datatypes, test_sql_functions_agg_cond_conv,
|
||||
test_sql_functions_date, test_sql_functions_string,
|
||||
test_sql_operators, test_sql_operators_precedence,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user