mirror of
https://github.com/minio/minio.git
synced 2025-11-12 14:51:36 -05:00
Concurrent CSV parsing and reduce S3 select allocations (#8200)
``` CSV parsing, BEFORE: BenchmarkReaderBasic-12 2842 407533 ns/op 397860 B/op 957 allocs/op BenchmarkReaderReplace-12 2718 429914 ns/op 397844 B/op 957 allocs/op BenchmarkReaderReplaceTwo-12 2718 435556 ns/op 397855 B/op 957 allocs/op BenchmarkAggregateCount_100K-12 171 6798974 ns/op 16667102 B/op 308077 allocs/op BenchmarkAggregateCount_1M-12 19 65657411 ns/op 168057743 B/op 3146610 allocs/op BenchmarkSelectAll_10M-12 1 20882119900 ns/op 2758799896 B/op 41978762 allocs/op CSV parsing, AFTER: BenchmarkReaderBasic-12 3721 312549 ns/op 101920 B/op 338 allocs/op BenchmarkReaderReplace-12 3776 318810 ns/op 101993 B/op 340 allocs/op BenchmarkReaderReplaceTwo-12 3610 330967 ns/op 102012 B/op 341 allocs/op BenchmarkAggregateCount_100K-12 295 4149588 ns/op 3553623 B/op 103261 allocs/op BenchmarkAggregateCount_1M-12 30 37746503 ns/op 33827931 B/op 1049435 allocs/op BenchmarkSelectAll_10M-12 1 17608495800 ns/op 1416504040 B/op 21007082 allocs/op ~ benchcmp old.txt new.txt benchmark old ns/op new ns/op delta BenchmarkReaderBasic-12 407533 312549 -23.31% BenchmarkReaderReplace-12 429914 318810 -25.84% BenchmarkReaderReplaceTwo-12 435556 330967 -24.01% BenchmarkAggregateCount_100K-12 6798974 4149588 -38.97% BenchmarkAggregateCount_1M-12 65657411 37746503 -42.51% BenchmarkSelectAll_10M-12 20882119900 17608495800 -15.68% benchmark old allocs new allocs delta BenchmarkReaderBasic-12 957 338 -64.68% BenchmarkReaderReplace-12 957 340 -64.47% BenchmarkReaderReplaceTwo-12 957 341 -64.37% BenchmarkAggregateCount_100K-12 308077 103261 -66.48% BenchmarkAggregateCount_1M-12 3146610 1049435 -66.65% BenchmarkSelectAll_10M-12 41978762 21007082 -49.96% benchmark old bytes new bytes delta BenchmarkReaderBasic-12 397860 101920 -74.38% BenchmarkReaderReplace-12 397844 101993 -74.36% BenchmarkReaderReplaceTwo-12 397855 102012 -74.36% BenchmarkAggregateCount_100K-12 16667102 3553623 -78.68% BenchmarkAggregateCount_1M-12 168057743 33827931 -79.87% BenchmarkSelectAll_10M-12 2758799896 1416504040 -48.66% ``` ``` BenchmarkReaderHuge/97K-12 2200 540840 ns/op 184.32 MB/s 1604450 B/op 687 allocs/op BenchmarkReaderHuge/194K-12 1522 752257 ns/op 265.04 MB/s 2143135 B/op 1335 allocs/op BenchmarkReaderHuge/389K-12 1190 947858 ns/op 420.69 MB/s 3221831 B/op 2630 allocs/op BenchmarkReaderHuge/778K-12 806 1472486 ns/op 541.61 MB/s 5201856 B/op 5187 allocs/op BenchmarkReaderHuge/1557K-12 426 2575269 ns/op 619.36 MB/s 9101330 B/op 10233 allocs/op BenchmarkReaderHuge/3115K-12 286 4034656 ns/op 790.66 MB/s 12397968 B/op 16099 allocs/op BenchmarkReaderHuge/6230K-12 172 6830563 ns/op 934.05 MB/s 16008416 B/op 26844 allocs/op BenchmarkReaderHuge/12461K-12 100 11409467 ns/op 1118.39 MB/s 22655163 B/op 48107 allocs/op BenchmarkReaderHuge/24922K-12 66 19780395 ns/op 1290.19 MB/s 35158559 B/op 90216 allocs/op BenchmarkReaderHuge/49844K-12 34 37282559 ns/op 1369.03 MB/s 60528624 B/op 174497 allocs/op ```
This commit is contained in:
committed by
Harshavardhana
parent
e7f491a14b
commit
ddea0bdf11
12
pkg/s3select/json/testdata/10.json
vendored
Normal file
12
pkg/s3select/json/testdata/10.json
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
[
|
||||
{
|
||||
"key_1": "value",
|
||||
"key_2": "value"
|
||||
}
|
||||
]
|
||||
[
|
||||
{
|
||||
"key_1": "value2",
|
||||
"key_2": "value3"
|
||||
}
|
||||
]
|
||||
8
pkg/s3select/json/testdata/11.json
vendored
Normal file
8
pkg/s3select/json/testdata/11.json
vendored
Normal file
@@ -0,0 +1,8 @@
|
||||
"a"
|
||||
1
|
||||
3.145
|
||||
["a"]
|
||||
{}
|
||||
{
|
||||
"a": 1
|
||||
}
|
||||
5
pkg/s3select/json/testdata/12.json
vendored
Normal file
5
pkg/s3select/json/testdata/12.json
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"a": 1
|
||||
}{
|
||||
"b": 2
|
||||
}
|
||||
1
pkg/s3select/json/testdata/2.json
vendored
Normal file
1
pkg/s3select/json/testdata/2.json
vendored
Normal file
@@ -0,0 +1 @@
|
||||
{"text": "hello world\\n2nd line"}
|
||||
1
pkg/s3select/json/testdata/3.json
vendored
Normal file
1
pkg/s3select/json/testdata/3.json
vendored
Normal file
@@ -0,0 +1 @@
|
||||
{"hello":"wor{l}d"}
|
||||
26
pkg/s3select/json/testdata/4.json
vendored
Normal file
26
pkg/s3select/json/testdata/4.json
vendored
Normal file
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"id": "0001",
|
||||
"type": "donut",
|
||||
"name": "Cake",
|
||||
"ppu": 0.55,
|
||||
"batters":
|
||||
{
|
||||
"batter":
|
||||
[
|
||||
{ "id": "1001", "type": "Regular" },
|
||||
{ "id": "1002", "type": "Chocolate" },
|
||||
{ "id": "1003", "type": "Blueberry" },
|
||||
{ "id": "1004", "type": "Devil's Food" }
|
||||
]
|
||||
},
|
||||
"topping":
|
||||
[
|
||||
{ "id": "5001", "type": "None" },
|
||||
{ "id": "5002", "type": "Glazed" },
|
||||
{ "id": "5005", "type": "Sugar" },
|
||||
{ "id": "5007", "type": "Powdered Sugar" },
|
||||
{ "id": "5006", "type": "Chocolate with Sprinkles" },
|
||||
{ "id": "5003", "type": "Chocolate" },
|
||||
{ "id": "5004", "type": "Maple" }
|
||||
]
|
||||
}
|
||||
5
pkg/s3select/json/testdata/5.json
vendored
Normal file
5
pkg/s3select/json/testdata/5.json
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"foo": {
|
||||
"bar": "baz"
|
||||
}
|
||||
}
|
||||
1
pkg/s3select/json/testdata/6.json
vendored
Normal file
1
pkg/s3select/json/testdata/6.json
vendored
Normal file
@@ -0,0 +1 @@
|
||||
{ "name": "John", "age":28, "hobby": { "name": "chess", "type": "boardgame" }}
|
||||
3
pkg/s3select/json/testdata/7.json
vendored
Normal file
3
pkg/s3select/json/testdata/7.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
{"name":"Michael", "age": 31}
|
||||
{"name":"Andy", "age": 30}
|
||||
{"name":"Justin", "age": 19}
|
||||
2
pkg/s3select/json/testdata/8.json
vendored
Normal file
2
pkg/s3select/json/testdata/8.json
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
{"a":"}"
|
||||
}
|
||||
6
pkg/s3select/json/testdata/9.json
vendored
Normal file
6
pkg/s3select/json/testdata/9.json
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
[
|
||||
{
|
||||
"key_1": "value",
|
||||
"key_2": "value"
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user