minio/go.mod
Klaus Post ddea0bdf11 Concurrent CSV parsing and reduce S3 select allocations (#8200)
```
CSV parsing, BEFORE:
BenchmarkReaderBasic-12         	    2842	    407533 ns/op	  397860 B/op	     957 allocs/op
BenchmarkReaderReplace-12       	    2718	    429914 ns/op	  397844 B/op	     957 allocs/op
BenchmarkReaderReplaceTwo-12    	    2718	    435556 ns/op	  397855 B/op	     957 allocs/op
BenchmarkAggregateCount_100K-12    	     171	   6798974 ns/op	16667102 B/op	  308077 allocs/op
BenchmarkAggregateCount_1M-12    	      19	  65657411 ns/op	168057743 B/op	 3146610 allocs/op
BenchmarkSelectAll_10M-12    	       1	20882119900 ns/op	2758799896 B/op	41978762 allocs/op

CSV parsing, AFTER:
BenchmarkReaderBasic-12         	    3721	    312549 ns/op	  101920 B/op	     338 allocs/op
BenchmarkReaderReplace-12       	    3776	    318810 ns/op	  101993 B/op	     340 allocs/op
BenchmarkReaderReplaceTwo-12    	    3610	    330967 ns/op	  102012 B/op	     341 allocs/op
BenchmarkAggregateCount_100K-12    	     295	   4149588 ns/op	 3553623 B/op	  103261 allocs/op
BenchmarkAggregateCount_1M-12    	      30	  37746503 ns/op	33827931 B/op	 1049435 allocs/op
BenchmarkSelectAll_10M-12    	       1	17608495800 ns/op	1416504040 B/op	21007082 allocs/op

~ benchcmp old.txt new.txt
benchmark                           old ns/op       new ns/op       delta
BenchmarkReaderBasic-12             407533          312549          -23.31%
BenchmarkReaderReplace-12           429914          318810          -25.84%
BenchmarkReaderReplaceTwo-12        435556          330967          -24.01%
BenchmarkAggregateCount_100K-12     6798974         4149588         -38.97%
BenchmarkAggregateCount_1M-12       65657411        37746503        -42.51%
BenchmarkSelectAll_10M-12           20882119900     17608495800     -15.68%

benchmark                           old allocs     new allocs     delta
BenchmarkReaderBasic-12             957            338            -64.68%
BenchmarkReaderReplace-12           957            340            -64.47%
BenchmarkReaderReplaceTwo-12        957            341            -64.37%
BenchmarkAggregateCount_100K-12     308077         103261         -66.48%
BenchmarkAggregateCount_1M-12       3146610        1049435        -66.65%
BenchmarkSelectAll_10M-12           41978762       21007082       -49.96%

benchmark                           old bytes      new bytes      delta
BenchmarkReaderBasic-12             397860         101920         -74.38%
BenchmarkReaderReplace-12           397844         101993         -74.36%
BenchmarkReaderReplaceTwo-12        397855         102012         -74.36%
BenchmarkAggregateCount_100K-12     16667102       3553623        -78.68%
BenchmarkAggregateCount_1M-12       168057743      33827931       -79.87%
BenchmarkSelectAll_10M-12           2758799896     1416504040     -48.66%
```

```
BenchmarkReaderHuge/97K-12         	    2200	    540840 ns/op	 184.32 MB/s	 1604450 B/op	     687 allocs/op
BenchmarkReaderHuge/194K-12        	    1522	    752257 ns/op	 265.04 MB/s	 2143135 B/op	    1335 allocs/op
BenchmarkReaderHuge/389K-12        	    1190	    947858 ns/op	 420.69 MB/s	 3221831 B/op	    2630 allocs/op
BenchmarkReaderHuge/778K-12        	     806	   1472486 ns/op	 541.61 MB/s	 5201856 B/op	    5187 allocs/op
BenchmarkReaderHuge/1557K-12       	     426	   2575269 ns/op	 619.36 MB/s	 9101330 B/op	   10233 allocs/op
BenchmarkReaderHuge/3115K-12       	     286	   4034656 ns/op	 790.66 MB/s	12397968 B/op	   16099 allocs/op
BenchmarkReaderHuge/6230K-12       	     172	   6830563 ns/op	 934.05 MB/s	16008416 B/op	   26844 allocs/op
BenchmarkReaderHuge/12461K-12      	     100	  11409467 ns/op	1118.39 MB/s	22655163 B/op	   48107 allocs/op
BenchmarkReaderHuge/24922K-12      	      66	  19780395 ns/op	1290.19 MB/s	35158559 B/op	   90216 allocs/op
BenchmarkReaderHuge/49844K-12      	      34	  37282559 ns/op	1369.03 MB/s	60528624 B/op	  174497 allocs/op
```
2019-09-13 14:18:35 -07:00

89 lines
3.4 KiB
Modula-2

module github.com/minio/minio
go 1.13
require (
cloud.google.com/go v0.37.2
github.com/Azure/azure-sdk-for-go v27.0.0+incompatible
github.com/Azure/go-autorest v11.7.0+incompatible
github.com/alecthomas/participle v0.2.1
github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5
github.com/aws/aws-sdk-go v1.20.21
github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2
github.com/cheggaaa/pb v1.0.28
github.com/coredns/coredns v1.4.0
github.com/coreos/etcd v3.3.12+incompatible
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/djherbis/atime v1.0.0
github.com/dustin/go-humanize v1.0.0
github.com/eclipse/paho.mqtt.golang v1.1.2-0.20190322152051-20337d8c3947
github.com/elazarl/go-bindata-assetfs v1.0.0
github.com/fatih/color v1.7.0
github.com/fatih/structs v1.1.0
github.com/go-sql-driver/mysql v1.4.1
github.com/golang/snappy v0.0.1
github.com/gomodule/redigo v2.0.0+incompatible
github.com/gorilla/handlers v1.4.0
github.com/gorilla/mux v1.7.0
github.com/gorilla/rpc v1.2.0+incompatible
github.com/hashicorp/vault v1.1.0
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf
github.com/json-iterator/go v1.1.7
github.com/klauspost/compress v1.5.0
github.com/klauspost/pgzip v1.2.1
github.com/klauspost/readahead v1.3.0
github.com/klauspost/reedsolomon v1.9.1
github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3
github.com/lib/pq v1.0.0
github.com/mattn/go-isatty v0.0.7
github.com/miekg/dns v1.1.8
github.com/minio/cli v1.21.0
github.com/minio/dsync/v2 v2.0.0
github.com/minio/gokrb5/v7 v7.2.5
github.com/minio/hdfs/v3 v3.0.1
github.com/minio/highwayhash v1.0.0
github.com/minio/lsync v1.0.1
github.com/minio/mc v0.0.0-20190908212443-54ee3a280031
github.com/minio/minio-go v0.0.0-20190327203652-5325257a208f
github.com/minio/minio-go/v6 v6.0.35
github.com/minio/parquet-go v0.0.0-20190318185229-9d767baf1679
github.com/minio/sha256-simd v0.1.0
github.com/minio/sio v0.2.0
github.com/mitchellh/go-homedir v1.1.0
github.com/nats-io/nats.go v1.8.0
github.com/nats-io/stan.go v0.4.5
github.com/ncw/directio v1.0.5
github.com/nsqio/go-nsq v1.0.7
github.com/pkg/errors v0.8.1
github.com/pkg/profile v1.3.0
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829
github.com/rjeczalik/notify v0.9.2
github.com/rs/cors v1.6.0
github.com/sirupsen/logrus v1.4.2
github.com/skyrings/skyring-common v0.0.0-20160929130248-d1c0bb1cbd5e
github.com/streadway/amqp v0.0.0-20190402114354-16ed540749f6
github.com/tidwall/gjson v1.2.1
github.com/tidwall/sjson v1.0.4
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a
go.uber.org/atomic v1.3.2
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456
google.golang.org/api v0.4.0
gopkg.in/Shopify/sarama.v1 v1.20.0
gopkg.in/ldap.v3 v3.0.3
gopkg.in/olivere/elastic.v5 v5.0.80
gopkg.in/yaml.v2 v2.2.2
)
// Added for go1.13 migration https://github.com/golang/go/issues/32805
replace github.com/gorilla/rpc v1.2.0+incompatible => github.com/gorilla/rpc v1.2.0
// Allow this for offline builds
replace github.com/eapache/go-xerial-snappy => github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21
replace github.com/eapache/queue => github.com/eapache/queue v1.1.0
replace github.com/mattn/go-runewidth => github.com/mattn/go-runewidth v0.0.4
replace github.com/mitchellh/mapstructure => github.com/mitchellh/mapstructure v1.1.2