Implement S3-HDFS gateway (#7440)

- [x] Support bucket and regular object operations
- [x] Supports Select API on HDFS
- [x] Implement multipart API support
- [x] Completion of ListObjects support
This commit is contained in:
Harshavardhana
2019-04-17 09:52:08 -07:00
committed by kannappanr
parent 1d49295943
commit 620e462413
19 changed files with 1088 additions and 291 deletions

13
go.mod
View File

@@ -4,7 +4,7 @@ go 1.12
require (
cloud.google.com/go v0.37.2
contrib.go.opencensus.io/exporter/ocagent v0.4.9 // indirect
contrib.go.opencensus.io/exporter/ocagent v0.4.10 // indirect
github.com/Azure/azure-sdk-for-go v27.0.0+incompatible
github.com/Azure/go-autorest v11.7.0+incompatible
github.com/DataDog/zstd v1.3.5 // indirect
@@ -12,6 +12,7 @@ require (
github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5
github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2
github.com/cheggaaa/pb v1.0.28
github.com/colinmarc/hdfs/v2 v2.0.0
github.com/coredns/coredns v1.4.0
github.com/coreos/bbolt v1.3.2 // indirect
github.com/coreos/etcd v3.3.12+incompatible
@@ -23,7 +24,6 @@ require (
github.com/elazarl/go-bindata-assetfs v1.0.0
github.com/fatih/color v1.7.0
github.com/fatih/structs v1.1.0
github.com/go-ini/ini v1.42.0 // indirect
github.com/go-sql-driver/mysql v1.4.1
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect
github.com/golang/snappy v0.0.1
@@ -54,7 +54,7 @@ require (
github.com/minio/blazer v0.0.0-20171126203752-2081f5bf0465
github.com/minio/cli v1.3.0
github.com/minio/dsync v0.0.0-20190131060523-fb604afd87b2
github.com/minio/highwayhash v0.0.0-20190131021015-02ca4b43caa3
github.com/minio/highwayhash v1.0.0
github.com/minio/lsync v0.0.0-20190207022115-a4e43e3d0887
github.com/minio/mc v0.0.0-20190401030144-a1355e50e2e8
github.com/minio/minio-go v0.0.0-20190327203652-5325257a208f
@@ -79,10 +79,10 @@ require (
github.com/segmentio/go-prompt v1.2.1-0.20161017233205-f0d19b6901ad
github.com/sirupsen/logrus v1.3.0 // indirect
github.com/skyrings/skyring-common v0.0.0-20160929130248-d1c0bb1cbd5e
github.com/smartystreets/assertions v0.0.0-20190401200700-3f99fa72afbb // indirect
github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 // indirect
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect
github.com/soheilhy/cmux v0.1.4 // indirect
github.com/streadway/amqp v0.0.0-20190312223743-14f78b41ce6d
github.com/streadway/amqp v0.0.0-20190402114354-16ed540749f6
github.com/tidwall/gjson v1.2.1
github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65 // indirect
github.com/tidwall/sjson v1.0.4
@@ -97,9 +97,10 @@ require (
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3
golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
golang.org/x/tools v0.0.0-20190408220357-e5b8258f4918 // indirect
google.golang.org/api v0.3.0
gopkg.in/Shopify/sarama.v1 v1.20.0
gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
gopkg.in/jcmturner/goidentity.v2 v2.0.0 // indirect
gopkg.in/olivere/elastic.v5 v5.0.80
gopkg.in/yaml.v2 v2.2.2
)