mirror of
https://github.com/minio/minio.git
synced 2025-01-11 15:03:22 -05:00
Bring in the list APIs implemented by Bala <bala@minio.io>
This commit is contained in:
parent
0a9496462a
commit
3f1c4bb4b0
68
JWT.md
68
JWT.md
@ -17,41 +17,45 @@ openssl rsa -in ~/.minio/web/private.key -outform PEM -pubout -out ~/.minio/web/
|
|||||||
minio server <testdir>
|
minio server <testdir>
|
||||||
```
|
```
|
||||||
|
|
||||||
### Now you can make curl requests to the server at port 9001.
|
### Implemented JSON RPC APIs.
|
||||||
|
|
||||||
Currently username and password are defaulted for testing purposes.
|
Namespace `Web`
|
||||||
|
|
||||||
```
|
* Login - waits for 'username, password' and on success replies a new JWT token.
|
||||||
curl -X POST -H "Content-Type: application/json" -d '{"username":"WLGDGYAQYIGI833EV05A", "password": "BYvgJM101sHngl2uzjXS/OBF/aMxAN06JrJ3qJlF"}' http://127.0.0.1:9001/login
|
* ResetToken - resets token, requires password and token.
|
||||||
{"token":"eyJhbGciOiJSUzUxMiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE0NTM1NDM0MjMsImlhdCI6MTQ1MzUwNzQyMywic3ViIjoiV0xHREdZQVFZSUdJODMzRVYwNUEifQ.zhL0vG5dwwak3SvpysW0CzdPRjpadrCLIpte2QHSxj2XjIQb2oK0dDD9Yvl-45E14CMVQhV3CCsf9LFaK2C94I5aop6nP7sSCyG2_l4w2xrfEPWKgyOY9P0QxUIPV3o43o2XjnMlU_6xE2mk8S9N7psk15sf0Ma1EoXkQlfqEZzbxyQjwKx4UxzkVpwN4k6wavtwU-rgVU0QwJwXXss0hVhY7HWtOzUGrhVWL42pOwNwZ73lrHpJkSyQi6fbc5lIALgFoeei_iSUXxRaJjvm36rn4vui3qLCoH79E-WhkoP_mqDvf_YfiTqcFHgdgnu2wtlQl90RNh2-wgR-XJiedQ"}
|
* Logout - currently a dummy operation.
|
||||||
|
* ListBuckets - lists buckets, requires valid token.
|
||||||
|
* ListObjects - lists objects, requires valid token.
|
||||||
|
* GetObjectURL - generates a url for download access, requires valid token.
|
||||||
|
|
||||||
|
### Now you can use `webrpc.js` to make requests.
|
||||||
|
|
||||||
|
- Login example
|
||||||
|
```js
|
||||||
|
var webRPC = require('webrpc');
|
||||||
|
var web = new webRPC("http://localhost:9001/rpc")
|
||||||
|
|
||||||
|
// Generate JWT Token.
|
||||||
|
web.Login({"username": "YOUR-ACCESS-KEY-ID", "password": "YOUR-SECRET-ACCESS-KEY"})
|
||||||
|
.then(function(data) {
|
||||||
|
console.log("success : ", data);
|
||||||
|
})
|
||||||
|
.catch(function(error) {
|
||||||
|
console.log("fail : ", error.toString());
|
||||||
|
});
|
||||||
```
|
```
|
||||||
|
|
||||||
Replies back with a token which can be used to logout
|
- ListBuckets example
|
||||||
|
```js
|
||||||
|
var webRPC = require('webrpc');
|
||||||
|
var web = new webRPC("http://localhost:9001/rpc", "my-token")
|
||||||
|
|
||||||
```
|
// Generate Token.
|
||||||
curl -i -X GET -H "Authorization: Bearer eyJhbGciOiJSUzUxMiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE0NTM1NDM0MjMsImlhdCI6MTQ1MzUwNzQyMywic3ViIjoiV0xHREdZQVFZSUdJODMzRVYwNUEifQ.zhL0vG5dwwak3SvpysW0CzdPRjpadrCLIpte2QHSxj2XjIQb2oK0dDD9Yvl-45E14CMVQhV3CCsf9LFaK2C94I5aop6nP7sSCyG2_l4w2xrfEPWKgyOY9P0QxUIPV3o43o2XjnMlU_6xE2mk8S9N7psk15sf0Ma1EoXkQlfqEZzbxyQjwKx4UxzkVpwN4k6wavtwU-rgVU0QwJwXXss0hVhY7HWtOzUGrhVWL42pOwNwZ73lrHpJkSyQi6fbc5lIALgFoeei_iSUXxRaJjvm36rn4vui3qLCoH79E-WhkoP_mqDvf_YfiTqcFHgdgnu2wtlQl90RNh2-wgR-XJiedQ" http://127.0.0.1:9001/logout
|
web.ListBuckets()
|
||||||
HTTP/1.1 200 OK
|
.then(function(data) {
|
||||||
Content-Type: application/json
|
console.log("Success : ", data);
|
||||||
Date: Sat, 23 Jan 2016 00:05:02 GMT
|
})
|
||||||
Content-Length: 0
|
.catch(function(error) {
|
||||||
```
|
console.log("fail : ", error.toString());
|
||||||
|
});
|
||||||
|
|
||||||
Now attempt with wrong authorization, you should get 401.
|
|
||||||
|
|
||||||
```
|
|
||||||
$ curl -i -X GET -H "Authorization: Bearer testing123" http://127.0.0.1:9001/logout
|
|
||||||
HTTP/1.1 401 Unauthorized
|
|
||||||
Date: Sat, 23 Jan 2016 00:05:58 GMT
|
|
||||||
Content-Length: 0
|
|
||||||
Content-Type: text/plain; charset=utf-8
|
|
||||||
```
|
|
||||||
|
|
||||||
Without authorization logout is not possible.
|
|
||||||
```
|
|
||||||
$ curl -i -X GET http://127.0.0.1:9001/logout
|
|
||||||
HTTP/1.1 401 Unauthorized
|
|
||||||
Date: Sat, 23 Jan 2016 00:07:00 GMT
|
|
||||||
Content-Length: 0
|
|
||||||
Content-Type: text/plain; charset=utf-8
|
|
||||||
```
|
```
|
||||||
|
@ -19,8 +19,9 @@ func AuthHandler(h http.Handler) http.Handler {
|
|||||||
|
|
||||||
// Ignore request if authorization header is not valid.
|
// Ignore request if authorization header is not valid.
|
||||||
func (h authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
func (h authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
// For login attempts please issue a new token.
|
// Let the top level caller handle if the requests should be
|
||||||
if r.Method == "POST" && r.URL.Path == "/login-token" {
|
// allowed.
|
||||||
|
if r.Header.Get("Authorization") == "" {
|
||||||
h.handler.ServeHTTP(w, r)
|
h.handler.ServeHTTP(w, r)
|
||||||
return
|
return
|
||||||
}
|
}
|
17
jwt.go
17
jwt.go
@ -45,14 +45,10 @@ func (b *JWTAuthBackend) GenerateToken(userName string) (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Authenticate -
|
// Authenticate -
|
||||||
func (b *JWTAuthBackend) Authenticate(user *User) bool {
|
func (b *JWTAuthBackend) Authenticate(args *LoginArgs, accessKeyID, secretAccessKey string) bool {
|
||||||
hashedPassword, _ := bcrypt.GenerateFromPassword([]byte("BYvgJM101sHngl2uzjXS/OBF/aMxAN06JrJ3qJlF"), 10)
|
hashedPassword, _ := bcrypt.GenerateFromPassword([]byte(secretAccessKey), 10)
|
||||||
testUser := User{
|
if args.Username == accessKeyID {
|
||||||
Username: "WLGDGYAQYIGI833EV05A",
|
return bcrypt.CompareHashAndPassword(hashedPassword, []byte(args.Password)) == nil
|
||||||
Password: string(hashedPassword),
|
|
||||||
}
|
|
||||||
if user.Username == testUser.Username {
|
|
||||||
return bcrypt.CompareHashAndPassword([]byte(testUser.Password), []byte(user.Password)) == nil
|
|
||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
@ -69,9 +65,8 @@ func (b *JWTAuthBackend) getTokenRemainingValidity(timestamp interface{}) int {
|
|||||||
return expireOffset
|
return expireOffset
|
||||||
}
|
}
|
||||||
|
|
||||||
// Logout - logout
|
// Logout - logout is not implemented yet.
|
||||||
func (b *JWTAuthBackend) Logout(tokenString string, token *jwt.Token) error {
|
func (b *JWTAuthBackend) Logout(tokenString string) error {
|
||||||
b.getTokenRemainingValidity(token.Claims["exp"])
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
40
routers.go
40
routers.go
@ -20,6 +20,8 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
router "github.com/gorilla/mux"
|
router "github.com/gorilla/mux"
|
||||||
|
jsonrpc "github.com/gorilla/rpc/v2"
|
||||||
|
"github.com/gorilla/rpc/v2/json"
|
||||||
"github.com/minio/minio/pkg/fs"
|
"github.com/minio/minio/pkg/fs"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -32,22 +34,16 @@ type CloudStorageAPI struct {
|
|||||||
|
|
||||||
// WebAPI container for Web API.
|
// WebAPI container for Web API.
|
||||||
type WebAPI struct {
|
type WebAPI struct {
|
||||||
Anonymous bool
|
Anonymous bool
|
||||||
AccessLog bool
|
AccessLog bool
|
||||||
|
AccessKeyID string
|
||||||
|
SecretAccessKey string
|
||||||
}
|
}
|
||||||
|
|
||||||
// registerWebAPI - register all the handlers to their respective paths
|
func getWebAPIHandler(web *WebAPI) http.Handler {
|
||||||
func registerWebAPI(mux *router.Router, w WebAPI) {
|
|
||||||
// root Router
|
|
||||||
root := mux.NewRoute().PathPrefix("/").Subrouter()
|
|
||||||
root.Methods("POST").Path("/login").HandlerFunc(w.LoginHandler)
|
|
||||||
root.Methods("GET").Path("/logout").HandlerFunc(w.LogoutHandler)
|
|
||||||
root.Methods("GET").Path("/login-refresh-token").HandlerFunc(w.RefreshTokenHandler)
|
|
||||||
}
|
|
||||||
|
|
||||||
func getWebAPIHandler(web WebAPI) http.Handler {
|
|
||||||
var mwHandlers = []MiddlewareHandler{
|
var mwHandlers = []MiddlewareHandler{
|
||||||
CorsHandler, // CORS added only for testing purposes.
|
TimeValidityHandler, // Validate time.
|
||||||
|
CorsHandler, // CORS added only for testing purposes.
|
||||||
}
|
}
|
||||||
if !web.Anonymous {
|
if !web.Anonymous {
|
||||||
mwHandlers = append(mwHandlers, AuthHandler)
|
mwHandlers = append(mwHandlers, AuthHandler)
|
||||||
@ -55,8 +51,17 @@ func getWebAPIHandler(web WebAPI) http.Handler {
|
|||||||
if web.AccessLog {
|
if web.AccessLog {
|
||||||
mwHandlers = append(mwHandlers, AccessLogHandler)
|
mwHandlers = append(mwHandlers, AccessLogHandler)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
s := jsonrpc.NewServer()
|
||||||
|
codec := json.NewCodec()
|
||||||
|
s.RegisterCodec(codec, "application/json")
|
||||||
|
s.RegisterCodec(codec, "application/json; charset=UTF-8")
|
||||||
|
s.RegisterService(web, "Web")
|
||||||
mux := router.NewRouter()
|
mux := router.NewRouter()
|
||||||
registerWebAPI(mux, web)
|
// Add new RPC services here
|
||||||
|
mux.Handle("/rpc", s)
|
||||||
|
// Enable this when we add assets.
|
||||||
|
// mux.Handle("/{file:.*}", http.FileServer(assetFS()))
|
||||||
return registerCustomMiddleware(mux, mwHandlers...)
|
return registerCustomMiddleware(mux, mwHandlers...)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -94,11 +99,14 @@ func registerCloudStorageAPI(mux *router.Router, a CloudStorageAPI) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// getNewWebAPI instantiate a new WebAPI.
|
// getNewWebAPI instantiate a new WebAPI.
|
||||||
func getNewWebAPI(conf cloudServerConfig) WebAPI {
|
func getNewWebAPI(conf cloudServerConfig) *WebAPI {
|
||||||
return WebAPI{
|
web := &WebAPI{
|
||||||
Anonymous: conf.Anonymous,
|
Anonymous: conf.Anonymous,
|
||||||
AccessLog: conf.AccessLog,
|
AccessLog: conf.AccessLog,
|
||||||
}
|
}
|
||||||
|
web.AccessKeyID = conf.AccessKeyID
|
||||||
|
web.SecretAccessKey = conf.SecretAccessKey
|
||||||
|
return web
|
||||||
}
|
}
|
||||||
|
|
||||||
// getNewCloudStorageAPI instantiate a new CloudStorageAPI.
|
// getNewCloudStorageAPI instantiate a new CloudStorageAPI.
|
||||||
|
@ -74,6 +74,10 @@ type cloudServerConfig struct {
|
|||||||
AccessLog bool // Enable access log handler
|
AccessLog bool // Enable access log handler
|
||||||
Anonymous bool // No signature turn off
|
Anonymous bool // No signature turn off
|
||||||
|
|
||||||
|
// Credentials.
|
||||||
|
AccessKeyID string // Access key id.
|
||||||
|
SecretAccessKey string // Secret access key.
|
||||||
|
|
||||||
/// FS options
|
/// FS options
|
||||||
Path string // Path to export for cloud storage
|
Path string // Path to export for cloud storage
|
||||||
MinFreeDisk int64 // Minimum free disk space for filesystem
|
MinFreeDisk int64 // Minimum free disk space for filesystem
|
||||||
@ -293,13 +297,13 @@ func (a accessKeys) JSON() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// initServer initialize server
|
// initServer initialize server
|
||||||
func initServer() *probe.Error {
|
func initServer() (*configV2, *probe.Error) {
|
||||||
conf, err := getConfig()
|
conf, err := getConfig()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err.Trace()
|
return nil, err.Trace()
|
||||||
}
|
}
|
||||||
if err := setLogger(conf); err != nil {
|
if err := setLogger(conf); err != nil {
|
||||||
return err.Trace()
|
return nil, err.Trace()
|
||||||
}
|
}
|
||||||
if conf != nil {
|
if conf != nil {
|
||||||
Println()
|
Println()
|
||||||
@ -319,7 +323,7 @@ func initServer() *probe.Error {
|
|||||||
Println("\t$ ./mc cp --recursive ~/Photos localhost:9000/photobucket")
|
Println("\t$ ./mc cp --recursive ~/Photos localhost:9000/photobucket")
|
||||||
}
|
}
|
||||||
Println()
|
Println()
|
||||||
return nil
|
return conf, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkServerSyntax(c *cli.Context) {
|
func checkServerSyntax(c *cli.Context) {
|
||||||
@ -338,7 +342,7 @@ func checkServerSyntax(c *cli.Context) {
|
|||||||
func serverMain(c *cli.Context) {
|
func serverMain(c *cli.Context) {
|
||||||
checkServerSyntax(c)
|
checkServerSyntax(c)
|
||||||
|
|
||||||
perr := initServer()
|
conf, perr := initServer()
|
||||||
fatalIf(perr.Trace(), "Failed to read config for minio.", nil)
|
fatalIf(perr.Trace(), "Failed to read config for minio.", nil)
|
||||||
|
|
||||||
certFile := c.GlobalString("cert")
|
certFile := c.GlobalString("cert")
|
||||||
@ -390,17 +394,19 @@ func serverMain(c *cli.Context) {
|
|||||||
}
|
}
|
||||||
tls := (certFile != "" && keyFile != "")
|
tls := (certFile != "" && keyFile != "")
|
||||||
apiServerConfig := cloudServerConfig{
|
apiServerConfig := cloudServerConfig{
|
||||||
Address: c.GlobalString("address"),
|
Address: c.GlobalString("address"),
|
||||||
WebAddress: c.GlobalString("web-address"),
|
WebAddress: c.GlobalString("web-address"),
|
||||||
AccessLog: c.GlobalBool("enable-accesslog"),
|
AccessLog: c.GlobalBool("enable-accesslog"),
|
||||||
Anonymous: c.GlobalBool("anonymous"),
|
Anonymous: c.GlobalBool("anonymous"),
|
||||||
Path: path,
|
AccessKeyID: conf.Credentials.AccessKeyID,
|
||||||
MinFreeDisk: minFreeDisk,
|
SecretAccessKey: conf.Credentials.SecretAccessKey,
|
||||||
Expiry: expiration,
|
Path: path,
|
||||||
TLS: tls,
|
MinFreeDisk: minFreeDisk,
|
||||||
CertFile: certFile,
|
Expiry: expiration,
|
||||||
KeyFile: keyFile,
|
TLS: tls,
|
||||||
RateLimit: c.GlobalInt("ratelimit"),
|
CertFile: certFile,
|
||||||
|
KeyFile: keyFile,
|
||||||
|
RateLimit: c.GlobalInt("ratelimit"),
|
||||||
}
|
}
|
||||||
perr = startServer(apiServerConfig)
|
perr = startServer(apiServerConfig)
|
||||||
errorIf(perr.Trace(), "Failed to start the minio server.", nil)
|
errorIf(perr.Trace(), "Failed to start the minio server.", nil)
|
||||||
|
@ -18,6 +18,9 @@ package main
|
|||||||
|
|
||||||
import "errors"
|
import "errors"
|
||||||
|
|
||||||
|
// errUnAuthorizedRequest - un authorized request.
|
||||||
|
var errUnAuthorizedRequest = errors.New("Unauthorized request")
|
||||||
|
|
||||||
// errSysLogNotSupported - this message is only meaningful on windows
|
// errSysLogNotSupported - this message is only meaningful on windows
|
||||||
var errSysLogNotSupported = errors.New("Syslog logger not supported on windows")
|
var errSysLogNotSupported = errors.New("Syslog logger not supported on windows")
|
||||||
|
|
||||||
|
27
vendor/github.com/gorilla/rpc/v2/LICENSE
generated
vendored
Normal file
27
vendor/github.com/gorilla/rpc/v2/LICENSE
generated
vendored
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
Copyright (c) 2012 Rodrigo Moraes. All rights reserved.
|
||||||
|
|
||||||
|
Redistribution and use in source and binary forms, with or without
|
||||||
|
modification, are permitted provided that the following conditions are
|
||||||
|
met:
|
||||||
|
|
||||||
|
* Redistributions of source code must retain the above copyright
|
||||||
|
notice, this list of conditions and the following disclaimer.
|
||||||
|
* Redistributions in binary form must reproduce the above
|
||||||
|
copyright notice, this list of conditions and the following disclaimer
|
||||||
|
in the documentation and/or other materials provided with the
|
||||||
|
distribution.
|
||||||
|
* Neither the name of Google Inc. nor the names of its
|
||||||
|
contributors may be used to endorse or promote products derived from
|
||||||
|
this software without specific prior written permission.
|
||||||
|
|
||||||
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||||
|
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||||
|
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||||
|
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||||
|
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
|
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||||
|
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||||
|
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||||
|
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||||
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
6
vendor/github.com/gorilla/rpc/v2/README.md
generated
vendored
Normal file
6
vendor/github.com/gorilla/rpc/v2/README.md
generated
vendored
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
rpc
|
||||||
|
===
|
||||||
|
|
||||||
|
gorilla/rpc is a foundation for RPC over HTTP services, providing access to the exported methods of an object through HTTP requests.
|
||||||
|
|
||||||
|
Read the full documentation here: http://www.gorillatoolkit.org/pkg/rpc
|
90
vendor/github.com/gorilla/rpc/v2/compression_selector.go
generated
vendored
Normal file
90
vendor/github.com/gorilla/rpc/v2/compression_selector.go
generated
vendored
Normal file
@ -0,0 +1,90 @@
|
|||||||
|
// Copyright 2009 The Go Authors. All rights reserved.
|
||||||
|
// Copyright 2012 The Gorilla Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package rpc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"compress/flate"
|
||||||
|
"compress/gzip"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"unicode"
|
||||||
|
)
|
||||||
|
|
||||||
|
// gzipWriter writes and closes the gzip writer.
|
||||||
|
type gzipWriter struct {
|
||||||
|
w *gzip.Writer
|
||||||
|
}
|
||||||
|
|
||||||
|
func (gw *gzipWriter) Write(p []byte) (n int, err error) {
|
||||||
|
defer gw.w.Close()
|
||||||
|
return gw.w.Write(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
// gzipEncoder implements the gzip compressed http encoder.
|
||||||
|
type gzipEncoder struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
func (enc *gzipEncoder) Encode(w http.ResponseWriter) io.Writer {
|
||||||
|
w.Header().Set("Content-Encoding", "gzip")
|
||||||
|
return &gzipWriter{gzip.NewWriter(w)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// flateWriter writes and closes the flate writer.
|
||||||
|
type flateWriter struct {
|
||||||
|
w *flate.Writer
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fw *flateWriter) Write(p []byte) (n int, err error) {
|
||||||
|
defer fw.w.Close()
|
||||||
|
return fw.w.Write(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
// flateEncoder implements the flate compressed http encoder.
|
||||||
|
type flateEncoder struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
func (enc *flateEncoder) Encode(w http.ResponseWriter) io.Writer {
|
||||||
|
fw, err := flate.NewWriter(w, flate.DefaultCompression)
|
||||||
|
if err != nil {
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Encoding", "deflate")
|
||||||
|
return &flateWriter{fw}
|
||||||
|
}
|
||||||
|
|
||||||
|
// CompressionSelector generates the compressed http encoder.
|
||||||
|
type CompressionSelector struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
// acceptedEnc returns the first compression type in "Accept-Encoding" header
|
||||||
|
// field of the request.
|
||||||
|
func acceptedEnc(req *http.Request) string {
|
||||||
|
encHeader := req.Header.Get("Accept-Encoding")
|
||||||
|
if encHeader == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
encTypes := strings.FieldsFunc(encHeader, func(r rune) bool {
|
||||||
|
return unicode.IsSpace(r) || r == ','
|
||||||
|
})
|
||||||
|
for _, enc := range encTypes {
|
||||||
|
if enc == "gzip" || enc == "deflate" {
|
||||||
|
return enc
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// Select method selects the correct compression encoder based on http HEADER.
|
||||||
|
func (_ *CompressionSelector) Select(r *http.Request) Encoder {
|
||||||
|
switch acceptedEnc(r) {
|
||||||
|
case "gzip":
|
||||||
|
return &gzipEncoder{}
|
||||||
|
case "flate":
|
||||||
|
return &flateEncoder{}
|
||||||
|
}
|
||||||
|
return DefaultEncoder
|
||||||
|
}
|
81
vendor/github.com/gorilla/rpc/v2/doc.go
generated
vendored
Normal file
81
vendor/github.com/gorilla/rpc/v2/doc.go
generated
vendored
Normal file
@ -0,0 +1,81 @@
|
|||||||
|
// Copyright 2009 The Go Authors. All rights reserved.
|
||||||
|
// Copyright 2012 The Gorilla Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
/*
|
||||||
|
Package gorilla/rpc is a foundation for RPC over HTTP services, providing
|
||||||
|
access to the exported methods of an object through HTTP requests.
|
||||||
|
|
||||||
|
This package derives from the standard net/rpc package but uses a single HTTP
|
||||||
|
request per call instead of persistent connections. Other differences
|
||||||
|
compared to net/rpc:
|
||||||
|
|
||||||
|
- Multiple codecs can be registered in the same server.
|
||||||
|
- A codec is chosen based on the "Content-Type" header from the request.
|
||||||
|
- Service methods also receive http.Request as parameter.
|
||||||
|
- This package can be used on Google App Engine.
|
||||||
|
|
||||||
|
Let's setup a server and register a codec and service:
|
||||||
|
|
||||||
|
import (
|
||||||
|
"http"
|
||||||
|
"github.com/gorilla/rpc/v2"
|
||||||
|
"github.com/gorilla/rpc/v2/json"
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
s := rpc.NewServer()
|
||||||
|
s.RegisterCodec(json.NewCodec(), "application/json")
|
||||||
|
s.RegisterService(new(HelloService), "")
|
||||||
|
http.Handle("/rpc", s)
|
||||||
|
}
|
||||||
|
|
||||||
|
This server handles requests to the "/rpc" path using a JSON codec.
|
||||||
|
A codec is tied to a content type. In the example above, the JSON codec is
|
||||||
|
registered to serve requests with "application/json" as the value for the
|
||||||
|
"Content-Type" header. If the header includes a charset definition, it is
|
||||||
|
ignored; only the media-type part is taken into account.
|
||||||
|
|
||||||
|
A service can be registered using a name. If the name is empty, like in the
|
||||||
|
example above, it will be inferred from the service type.
|
||||||
|
|
||||||
|
That's all about the server setup. Now let's define a simple service:
|
||||||
|
|
||||||
|
type HelloArgs struct {
|
||||||
|
Who string
|
||||||
|
}
|
||||||
|
|
||||||
|
type HelloReply struct {
|
||||||
|
Message string
|
||||||
|
}
|
||||||
|
|
||||||
|
type HelloService struct {}
|
||||||
|
|
||||||
|
func (h *HelloService) Say(r *http.Request, args *HelloArgs, reply *HelloReply) error {
|
||||||
|
reply.Message = "Hello, " + args.Who + "!"
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
The example above defines a service with a method "HelloService.Say" and
|
||||||
|
the arguments and reply related to that method.
|
||||||
|
|
||||||
|
The service must be exported (begin with an upper case letter) or local
|
||||||
|
(defined in the package registering the service).
|
||||||
|
|
||||||
|
When a service is registered, the server inspects the service methods
|
||||||
|
and make available the ones that follow these rules:
|
||||||
|
|
||||||
|
- The method name is exported.
|
||||||
|
- The method has three arguments: *http.Request, *args, *reply.
|
||||||
|
- All three arguments are pointers.
|
||||||
|
- The second and third arguments are exported or local.
|
||||||
|
- The method has return type error.
|
||||||
|
|
||||||
|
All other methods are ignored.
|
||||||
|
|
||||||
|
Gorilla has packages with common RPC codecs. Check out their documentation:
|
||||||
|
|
||||||
|
JSON: http://gorilla-web.appspot.com/pkg/rpc/json
|
||||||
|
*/
|
||||||
|
package rpc
|
43
vendor/github.com/gorilla/rpc/v2/encoder_selector.go
generated
vendored
Normal file
43
vendor/github.com/gorilla/rpc/v2/encoder_selector.go
generated
vendored
Normal file
@ -0,0 +1,43 @@
|
|||||||
|
// Copyright 2009 The Go Authors. All rights reserved.
|
||||||
|
// Copyright 2012 The Gorilla Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package rpc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Encoder interface contains the encoder for http response.
|
||||||
|
// Eg. gzip, flate compressions.
|
||||||
|
type Encoder interface {
|
||||||
|
Encode(w http.ResponseWriter) io.Writer
|
||||||
|
}
|
||||||
|
|
||||||
|
type encoder struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
func (_ *encoder) Encode(w http.ResponseWriter) io.Writer {
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
var DefaultEncoder = &encoder{}
|
||||||
|
|
||||||
|
// EncoderSelector interface provides a way to select encoder using the http
|
||||||
|
// request. Typically people can use this to check HEADER of the request and
|
||||||
|
// figure out client capabilities.
|
||||||
|
// Eg. "Accept-Encoding" tells about supported compressions.
|
||||||
|
type EncoderSelector interface {
|
||||||
|
Select(r *http.Request) Encoder
|
||||||
|
}
|
||||||
|
|
||||||
|
type encoderSelector struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
func (_ *encoderSelector) Select(_ *http.Request) Encoder {
|
||||||
|
return DefaultEncoder
|
||||||
|
}
|
||||||
|
|
||||||
|
var DefaultEncoderSelector = &encoderSelector{}
|
61
vendor/github.com/gorilla/rpc/v2/json/client.go
generated
vendored
Normal file
61
vendor/github.com/gorilla/rpc/v2/json/client.go
generated
vendored
Normal file
@ -0,0 +1,61 @@
|
|||||||
|
// Copyright 2009 The Go Authors. All rights reserved.
|
||||||
|
// Copyright 2012-2013 The Gorilla Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package json
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"math/rand"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ----------------------------------------------------------------------------
|
||||||
|
// Request and Response
|
||||||
|
// ----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// clientRequest represents a JSON-RPC request sent by a client.
|
||||||
|
type clientRequest struct {
|
||||||
|
// A String containing the name of the method to be invoked.
|
||||||
|
Method string `json:"method"`
|
||||||
|
// Object to pass as request parameter to the method.
|
||||||
|
Params [1]interface{} `json:"params"`
|
||||||
|
// The request id. This can be of any type. It is used to match the
|
||||||
|
// response with the request that it is replying to.
|
||||||
|
Id uint64 `json:"id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// clientResponse represents a JSON-RPC response returned to a client.
|
||||||
|
type clientResponse struct {
|
||||||
|
Result *json.RawMessage `json:"result"`
|
||||||
|
Error interface{} `json:"error"`
|
||||||
|
Id uint64 `json:"id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// EncodeClientRequest encodes parameters for a JSON-RPC client request.
|
||||||
|
func EncodeClientRequest(method string, args interface{}) ([]byte, error) {
|
||||||
|
c := &clientRequest{
|
||||||
|
Method: method,
|
||||||
|
Params: [1]interface{}{args},
|
||||||
|
Id: uint64(rand.Int63()),
|
||||||
|
}
|
||||||
|
return json.Marshal(c)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DecodeClientResponse decodes the response body of a client request into
|
||||||
|
// the interface reply.
|
||||||
|
func DecodeClientResponse(r io.Reader, reply interface{}) error {
|
||||||
|
var c clientResponse
|
||||||
|
if err := json.NewDecoder(r).Decode(&c); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if c.Error != nil {
|
||||||
|
return &Error{Data: c.Error}
|
||||||
|
}
|
||||||
|
if c.Result == nil {
|
||||||
|
return fmt.Errorf("Unexpected null result")
|
||||||
|
}
|
||||||
|
return json.Unmarshal(*c.Result, reply)
|
||||||
|
}
|
58
vendor/github.com/gorilla/rpc/v2/json/doc.go
generated
vendored
Normal file
58
vendor/github.com/gorilla/rpc/v2/json/doc.go
generated
vendored
Normal file
@ -0,0 +1,58 @@
|
|||||||
|
// Copyright 2009 The Go Authors. All rights reserved.
|
||||||
|
// Copyright 2012 The Gorilla Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
/*
|
||||||
|
Package gorilla/rpc/json provides a codec for JSON-RPC over HTTP services.
|
||||||
|
|
||||||
|
To register the codec in a RPC server:
|
||||||
|
|
||||||
|
import (
|
||||||
|
"http"
|
||||||
|
"github.com/gorilla/rpc/v2"
|
||||||
|
"github.com/gorilla/rpc/v2/json"
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
s := rpc.NewServer()
|
||||||
|
s.RegisterCodec(json.NewCodec(), "application/json")
|
||||||
|
// [...]
|
||||||
|
http.Handle("/rpc", s)
|
||||||
|
}
|
||||||
|
|
||||||
|
A codec is tied to a content type. In the example above, the server will use
|
||||||
|
the JSON codec for requests with "application/json" as the value for the
|
||||||
|
"Content-Type" header.
|
||||||
|
|
||||||
|
This package follows the JSON-RPC 1.0 specification:
|
||||||
|
|
||||||
|
http://json-rpc.org/wiki/specification
|
||||||
|
|
||||||
|
Request format is:
|
||||||
|
|
||||||
|
method:
|
||||||
|
The name of the method to be invoked, as a string in dotted notation
|
||||||
|
as in "Service.Method".
|
||||||
|
params:
|
||||||
|
An array with a single object to pass as argument to the method.
|
||||||
|
id:
|
||||||
|
The request id, a uint. It is used to match the response with the
|
||||||
|
request that it is replying to.
|
||||||
|
|
||||||
|
Response format is:
|
||||||
|
|
||||||
|
result:
|
||||||
|
The Object that was returned by the invoked method,
|
||||||
|
or null in case there was an error invoking the method.
|
||||||
|
error:
|
||||||
|
An Error object if there was an error invoking the method,
|
||||||
|
or null if there was no error.
|
||||||
|
id:
|
||||||
|
The same id as the request it is responding to.
|
||||||
|
|
||||||
|
Check the gorilla/rpc documentation for more details:
|
||||||
|
|
||||||
|
http://gorilla-web.appspot.com/pkg/rpc
|
||||||
|
*/
|
||||||
|
package json
|
132
vendor/github.com/gorilla/rpc/v2/json/json_test.go
generated
vendored
Normal file
132
vendor/github.com/gorilla/rpc/v2/json/json_test.go
generated
vendored
Normal file
@ -0,0 +1,132 @@
|
|||||||
|
// Copyright 2009 The Go Authors. All rights reserved.
|
||||||
|
// Copyright 2012 The Gorilla Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package json
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/gorilla/rpc/v2"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
ErrResponseError = errors.New("response error")
|
||||||
|
ErrResponseJsonError = &Error{Data: map[string]interface{}{
|
||||||
|
"stackstrace": map[string]interface{}{"0": "foo()"},
|
||||||
|
"error": "a message",
|
||||||
|
}}
|
||||||
|
)
|
||||||
|
|
||||||
|
type Service1Request struct {
|
||||||
|
A int
|
||||||
|
B int
|
||||||
|
}
|
||||||
|
|
||||||
|
type Service1Response struct {
|
||||||
|
Result int
|
||||||
|
}
|
||||||
|
|
||||||
|
type Service1 struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *Service1) Multiply(r *http.Request, req *Service1Request, res *Service1Response) error {
|
||||||
|
res.Result = req.A * req.B
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *Service1) ResponseError(r *http.Request, req *Service1Request, res *Service1Response) error {
|
||||||
|
return ErrResponseError
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *Service1) ResponseJsonError(r *http.Request, req *Service1Request, res *Service1Response) error {
|
||||||
|
return ErrResponseJsonError
|
||||||
|
}
|
||||||
|
|
||||||
|
func execute(t *testing.T, s *rpc.Server, method string, req, res interface{}) error {
|
||||||
|
if !s.HasMethod(method) {
|
||||||
|
t.Fatal("Expected to be registered:", method)
|
||||||
|
}
|
||||||
|
|
||||||
|
buf, _ := EncodeClientRequest(method, req)
|
||||||
|
body := bytes.NewBuffer(buf)
|
||||||
|
r, _ := http.NewRequest("POST", "http://localhost:8080/", body)
|
||||||
|
r.Header.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
s.ServeHTTP(w, r)
|
||||||
|
|
||||||
|
return DecodeClientResponse(w.Body, res)
|
||||||
|
}
|
||||||
|
|
||||||
|
func executeRaw(t *testing.T, s *rpc.Server, req json.RawMessage) (int, *bytes.Buffer) {
|
||||||
|
r, _ := http.NewRequest("POST", "http://localhost:8080/", bytes.NewBuffer(req))
|
||||||
|
r.Header.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
s.ServeHTTP(w, r)
|
||||||
|
|
||||||
|
return w.Code, w.Body
|
||||||
|
}
|
||||||
|
|
||||||
|
func field(name string, blob json.RawMessage) (v interface{}, ok bool) {
|
||||||
|
var obj map[string]interface{}
|
||||||
|
if err := json.Unmarshal(blob, &obj); err != nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
v, ok = obj[name]
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestService(t *testing.T) {
|
||||||
|
s := rpc.NewServer()
|
||||||
|
s.RegisterCodec(NewCodec(), "application/json")
|
||||||
|
s.RegisterService(new(Service1), "")
|
||||||
|
|
||||||
|
var res Service1Response
|
||||||
|
if err := execute(t, s, "Service1.Multiply", &Service1Request{4, 2}, &res); err != nil {
|
||||||
|
t.Error("Expected err to be nil, but got", err)
|
||||||
|
}
|
||||||
|
if res.Result != 8 {
|
||||||
|
t.Error("Expected res.Result to be 8, but got", res.Result)
|
||||||
|
}
|
||||||
|
if err := execute(t, s, "Service1.ResponseError", &Service1Request{4, 2}, &res); err == nil {
|
||||||
|
t.Errorf("Expected to get %q, but got nil", ErrResponseError)
|
||||||
|
} else if err.Error() != ErrResponseError.Error() {
|
||||||
|
t.Errorf("Expected to get %q, but got %q", ErrResponseError, err)
|
||||||
|
}
|
||||||
|
if code, res := executeRaw(t, s, json.RawMessage(`{"method":"Service1.Multiply","params":null,"id":5}`)); code != 400 {
|
||||||
|
t.Error("Expected response code to be 400, but got", code)
|
||||||
|
} else if v, ok := field("result", res.Bytes()); !ok || v != nil {
|
||||||
|
t.Errorf("Expected ok to be true and v to be nil, but got %v and %v", ok, v)
|
||||||
|
}
|
||||||
|
if err := execute(t, s, "Service1.ResponseJsonError", &Service1Request{4, 2}, &res); err == nil {
|
||||||
|
t.Errorf("Expected to get %q, but got nil", ErrResponseError)
|
||||||
|
} else if jsonErr, ok := err.(*Error); !ok {
|
||||||
|
t.Error("Expected err to be of a *json.Error type")
|
||||||
|
} else if !reflect.DeepEqual(jsonErr.Data, ErrResponseJsonError.Data) {
|
||||||
|
t.Errorf("Expected jsonErr to be %q, but got %q", ErrResponseJsonError, jsonErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClientNullResult(t *testing.T) {
|
||||||
|
data := `{"jsonrpc": "2.0", "id": 8674665223082153551, "result": null}`
|
||||||
|
reader := bytes.NewReader([]byte(data))
|
||||||
|
|
||||||
|
var reply interface{}
|
||||||
|
|
||||||
|
err := DecodeClientResponse(reader, &reply)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err.Error() != "Unexpected null result" {
|
||||||
|
t.Fatalf("Unexpected error: %s", err)
|
||||||
|
}
|
||||||
|
}
|
155
vendor/github.com/gorilla/rpc/v2/json/server.go
generated
vendored
Normal file
155
vendor/github.com/gorilla/rpc/v2/json/server.go
generated
vendored
Normal file
@ -0,0 +1,155 @@
|
|||||||
|
// Copyright 2009 The Go Authors. All rights reserved.
|
||||||
|
// Copyright 2012 The Gorilla Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package json
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"github.com/gorilla/rpc/v2"
|
||||||
|
)
|
||||||
|
|
||||||
|
var null = json.RawMessage([]byte("null"))
|
||||||
|
|
||||||
|
// An Error is a wrapper for a JSON interface value. It can be used by either
|
||||||
|
// a service's handler func to write more complex JSON data to an error field
|
||||||
|
// of a server's response, or by a client to read it.
|
||||||
|
type Error struct {
|
||||||
|
Data interface{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Error) Error() string {
|
||||||
|
return fmt.Sprintf("%v", e.Data)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----------------------------------------------------------------------------
|
||||||
|
// Request and Response
|
||||||
|
// ----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// serverRequest represents a JSON-RPC request received by the server.
|
||||||
|
type serverRequest struct {
|
||||||
|
// A String containing the name of the method to be invoked.
|
||||||
|
Method string `json:"method"`
|
||||||
|
// An Array of objects to pass as arguments to the method.
|
||||||
|
Params *json.RawMessage `json:"params"`
|
||||||
|
// The request id. This can be of any type. It is used to match the
|
||||||
|
// response with the request that it is replying to.
|
||||||
|
Id *json.RawMessage `json:"id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// serverResponse represents a JSON-RPC response returned by the server.
|
||||||
|
type serverResponse struct {
|
||||||
|
// The Object that was returned by the invoked method. This must be null
|
||||||
|
// in case there was an error invoking the method.
|
||||||
|
Result interface{} `json:"result"`
|
||||||
|
// An Error object if there was an error invoking the method. It must be
|
||||||
|
// null if there was no error.
|
||||||
|
Error interface{} `json:"error"`
|
||||||
|
// This must be the same id as the request it is responding to.
|
||||||
|
Id *json.RawMessage `json:"id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----------------------------------------------------------------------------
|
||||||
|
// Codec
|
||||||
|
// ----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// NewCodec returns a new JSON Codec.
|
||||||
|
func NewCodec() *Codec {
|
||||||
|
return &Codec{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Codec creates a CodecRequest to process each request.
|
||||||
|
type Codec struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewRequest returns a CodecRequest.
|
||||||
|
func (c *Codec) NewRequest(r *http.Request) rpc.CodecRequest {
|
||||||
|
return newCodecRequest(r)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----------------------------------------------------------------------------
|
||||||
|
// CodecRequest
|
||||||
|
// ----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// newCodecRequest returns a new CodecRequest.
|
||||||
|
func newCodecRequest(r *http.Request) rpc.CodecRequest {
|
||||||
|
// Decode the request body and check if RPC method is valid.
|
||||||
|
req := new(serverRequest)
|
||||||
|
err := json.NewDecoder(r.Body).Decode(req)
|
||||||
|
r.Body.Close()
|
||||||
|
return &CodecRequest{request: req, err: err}
|
||||||
|
}
|
||||||
|
|
||||||
|
// CodecRequest decodes and encodes a single request.
|
||||||
|
type CodecRequest struct {
|
||||||
|
request *serverRequest
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Method returns the RPC method for the current request.
|
||||||
|
//
|
||||||
|
// The method uses a dotted notation as in "Service.Method".
|
||||||
|
func (c *CodecRequest) Method() (string, error) {
|
||||||
|
if c.err == nil {
|
||||||
|
return c.request.Method, nil
|
||||||
|
}
|
||||||
|
return "", c.err
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadRequest fills the request object for the RPC method.
|
||||||
|
func (c *CodecRequest) ReadRequest(args interface{}) error {
|
||||||
|
if c.err == nil {
|
||||||
|
if c.request.Params != nil {
|
||||||
|
// JSON params is array value. RPC params is struct.
|
||||||
|
// Unmarshal into array containing the request struct.
|
||||||
|
params := [1]interface{}{args}
|
||||||
|
c.err = json.Unmarshal(*c.request.Params, ¶ms)
|
||||||
|
} else {
|
||||||
|
c.err = errors.New("rpc: method request ill-formed: missing params field")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return c.err
|
||||||
|
}
|
||||||
|
|
||||||
|
// WriteResponse encodes the response and writes it to the ResponseWriter.
|
||||||
|
func (c *CodecRequest) WriteResponse(w http.ResponseWriter, reply interface{}) {
|
||||||
|
if c.request.Id != nil {
|
||||||
|
// Id is null for notifications and they don't have a response.
|
||||||
|
res := &serverResponse{
|
||||||
|
Result: reply,
|
||||||
|
Error: &null,
|
||||||
|
Id: c.request.Id,
|
||||||
|
}
|
||||||
|
c.writeServerResponse(w, 200, res)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *CodecRequest) WriteError(w http.ResponseWriter, _ int, err error) {
|
||||||
|
res := &serverResponse{
|
||||||
|
Result: &null,
|
||||||
|
Id: c.request.Id,
|
||||||
|
}
|
||||||
|
if jsonErr, ok := err.(*Error); ok {
|
||||||
|
res.Error = jsonErr.Data
|
||||||
|
} else {
|
||||||
|
res.Error = err.Error()
|
||||||
|
}
|
||||||
|
c.writeServerResponse(w, 400, res)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *CodecRequest) writeServerResponse(w http.ResponseWriter, status int, res *serverResponse) {
|
||||||
|
b, err := json.Marshal(res)
|
||||||
|
if err == nil {
|
||||||
|
w.WriteHeader(status)
|
||||||
|
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||||
|
w.Write(b)
|
||||||
|
} else {
|
||||||
|
// Not sure in which case will this happen. But seems harmless.
|
||||||
|
rpc.WriteError(w, 400, err.Error())
|
||||||
|
}
|
||||||
|
}
|
164
vendor/github.com/gorilla/rpc/v2/map.go
generated
vendored
Normal file
164
vendor/github.com/gorilla/rpc/v2/map.go
generated
vendored
Normal file
@ -0,0 +1,164 @@
|
|||||||
|
// Copyright 2009 The Go Authors. All rights reserved.
|
||||||
|
// Copyright 2012 The Gorilla Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package rpc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"unicode"
|
||||||
|
"unicode/utf8"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
// Precompute the reflect.Type of error and http.Request
|
||||||
|
typeOfError = reflect.TypeOf((*error)(nil)).Elem()
|
||||||
|
typeOfRequest = reflect.TypeOf((*http.Request)(nil)).Elem()
|
||||||
|
)
|
||||||
|
|
||||||
|
// ----------------------------------------------------------------------------
|
||||||
|
// service
|
||||||
|
// ----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
type service struct {
|
||||||
|
name string // name of service
|
||||||
|
rcvr reflect.Value // receiver of methods for the service
|
||||||
|
rcvrType reflect.Type // type of the receiver
|
||||||
|
methods map[string]*serviceMethod // registered methods
|
||||||
|
}
|
||||||
|
|
||||||
|
type serviceMethod struct {
|
||||||
|
method reflect.Method // receiver method
|
||||||
|
argsType reflect.Type // type of the request argument
|
||||||
|
replyType reflect.Type // type of the response argument
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----------------------------------------------------------------------------
|
||||||
|
// serviceMap
|
||||||
|
// ----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// serviceMap is a registry for services.
|
||||||
|
type serviceMap struct {
|
||||||
|
mutex sync.Mutex
|
||||||
|
services map[string]*service
|
||||||
|
}
|
||||||
|
|
||||||
|
// register adds a new service using reflection to extract its methods.
|
||||||
|
func (m *serviceMap) register(rcvr interface{}, name string) error {
|
||||||
|
// Setup service.
|
||||||
|
s := &service{
|
||||||
|
name: name,
|
||||||
|
rcvr: reflect.ValueOf(rcvr),
|
||||||
|
rcvrType: reflect.TypeOf(rcvr),
|
||||||
|
methods: make(map[string]*serviceMethod),
|
||||||
|
}
|
||||||
|
if name == "" {
|
||||||
|
s.name = reflect.Indirect(s.rcvr).Type().Name()
|
||||||
|
if !isExported(s.name) {
|
||||||
|
return fmt.Errorf("rpc: type %q is not exported", s.name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if s.name == "" {
|
||||||
|
return fmt.Errorf("rpc: no service name for type %q",
|
||||||
|
s.rcvrType.String())
|
||||||
|
}
|
||||||
|
// Setup methods.
|
||||||
|
for i := 0; i < s.rcvrType.NumMethod(); i++ {
|
||||||
|
method := s.rcvrType.Method(i)
|
||||||
|
mtype := method.Type
|
||||||
|
// Method must be exported.
|
||||||
|
if method.PkgPath != "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Method needs four ins: receiver, *http.Request, *args, *reply.
|
||||||
|
if mtype.NumIn() != 4 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// First argument must be a pointer and must be http.Request.
|
||||||
|
reqType := mtype.In(1)
|
||||||
|
if reqType.Kind() != reflect.Ptr || reqType.Elem() != typeOfRequest {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Second argument must be a pointer and must be exported.
|
||||||
|
args := mtype.In(2)
|
||||||
|
if args.Kind() != reflect.Ptr || !isExportedOrBuiltin(args) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Third argument must be a pointer and must be exported.
|
||||||
|
reply := mtype.In(3)
|
||||||
|
if reply.Kind() != reflect.Ptr || !isExportedOrBuiltin(reply) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Method needs one out: error.
|
||||||
|
if mtype.NumOut() != 1 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if returnType := mtype.Out(0); returnType != typeOfError {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
s.methods[method.Name] = &serviceMethod{
|
||||||
|
method: method,
|
||||||
|
argsType: args.Elem(),
|
||||||
|
replyType: reply.Elem(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(s.methods) == 0 {
|
||||||
|
return fmt.Errorf("rpc: %q has no exported methods of suitable type",
|
||||||
|
s.name)
|
||||||
|
}
|
||||||
|
// Add to the map.
|
||||||
|
m.mutex.Lock()
|
||||||
|
defer m.mutex.Unlock()
|
||||||
|
if m.services == nil {
|
||||||
|
m.services = make(map[string]*service)
|
||||||
|
} else if _, ok := m.services[s.name]; ok {
|
||||||
|
return fmt.Errorf("rpc: service already defined: %q", s.name)
|
||||||
|
}
|
||||||
|
m.services[s.name] = s
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// get returns a registered service given a method name.
|
||||||
|
//
|
||||||
|
// The method name uses a dotted notation as in "Service.Method".
|
||||||
|
func (m *serviceMap) get(method string) (*service, *serviceMethod, error) {
|
||||||
|
parts := strings.Split(method, ".")
|
||||||
|
if len(parts) != 2 {
|
||||||
|
err := fmt.Errorf("rpc: service/method request ill-formed: %q", method)
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
m.mutex.Lock()
|
||||||
|
service := m.services[parts[0]]
|
||||||
|
m.mutex.Unlock()
|
||||||
|
if service == nil {
|
||||||
|
err := fmt.Errorf("rpc: can't find service %q", method)
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
serviceMethod := service.methods[parts[1]]
|
||||||
|
if serviceMethod == nil {
|
||||||
|
err := fmt.Errorf("rpc: can't find method %q", method)
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
return service, serviceMethod, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// isExported returns true of a string is an exported (upper case) name.
|
||||||
|
func isExported(name string) bool {
|
||||||
|
rune, _ := utf8.DecodeRuneInString(name)
|
||||||
|
return unicode.IsUpper(rune)
|
||||||
|
}
|
||||||
|
|
||||||
|
// isExportedOrBuiltin returns true if a type is exported or a builtin.
|
||||||
|
func isExportedOrBuiltin(t reflect.Type) bool {
|
||||||
|
for t.Kind() == reflect.Ptr {
|
||||||
|
t = t.Elem()
|
||||||
|
}
|
||||||
|
// PkgPath will be non-empty even for an exported type,
|
||||||
|
// so we need to check the type name as well.
|
||||||
|
return isExported(t.Name()) || t.PkgPath() == ""
|
||||||
|
}
|
158
vendor/github.com/gorilla/rpc/v2/server.go
generated
vendored
Normal file
158
vendor/github.com/gorilla/rpc/v2/server.go
generated
vendored
Normal file
@ -0,0 +1,158 @@
|
|||||||
|
// Copyright 2009 The Go Authors. All rights reserved.
|
||||||
|
// Copyright 2012 The Gorilla Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package rpc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ----------------------------------------------------------------------------
|
||||||
|
// Codec
|
||||||
|
// ----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// Codec creates a CodecRequest to process each request.
|
||||||
|
type Codec interface {
|
||||||
|
NewRequest(*http.Request) CodecRequest
|
||||||
|
}
|
||||||
|
|
||||||
|
// CodecRequest decodes a request and encodes a response using a specific
|
||||||
|
// serialization scheme.
|
||||||
|
type CodecRequest interface {
|
||||||
|
// Reads the request and returns the RPC method name.
|
||||||
|
Method() (string, error)
|
||||||
|
// Reads the request filling the RPC method args.
|
||||||
|
ReadRequest(interface{}) error
|
||||||
|
// Writes the response using the RPC method reply.
|
||||||
|
WriteResponse(http.ResponseWriter, interface{})
|
||||||
|
// Writes an error produced by the server.
|
||||||
|
WriteError(w http.ResponseWriter, status int, err error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----------------------------------------------------------------------------
|
||||||
|
// Server
|
||||||
|
// ----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// NewServer returns a new RPC server.
|
||||||
|
func NewServer() *Server {
|
||||||
|
return &Server{
|
||||||
|
codecs: make(map[string]Codec),
|
||||||
|
services: new(serviceMap),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Server serves registered RPC services using registered codecs.
|
||||||
|
type Server struct {
|
||||||
|
codecs map[string]Codec
|
||||||
|
services *serviceMap
|
||||||
|
}
|
||||||
|
|
||||||
|
// RegisterCodec adds a new codec to the server.
|
||||||
|
//
|
||||||
|
// Codecs are defined to process a given serialization scheme, e.g., JSON or
|
||||||
|
// XML. A codec is chosen based on the "Content-Type" header from the request,
|
||||||
|
// excluding the charset definition.
|
||||||
|
func (s *Server) RegisterCodec(codec Codec, contentType string) {
|
||||||
|
s.codecs[strings.ToLower(contentType)] = codec
|
||||||
|
}
|
||||||
|
|
||||||
|
// RegisterService adds a new service to the server.
|
||||||
|
//
|
||||||
|
// The name parameter is optional: if empty it will be inferred from
|
||||||
|
// the receiver type name.
|
||||||
|
//
|
||||||
|
// Methods from the receiver will be extracted if these rules are satisfied:
|
||||||
|
//
|
||||||
|
// - The receiver is exported (begins with an upper case letter) or local
|
||||||
|
// (defined in the package registering the service).
|
||||||
|
// - The method name is exported.
|
||||||
|
// - The method has three arguments: *http.Request, *args, *reply.
|
||||||
|
// - All three arguments are pointers.
|
||||||
|
// - The second and third arguments are exported or local.
|
||||||
|
// - The method has return type error.
|
||||||
|
//
|
||||||
|
// All other methods are ignored.
|
||||||
|
func (s *Server) RegisterService(receiver interface{}, name string) error {
|
||||||
|
return s.services.register(receiver, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// HasMethod returns true if the given method is registered.
|
||||||
|
//
|
||||||
|
// The method uses a dotted notation as in "Service.Method".
|
||||||
|
func (s *Server) HasMethod(method string) bool {
|
||||||
|
if _, _, err := s.services.get(method); err == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// ServeHTTP
|
||||||
|
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method != "POST" {
|
||||||
|
WriteError(w, 405, "rpc: POST method required, received "+r.Method)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
contentType := r.Header.Get("Content-Type")
|
||||||
|
idx := strings.Index(contentType, ";")
|
||||||
|
if idx != -1 {
|
||||||
|
contentType = contentType[:idx]
|
||||||
|
}
|
||||||
|
codec := s.codecs[strings.ToLower(contentType)]
|
||||||
|
if codec == nil {
|
||||||
|
WriteError(w, 415, "rpc: unrecognized Content-Type: "+contentType)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Create a new codec request.
|
||||||
|
codecReq := codec.NewRequest(r)
|
||||||
|
// Get service method to be called.
|
||||||
|
method, errMethod := codecReq.Method()
|
||||||
|
if errMethod != nil {
|
||||||
|
codecReq.WriteError(w, 400, errMethod)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
serviceSpec, methodSpec, errGet := s.services.get(method)
|
||||||
|
if errGet != nil {
|
||||||
|
codecReq.WriteError(w, 400, errGet)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Decode the args.
|
||||||
|
args := reflect.New(methodSpec.argsType)
|
||||||
|
if errRead := codecReq.ReadRequest(args.Interface()); errRead != nil {
|
||||||
|
codecReq.WriteError(w, 400, errRead)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Call the service method.
|
||||||
|
reply := reflect.New(methodSpec.replyType)
|
||||||
|
errValue := methodSpec.method.Func.Call([]reflect.Value{
|
||||||
|
serviceSpec.rcvr,
|
||||||
|
reflect.ValueOf(r),
|
||||||
|
args,
|
||||||
|
reply,
|
||||||
|
})
|
||||||
|
// Cast the result to error if needed.
|
||||||
|
var errResult error
|
||||||
|
errInter := errValue[0].Interface()
|
||||||
|
if errInter != nil {
|
||||||
|
errResult = errInter.(error)
|
||||||
|
}
|
||||||
|
// Prevents Internet Explorer from MIME-sniffing a response away
|
||||||
|
// from the declared content-type
|
||||||
|
w.Header().Set("x-content-type-options", "nosniff")
|
||||||
|
// Encode the response.
|
||||||
|
if errResult == nil {
|
||||||
|
codecReq.WriteResponse(w, reply.Interface())
|
||||||
|
} else {
|
||||||
|
codecReq.WriteError(w, 400, errResult)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func WriteError(w http.ResponseWriter, status int, msg string) {
|
||||||
|
w.WriteHeader(status)
|
||||||
|
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||||
|
fmt.Fprint(w, msg)
|
||||||
|
}
|
54
vendor/github.com/gorilla/rpc/v2/server_test.go
generated
vendored
Normal file
54
vendor/github.com/gorilla/rpc/v2/server_test.go
generated
vendored
Normal file
@ -0,0 +1,54 @@
|
|||||||
|
// Copyright 2009 The Go Authors. All rights reserved.
|
||||||
|
// Copyright 2012 The Gorilla Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package rpc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Service1Request struct {
|
||||||
|
A int
|
||||||
|
B int
|
||||||
|
}
|
||||||
|
|
||||||
|
type Service1Response struct {
|
||||||
|
Result int
|
||||||
|
}
|
||||||
|
|
||||||
|
type Service1 struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *Service1) Multiply(r *http.Request, req *Service1Request, res *Service1Response) error {
|
||||||
|
res.Result = req.A * req.B
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type Service2 struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRegisterService(t *testing.T) {
|
||||||
|
var err error
|
||||||
|
s := NewServer()
|
||||||
|
service1 := new(Service1)
|
||||||
|
service2 := new(Service2)
|
||||||
|
|
||||||
|
// Inferred name.
|
||||||
|
err = s.RegisterService(service1, "")
|
||||||
|
if err != nil || !s.HasMethod("Service1.Multiply") {
|
||||||
|
t.Errorf("Expected to be registered: Service1.Multiply")
|
||||||
|
}
|
||||||
|
// Provided name.
|
||||||
|
err = s.RegisterService(service1, "Foo")
|
||||||
|
if err != nil || !s.HasMethod("Foo.Multiply") {
|
||||||
|
t.Errorf("Expected to be registered: Foo.Multiply")
|
||||||
|
}
|
||||||
|
// No methods.
|
||||||
|
err = s.RegisterService(service2, "")
|
||||||
|
if err == nil {
|
||||||
|
t.Errorf("Expected error on service2")
|
||||||
|
}
|
||||||
|
}
|
21
vendor/github.com/minio/minio-go/CONTRIBUTING.md
generated
vendored
Normal file
21
vendor/github.com/minio/minio-go/CONTRIBUTING.md
generated
vendored
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
|
||||||
|
### Developer Guidelines
|
||||||
|
|
||||||
|
``minio-go`` welcomes your contribution. To make the process as seamless as possible, we ask for the following:
|
||||||
|
|
||||||
|
* Go ahead and fork the project and make your changes. We encourage pull requests to discuss code changes.
|
||||||
|
- Fork it
|
||||||
|
- Create your feature branch (git checkout -b my-new-feature)
|
||||||
|
- Commit your changes (git commit -am 'Add some feature')
|
||||||
|
- Push to the branch (git push origin my-new-feature)
|
||||||
|
- Create new Pull Request
|
||||||
|
|
||||||
|
* When you're ready to create a pull request, be sure to:
|
||||||
|
- Have test cases for the new code. If you have questions about how to do it, please ask in your pull request.
|
||||||
|
- Run `go fmt`
|
||||||
|
- Squash your commits into a single commit. `git rebase -i`. It's okay to force update your pull request.
|
||||||
|
- Make sure `go test -race ./...` and `go build` completes.
|
||||||
|
|
||||||
|
* Read [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) article from Golang project
|
||||||
|
- `minio-go` project is strictly conformant with Golang style
|
||||||
|
- if you happen to observe offending code, please feel free to send a pull request
|
83
vendor/github.com/minio/minio-go/INSTALLGO.md
generated
vendored
Normal file
83
vendor/github.com/minio/minio-go/INSTALLGO.md
generated
vendored
Normal file
@ -0,0 +1,83 @@
|
|||||||
|
## Ubuntu (Kylin) 14.04
|
||||||
|
### Build Dependencies
|
||||||
|
This installation guide is based on Ubuntu 14.04+ on x86-64 platform.
|
||||||
|
|
||||||
|
##### Install Git, GCC
|
||||||
|
```sh
|
||||||
|
$ sudo apt-get install git build-essential
|
||||||
|
```
|
||||||
|
|
||||||
|
##### Install Go 1.5+
|
||||||
|
|
||||||
|
Download Go 1.5+ from [https://golang.org/dl/](https://golang.org/dl/).
|
||||||
|
|
||||||
|
```sh
|
||||||
|
$ wget https://storage.googleapis.com/golang/go1.5.1.linux-amd64.tar.gz
|
||||||
|
$ mkdir -p ${HOME}/bin/
|
||||||
|
$ mkdir -p ${HOME}/go/
|
||||||
|
$ tar -C ${HOME}/bin/ -xzf go1.5.1.linux-amd64.tar.gz
|
||||||
|
```
|
||||||
|
##### Setup GOROOT and GOPATH
|
||||||
|
|
||||||
|
Add the following exports to your ``~/.bashrc``. Environment variable GOROOT specifies the location of your golang binaries
|
||||||
|
and GOPATH specifies the location of your project workspace.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
export GOROOT=${HOME}/bin/go
|
||||||
|
export GOPATH=${HOME}/go
|
||||||
|
export PATH=$PATH:${HOME}/bin/go/bin:${GOPATH}/bin
|
||||||
|
```
|
||||||
|
```sh
|
||||||
|
$ source ~/.bashrc
|
||||||
|
```
|
||||||
|
|
||||||
|
##### Testing it all
|
||||||
|
```sh
|
||||||
|
$ go env
|
||||||
|
```
|
||||||
|
|
||||||
|
## OS X (Yosemite) 10.10
|
||||||
|
### Build Dependencies
|
||||||
|
This installation document assumes OS X Yosemite 10.10+ on x86-64 platform.
|
||||||
|
|
||||||
|
##### Install brew
|
||||||
|
```sh
|
||||||
|
$ ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
|
||||||
|
```
|
||||||
|
|
||||||
|
##### Install Git, Python
|
||||||
|
```sh
|
||||||
|
$ brew install git python
|
||||||
|
```
|
||||||
|
|
||||||
|
##### Install Go 1.5+
|
||||||
|
|
||||||
|
Install golang binaries using `brew`
|
||||||
|
|
||||||
|
```sh
|
||||||
|
$ brew install go
|
||||||
|
$ mkdir -p $HOME/go
|
||||||
|
```
|
||||||
|
|
||||||
|
##### Setup GOROOT and GOPATH
|
||||||
|
|
||||||
|
Add the following exports to your ``~/.bash_profile``. Environment variable GOROOT specifies the location of your golang binaries
|
||||||
|
and GOPATH specifies the location of your project workspace.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
export GOPATH=${HOME}/go
|
||||||
|
export GOVERSION=$(brew list go | head -n 1 | cut -d '/' -f 6)
|
||||||
|
export GOROOT=$(brew --prefix)/Cellar/go/${GOVERSION}/libexec
|
||||||
|
export PATH=$PATH:${GOPATH}/bin
|
||||||
|
```
|
||||||
|
|
||||||
|
##### Source the new enviornment
|
||||||
|
|
||||||
|
```sh
|
||||||
|
$ source ~/.bash_profile
|
||||||
|
```
|
||||||
|
|
||||||
|
##### Testing it all
|
||||||
|
```sh
|
||||||
|
$ go env
|
||||||
|
```
|
202
vendor/github.com/minio/minio-go/LICENSE
generated
vendored
Normal file
202
vendor/github.com/minio/minio-go/LICENSE
generated
vendored
Normal file
@ -0,0 +1,202 @@
|
|||||||
|
|
||||||
|
Apache License
|
||||||
|
Version 2.0, January 2004
|
||||||
|
http://www.apache.org/licenses/
|
||||||
|
|
||||||
|
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||||
|
|
||||||
|
1. Definitions.
|
||||||
|
|
||||||
|
"License" shall mean the terms and conditions for use, reproduction,
|
||||||
|
and distribution as defined by Sections 1 through 9 of this document.
|
||||||
|
|
||||||
|
"Licensor" shall mean the copyright owner or entity authorized by
|
||||||
|
the copyright owner that is granting the License.
|
||||||
|
|
||||||
|
"Legal Entity" shall mean the union of the acting entity and all
|
||||||
|
other entities that control, are controlled by, or are under common
|
||||||
|
control with that entity. For the purposes of this definition,
|
||||||
|
"control" means (i) the power, direct or indirect, to cause the
|
||||||
|
direction or management of such entity, whether by contract or
|
||||||
|
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||||
|
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||||
|
|
||||||
|
"You" (or "Your") shall mean an individual or Legal Entity
|
||||||
|
exercising permissions granted by this License.
|
||||||
|
|
||||||
|
"Source" form shall mean the preferred form for making modifications,
|
||||||
|
including but not limited to software source code, documentation
|
||||||
|
source, and configuration files.
|
||||||
|
|
||||||
|
"Object" form shall mean any form resulting from mechanical
|
||||||
|
transformation or translation of a Source form, including but
|
||||||
|
not limited to compiled object code, generated documentation,
|
||||||
|
and conversions to other media types.
|
||||||
|
|
||||||
|
"Work" shall mean the work of authorship, whether in Source or
|
||||||
|
Object form, made available under the License, as indicated by a
|
||||||
|
copyright notice that is included in or attached to the work
|
||||||
|
(an example is provided in the Appendix below).
|
||||||
|
|
||||||
|
"Derivative Works" shall mean any work, whether in Source or Object
|
||||||
|
form, that is based on (or derived from) the Work and for which the
|
||||||
|
editorial revisions, annotations, elaborations, or other modifications
|
||||||
|
represent, as a whole, an original work of authorship. For the purposes
|
||||||
|
of this License, Derivative Works shall not include works that remain
|
||||||
|
separable from, or merely link (or bind by name) to the interfaces of,
|
||||||
|
the Work and Derivative Works thereof.
|
||||||
|
|
||||||
|
"Contribution" shall mean any work of authorship, including
|
||||||
|
the original version of the Work and any modifications or additions
|
||||||
|
to that Work or Derivative Works thereof, that is intentionally
|
||||||
|
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||||
|
or by an individual or Legal Entity authorized to submit on behalf of
|
||||||
|
the copyright owner. For the purposes of this definition, "submitted"
|
||||||
|
means any form of electronic, verbal, or written communication sent
|
||||||
|
to the Licensor or its representatives, including but not limited to
|
||||||
|
communication on electronic mailing lists, source code control systems,
|
||||||
|
and issue tracking systems that are managed by, or on behalf of, the
|
||||||
|
Licensor for the purpose of discussing and improving the Work, but
|
||||||
|
excluding communication that is conspicuously marked or otherwise
|
||||||
|
designated in writing by the copyright owner as "Not a Contribution."
|
||||||
|
|
||||||
|
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||||
|
on behalf of whom a Contribution has been received by Licensor and
|
||||||
|
subsequently incorporated within the Work.
|
||||||
|
|
||||||
|
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
copyright license to reproduce, prepare Derivative Works of,
|
||||||
|
publicly display, publicly perform, sublicense, and distribute the
|
||||||
|
Work and such Derivative Works in Source or Object form.
|
||||||
|
|
||||||
|
3. Grant of Patent License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
(except as stated in this section) patent license to make, have made,
|
||||||
|
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||||
|
where such license applies only to those patent claims licensable
|
||||||
|
by such Contributor that are necessarily infringed by their
|
||||||
|
Contribution(s) alone or by combination of their Contribution(s)
|
||||||
|
with the Work to which such Contribution(s) was submitted. If You
|
||||||
|
institute patent litigation against any entity (including a
|
||||||
|
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||||
|
or a Contribution incorporated within the Work constitutes direct
|
||||||
|
or contributory patent infringement, then any patent licenses
|
||||||
|
granted to You under this License for that Work shall terminate
|
||||||
|
as of the date such litigation is filed.
|
||||||
|
|
||||||
|
4. Redistribution. You may reproduce and distribute copies of the
|
||||||
|
Work or Derivative Works thereof in any medium, with or without
|
||||||
|
modifications, and in Source or Object form, provided that You
|
||||||
|
meet the following conditions:
|
||||||
|
|
||||||
|
(a) You must give any other recipients of the Work or
|
||||||
|
Derivative Works a copy of this License; and
|
||||||
|
|
||||||
|
(b) You must cause any modified files to carry prominent notices
|
||||||
|
stating that You changed the files; and
|
||||||
|
|
||||||
|
(c) You must retain, in the Source form of any Derivative Works
|
||||||
|
that You distribute, all copyright, patent, trademark, and
|
||||||
|
attribution notices from the Source form of the Work,
|
||||||
|
excluding those notices that do not pertain to any part of
|
||||||
|
the Derivative Works; and
|
||||||
|
|
||||||
|
(d) If the Work includes a "NOTICE" text file as part of its
|
||||||
|
distribution, then any Derivative Works that You distribute must
|
||||||
|
include a readable copy of the attribution notices contained
|
||||||
|
within such NOTICE file, excluding those notices that do not
|
||||||
|
pertain to any part of the Derivative Works, in at least one
|
||||||
|
of the following places: within a NOTICE text file distributed
|
||||||
|
as part of the Derivative Works; within the Source form or
|
||||||
|
documentation, if provided along with the Derivative Works; or,
|
||||||
|
within a display generated by the Derivative Works, if and
|
||||||
|
wherever such third-party notices normally appear. The contents
|
||||||
|
of the NOTICE file are for informational purposes only and
|
||||||
|
do not modify the License. You may add Your own attribution
|
||||||
|
notices within Derivative Works that You distribute, alongside
|
||||||
|
or as an addendum to the NOTICE text from the Work, provided
|
||||||
|
that such additional attribution notices cannot be construed
|
||||||
|
as modifying the License.
|
||||||
|
|
||||||
|
You may add Your own copyright statement to Your modifications and
|
||||||
|
may provide additional or different license terms and conditions
|
||||||
|
for use, reproduction, or distribution of Your modifications, or
|
||||||
|
for any such Derivative Works as a whole, provided Your use,
|
||||||
|
reproduction, and distribution of the Work otherwise complies with
|
||||||
|
the conditions stated in this License.
|
||||||
|
|
||||||
|
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||||
|
any Contribution intentionally submitted for inclusion in the Work
|
||||||
|
by You to the Licensor shall be under the terms and conditions of
|
||||||
|
this License, without any additional terms or conditions.
|
||||||
|
Notwithstanding the above, nothing herein shall supersede or modify
|
||||||
|
the terms of any separate license agreement you may have executed
|
||||||
|
with Licensor regarding such Contributions.
|
||||||
|
|
||||||
|
6. Trademarks. This License does not grant permission to use the trade
|
||||||
|
names, trademarks, service marks, or product names of the Licensor,
|
||||||
|
except as required for reasonable and customary use in describing the
|
||||||
|
origin of the Work and reproducing the content of the NOTICE file.
|
||||||
|
|
||||||
|
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||||
|
agreed to in writing, Licensor provides the Work (and each
|
||||||
|
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
implied, including, without limitation, any warranties or conditions
|
||||||
|
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||||
|
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||||
|
appropriateness of using or redistributing the Work and assume any
|
||||||
|
risks associated with Your exercise of permissions under this License.
|
||||||
|
|
||||||
|
8. Limitation of Liability. In no event and under no legal theory,
|
||||||
|
whether in tort (including negligence), contract, or otherwise,
|
||||||
|
unless required by applicable law (such as deliberate and grossly
|
||||||
|
negligent acts) or agreed to in writing, shall any Contributor be
|
||||||
|
liable to You for damages, including any direct, indirect, special,
|
||||||
|
incidental, or consequential damages of any character arising as a
|
||||||
|
result of this License or out of the use or inability to use the
|
||||||
|
Work (including but not limited to damages for loss of goodwill,
|
||||||
|
work stoppage, computer failure or malfunction, or any and all
|
||||||
|
other commercial damages or losses), even if such Contributor
|
||||||
|
has been advised of the possibility of such damages.
|
||||||
|
|
||||||
|
9. Accepting Warranty or Additional Liability. While redistributing
|
||||||
|
the Work or Derivative Works thereof, You may choose to offer,
|
||||||
|
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||||
|
or other liability obligations and/or rights consistent with this
|
||||||
|
License. However, in accepting such obligations, You may act only
|
||||||
|
on Your own behalf and on Your sole responsibility, not on behalf
|
||||||
|
of any other Contributor, and only if You agree to indemnify,
|
||||||
|
defend, and hold each Contributor harmless for any liability
|
||||||
|
incurred by, or claims asserted against, such Contributor by reason
|
||||||
|
of your accepting any such warranty or additional liability.
|
||||||
|
|
||||||
|
END OF TERMS AND CONDITIONS
|
||||||
|
|
||||||
|
APPENDIX: How to apply the Apache License to your work.
|
||||||
|
|
||||||
|
To apply the Apache License to your work, attach the following
|
||||||
|
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||||
|
replaced with your own identifying information. (Don't include
|
||||||
|
the brackets!) The text should be enclosed in the appropriate
|
||||||
|
comment syntax for the file format. We also recommend that a
|
||||||
|
file or class name and description of purpose be included on the
|
||||||
|
same "printed page" as the copyright notice for easier
|
||||||
|
identification within third-party archives.
|
||||||
|
|
||||||
|
Copyright [yyyy] [name of copyright owner]
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
19
vendor/github.com/minio/minio-go/MAINTAINERS.md
generated
vendored
Normal file
19
vendor/github.com/minio/minio-go/MAINTAINERS.md
generated
vendored
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
# For maintainers only
|
||||||
|
|
||||||
|
## Responsibilities
|
||||||
|
|
||||||
|
Please go through this link [Maintainer Responsibility](https://gist.github.com/abperiasamy/f4d9b31d3186bbd26522)
|
||||||
|
|
||||||
|
### Making new releases
|
||||||
|
|
||||||
|
Edit `libraryVersion` constant in `api.go`.
|
||||||
|
|
||||||
|
```
|
||||||
|
$ grep libraryVersion api.go
|
||||||
|
libraryVersion = "0.3.0"
|
||||||
|
```
|
||||||
|
|
||||||
|
```
|
||||||
|
$ git tag 0.3.0
|
||||||
|
$ git push --tags
|
||||||
|
```
|
98
vendor/github.com/minio/minio-go/README.md
generated
vendored
Normal file
98
vendor/github.com/minio/minio-go/README.md
generated
vendored
Normal file
@ -0,0 +1,98 @@
|
|||||||
|
# Minio Go Library for Amazon S3 Compatible Cloud Storage [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/minio/minio?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
|
||||||
|
|
||||||
|
## Description
|
||||||
|
|
||||||
|
Minio Go library is a simple client library for S3 compatible cloud storage servers. Supports AWS Signature Version 4 and 2. AWS Signature Version 4 is chosen as default.
|
||||||
|
|
||||||
|
List of supported cloud storage providers.
|
||||||
|
|
||||||
|
- AWS Signature Version 4
|
||||||
|
- Amazon S3
|
||||||
|
- Minio
|
||||||
|
|
||||||
|
- AWS Signature Version 2
|
||||||
|
- Google Cloud Storage (Compatibility Mode)
|
||||||
|
- Openstack Swift + Swift3 middleware
|
||||||
|
- Ceph Object Gateway
|
||||||
|
- Riak CS
|
||||||
|
|
||||||
|
## Install
|
||||||
|
|
||||||
|
If you do not have a working Golang environment, please follow [Install Golang](./INSTALLGO.md).
|
||||||
|
|
||||||
|
```sh
|
||||||
|
$ go get github.com/minio/minio-go
|
||||||
|
```
|
||||||
|
|
||||||
|
## Example
|
||||||
|
|
||||||
|
### ListBuckets()
|
||||||
|
|
||||||
|
This example shows how to List your buckets.
|
||||||
|
|
||||||
|
```go
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
|
||||||
|
"github.com/minio/minio-go"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
// Requests are always secure (HTTPS) by default. Set insecure=true to enable insecure (HTTP) access.
|
||||||
|
// This boolean value is the last argument for New().
|
||||||
|
|
||||||
|
// New returns an Amazon S3 compatible client object. API copatibality (v2 or v4) is automatically
|
||||||
|
// determined based on the Endpoint value.
|
||||||
|
s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", false)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalln(err)
|
||||||
|
}
|
||||||
|
buckets, err := s3Client.ListBuckets()
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalln(err)
|
||||||
|
}
|
||||||
|
for _, bucket := range buckets {
|
||||||
|
log.Println(bucket)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Documentation
|
||||||
|
|
||||||
|
### Bucket Operations.
|
||||||
|
* [MakeBucket(bucketName, BucketACL, location) error](examples/s3/makebucket.go)
|
||||||
|
* [BucketExists(bucketName) error](examples/s3/bucketexists.go)
|
||||||
|
* [RemoveBucket(bucketName) error](examples/s3/removebucket.go)
|
||||||
|
* [GetBucketACL(bucketName) (BucketACL, error)](examples/s3/getbucketacl.go)
|
||||||
|
* [SetBucketACL(bucketName, BucketACL) error)](examples/s3/setbucketacl.go)
|
||||||
|
* [ListBuckets() []BucketInfo](examples/s3/listbuckets.go)
|
||||||
|
* [ListObjects(bucketName, objectPrefix, recursive, chan<- struct{}) <-chan ObjectInfo](examples/s3/listobjects.go)
|
||||||
|
* [ListIncompleteUploads(bucketName, prefix, recursive, chan<- struct{}) <-chan ObjectMultipartInfo](examples/s3/listincompleteuploads.go)
|
||||||
|
|
||||||
|
### Object Operations.
|
||||||
|
* [PutObject(bucketName, objectName, io.Reader, contentType) error](examples/s3/putobject.go)
|
||||||
|
* [GetObject(bucketName, objectName) (*Object, error)](examples/s3/getobject.go)
|
||||||
|
* [StatObject(bucketName, objectName) (ObjectInfo, error)](examples/s3/statobject.go)
|
||||||
|
* [RemoveObject(bucketName, objectName) error](examples/s3/removeobject.go)
|
||||||
|
* [RemoveIncompleteUpload(bucketName, objectName) <-chan error](examples/s3/removeincompleteupload.go)
|
||||||
|
|
||||||
|
### File Object Operations.
|
||||||
|
* [FPutObject(bucketName, objectName, filePath, contentType) (size, error)](examples/s3/fputobject.go)
|
||||||
|
* [FGetObject(bucketName, objectName, filePath) error](examples/s3/fgetobject.go)
|
||||||
|
|
||||||
|
### Presigned Operations.
|
||||||
|
* [PresignedGetObject(bucketName, objectName, time.Duration) (string, error)](examples/s3/presignedgetobject.go)
|
||||||
|
* [PresignedPutObject(bucketName, objectName, time.Duration) (string, error)](examples/s3/presignedputobject.go)
|
||||||
|
* [PresignedPostPolicy(NewPostPolicy()) (map[string]string, error)](examples/s3/presignedpostpolicy.go)
|
||||||
|
|
||||||
|
### API Reference
|
||||||
|
|
||||||
|
[![GoDoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](http://godoc.org/github.com/minio/minio-go)
|
||||||
|
|
||||||
|
## Contribute
|
||||||
|
|
||||||
|
[Contributors Guide](./CONTRIBUTING.md)
|
||||||
|
|
||||||
|
[![Build Status](https://travis-ci.org/minio/minio-go.svg)](https://travis-ci.org/minio/minio-go) [![Build status](https://ci.appveyor.com/api/projects/status/1ep7n2resn6fk1w6?svg=true)](https://ci.appveyor.com/project/harshavardhana/minio-go)
|
76
vendor/github.com/minio/minio-go/api-definitions.go
generated
vendored
Normal file
76
vendor/github.com/minio/minio-go/api-definitions.go
generated
vendored
Normal file
@ -0,0 +1,76 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
// BucketInfo container for bucket metadata.
|
||||||
|
type BucketInfo struct {
|
||||||
|
// The name of the bucket.
|
||||||
|
Name string `json:"name"`
|
||||||
|
// Date the bucket was created.
|
||||||
|
CreationDate time.Time `json:"creationDate"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ObjectInfo container for object metadata.
|
||||||
|
type ObjectInfo struct {
|
||||||
|
// An ETag is optionally set to md5sum of an object. In case of multipart objects,
|
||||||
|
// ETag is of the form MD5SUM-N where MD5SUM is md5sum of all individual md5sums of
|
||||||
|
// each parts concatenated into one string.
|
||||||
|
ETag string `json:"etag"`
|
||||||
|
|
||||||
|
Key string `json:"name"` // Name of the object
|
||||||
|
LastModified time.Time `json:"lastModified"` // Date and time the object was last modified.
|
||||||
|
Size int64 `json:"size"` // Size in bytes of the object.
|
||||||
|
ContentType string `json:"contentType"` // A standard MIME type describing the format of the object data.
|
||||||
|
|
||||||
|
// Owner name.
|
||||||
|
Owner struct {
|
||||||
|
DisplayName string `json:"name"`
|
||||||
|
ID string `json:"id"`
|
||||||
|
} `json:"owner"`
|
||||||
|
|
||||||
|
// The class of storage used to store the object.
|
||||||
|
StorageClass string `json:"storageClass"`
|
||||||
|
|
||||||
|
// Error
|
||||||
|
Err error `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ObjectMultipartInfo container for multipart object metadata.
|
||||||
|
type ObjectMultipartInfo struct {
|
||||||
|
// Date and time at which the multipart upload was initiated.
|
||||||
|
Initiated time.Time `type:"timestamp" timestampFormat:"iso8601"`
|
||||||
|
|
||||||
|
Initiator initiator
|
||||||
|
Owner owner
|
||||||
|
|
||||||
|
// The type of storage to use for the object. Defaults to 'STANDARD'.
|
||||||
|
StorageClass string
|
||||||
|
|
||||||
|
// Key of the object for which the multipart upload was initiated.
|
||||||
|
Key string
|
||||||
|
|
||||||
|
// Size in bytes of the object.
|
||||||
|
Size int64
|
||||||
|
|
||||||
|
// Upload ID that identifies the multipart upload.
|
||||||
|
UploadID string `xml:"UploadId"`
|
||||||
|
|
||||||
|
// Error
|
||||||
|
Err error
|
||||||
|
}
|
236
vendor/github.com/minio/minio-go/api-error-response.go
generated
vendored
Normal file
236
vendor/github.com/minio/minio-go/api-error-response.go
generated
vendored
Normal file
@ -0,0 +1,236 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/xml"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
)
|
||||||
|
|
||||||
|
/* **** SAMPLE ERROR RESPONSE ****
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<Error>
|
||||||
|
<Code>AccessDenied</Code>
|
||||||
|
<Message>Access Denied</Message>
|
||||||
|
<BucketName>bucketName</BucketName>
|
||||||
|
<Key>objectName</Key>
|
||||||
|
<RequestId>F19772218238A85A</RequestId>
|
||||||
|
<HostId>GuWkjyviSiGHizehqpmsD1ndz5NClSP19DOT+s2mv7gXGQ8/X1lhbDGiIJEXpGFD</HostId>
|
||||||
|
</Error>
|
||||||
|
*/
|
||||||
|
|
||||||
|
// ErrorResponse - Is the typed error returned by all API operations.
|
||||||
|
type ErrorResponse struct {
|
||||||
|
XMLName xml.Name `xml:"Error" json:"-"`
|
||||||
|
Code string
|
||||||
|
Message string
|
||||||
|
BucketName string
|
||||||
|
Key string
|
||||||
|
RequestID string `xml:"RequestId"`
|
||||||
|
HostID string `xml:"HostId"`
|
||||||
|
|
||||||
|
// Region where the bucket is located. This header is returned
|
||||||
|
// only in HEAD bucket and ListObjects response.
|
||||||
|
AmzBucketRegion string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ToErrorResponse - Returns parsed ErrorResponse struct from body and
|
||||||
|
// http headers.
|
||||||
|
//
|
||||||
|
// For example:
|
||||||
|
//
|
||||||
|
// import s3 "github.com/minio/minio-go"
|
||||||
|
// ...
|
||||||
|
// ...
|
||||||
|
// reader, stat, err := s3.GetObject(...)
|
||||||
|
// if err != nil {
|
||||||
|
// resp := s3.ToErrorResponse(err)
|
||||||
|
// }
|
||||||
|
// ...
|
||||||
|
func ToErrorResponse(err error) ErrorResponse {
|
||||||
|
switch err := err.(type) {
|
||||||
|
case ErrorResponse:
|
||||||
|
return err
|
||||||
|
default:
|
||||||
|
return ErrorResponse{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Error - Returns HTTP error string
|
||||||
|
func (e ErrorResponse) Error() string {
|
||||||
|
return e.Message
|
||||||
|
}
|
||||||
|
|
||||||
|
// Common string for errors to report issue location in unexpected
|
||||||
|
// cases.
|
||||||
|
const (
|
||||||
|
reportIssue = "Please report this issue at https://github.com/minio/minio-go/issues."
|
||||||
|
)
|
||||||
|
|
||||||
|
// httpRespToErrorResponse returns a new encoded ErrorResponse
|
||||||
|
// structure as error.
|
||||||
|
func httpRespToErrorResponse(resp *http.Response, bucketName, objectName string) error {
|
||||||
|
if resp == nil {
|
||||||
|
msg := "Response is empty. " + reportIssue
|
||||||
|
return ErrInvalidArgument(msg)
|
||||||
|
}
|
||||||
|
var errResp ErrorResponse
|
||||||
|
err := xmlDecoder(resp.Body, &errResp)
|
||||||
|
// Xml decoding failed with no body, fall back to HTTP headers.
|
||||||
|
if err != nil {
|
||||||
|
switch resp.StatusCode {
|
||||||
|
case http.StatusNotFound:
|
||||||
|
if objectName == "" {
|
||||||
|
errResp = ErrorResponse{
|
||||||
|
Code: "NoSuchBucket",
|
||||||
|
Message: "The specified bucket does not exist.",
|
||||||
|
BucketName: bucketName,
|
||||||
|
RequestID: resp.Header.Get("x-amz-request-id"),
|
||||||
|
HostID: resp.Header.Get("x-amz-id-2"),
|
||||||
|
AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"),
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
errResp = ErrorResponse{
|
||||||
|
Code: "NoSuchKey",
|
||||||
|
Message: "The specified key does not exist.",
|
||||||
|
BucketName: bucketName,
|
||||||
|
Key: objectName,
|
||||||
|
RequestID: resp.Header.Get("x-amz-request-id"),
|
||||||
|
HostID: resp.Header.Get("x-amz-id-2"),
|
||||||
|
AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case http.StatusForbidden:
|
||||||
|
errResp = ErrorResponse{
|
||||||
|
Code: "AccessDenied",
|
||||||
|
Message: "Access Denied.",
|
||||||
|
BucketName: bucketName,
|
||||||
|
Key: objectName,
|
||||||
|
RequestID: resp.Header.Get("x-amz-request-id"),
|
||||||
|
HostID: resp.Header.Get("x-amz-id-2"),
|
||||||
|
AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"),
|
||||||
|
}
|
||||||
|
case http.StatusConflict:
|
||||||
|
errResp = ErrorResponse{
|
||||||
|
Code: "Conflict",
|
||||||
|
Message: "Bucket not empty.",
|
||||||
|
BucketName: bucketName,
|
||||||
|
RequestID: resp.Header.Get("x-amz-request-id"),
|
||||||
|
HostID: resp.Header.Get("x-amz-id-2"),
|
||||||
|
AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"),
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
errResp = ErrorResponse{
|
||||||
|
Code: resp.Status,
|
||||||
|
Message: resp.Status,
|
||||||
|
BucketName: bucketName,
|
||||||
|
RequestID: resp.Header.Get("x-amz-request-id"),
|
||||||
|
HostID: resp.Header.Get("x-amz-id-2"),
|
||||||
|
AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// AccessDenied without a signature mismatch code, usually means
|
||||||
|
// that the bucket policy has certain restrictions where some API
|
||||||
|
// operations are not allowed. Handle this case so that top level
|
||||||
|
// callers can interpret this easily and fall back if needed to a
|
||||||
|
// lower functionality call. Read each individual API specific
|
||||||
|
// code for such fallbacks.
|
||||||
|
if errResp.Code == "AccessDenied" && errResp.Message == "Access Denied" {
|
||||||
|
errResp.Code = "NotImplemented"
|
||||||
|
errResp.Message = "Operation is not allowed according to your bucket policy."
|
||||||
|
}
|
||||||
|
return errResp
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrEntityTooLarge - Input size is larger than supported maximum.
|
||||||
|
func ErrEntityTooLarge(totalSize, maxObjectSize int64, bucketName, objectName string) error {
|
||||||
|
msg := fmt.Sprintf("Your proposed upload size ‘%d’ exceeds the maximum allowed object size ‘%d’ for single PUT operation.", totalSize, maxObjectSize)
|
||||||
|
return ErrorResponse{
|
||||||
|
Code: "EntityTooLarge",
|
||||||
|
Message: msg,
|
||||||
|
BucketName: bucketName,
|
||||||
|
Key: objectName,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrEntityTooSmall - Input size is smaller than supported minimum.
|
||||||
|
func ErrEntityTooSmall(totalSize int64, bucketName, objectName string) error {
|
||||||
|
msg := fmt.Sprintf("Your proposed upload size ‘%d’ is below the minimum allowed object size '0B' for single PUT operation.", totalSize)
|
||||||
|
return ErrorResponse{
|
||||||
|
Code: "EntityTooLarge",
|
||||||
|
Message: msg,
|
||||||
|
BucketName: bucketName,
|
||||||
|
Key: objectName,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrUnexpectedEOF - Unexpected end of file reached.
|
||||||
|
func ErrUnexpectedEOF(totalRead, totalSize int64, bucketName, objectName string) error {
|
||||||
|
msg := fmt.Sprintf("Data read ‘%s’ is not equal to the size ‘%s’ of the input Reader.",
|
||||||
|
strconv.FormatInt(totalRead, 10), strconv.FormatInt(totalSize, 10))
|
||||||
|
return ErrorResponse{
|
||||||
|
Code: "UnexpectedEOF",
|
||||||
|
Message: msg,
|
||||||
|
BucketName: bucketName,
|
||||||
|
Key: objectName,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrInvalidBucketName - Invalid bucket name response.
|
||||||
|
func ErrInvalidBucketName(message string) error {
|
||||||
|
return ErrorResponse{
|
||||||
|
Code: "InvalidBucketName",
|
||||||
|
Message: message,
|
||||||
|
RequestID: "minio",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrInvalidObjectName - Invalid object name response.
|
||||||
|
func ErrInvalidObjectName(message string) error {
|
||||||
|
return ErrorResponse{
|
||||||
|
Code: "NoSuchKey",
|
||||||
|
Message: message,
|
||||||
|
RequestID: "minio",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrInvalidParts - Invalid number of parts.
|
||||||
|
func ErrInvalidParts(expectedParts, uploadedParts int) error {
|
||||||
|
msg := fmt.Sprintf("Unexpected number of parts found Want %d, Got %d", expectedParts, uploadedParts)
|
||||||
|
return ErrorResponse{
|
||||||
|
Code: "InvalidParts",
|
||||||
|
Message: msg,
|
||||||
|
RequestID: "minio",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrInvalidObjectPrefix - Invalid object prefix response is
|
||||||
|
// similar to object name response.
|
||||||
|
var ErrInvalidObjectPrefix = ErrInvalidObjectName
|
||||||
|
|
||||||
|
// ErrInvalidArgument - Invalid argument response.
|
||||||
|
func ErrInvalidArgument(message string) error {
|
||||||
|
return ErrorResponse{
|
||||||
|
Code: "InvalidArgument",
|
||||||
|
Message: message,
|
||||||
|
RequestID: "minio",
|
||||||
|
}
|
||||||
|
}
|
104
vendor/github.com/minio/minio-go/api-get-object-file.go
generated
vendored
Normal file
104
vendor/github.com/minio/minio-go/api-get-object-file.go
generated
vendored
Normal file
@ -0,0 +1,104 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FGetObject - download contents of an object to a local file.
|
||||||
|
func (c Client) FGetObject(bucketName, objectName, filePath string) error {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := isValidObjectName(objectName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify if destination already exists.
|
||||||
|
st, err := os.Stat(filePath)
|
||||||
|
if err == nil {
|
||||||
|
// If the destination exists and is a directory.
|
||||||
|
if st.IsDir() {
|
||||||
|
return ErrInvalidArgument("fileName is a directory.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Proceed if file does not exist. return for all other errors.
|
||||||
|
if err != nil {
|
||||||
|
if !os.IsNotExist(err) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract top level direcotry.
|
||||||
|
objectDir, _ := filepath.Split(filePath)
|
||||||
|
if objectDir != "" {
|
||||||
|
// Create any missing top level directories.
|
||||||
|
if err := os.MkdirAll(objectDir, 0700); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Gather md5sum.
|
||||||
|
objectStat, err := c.StatObject(bucketName, objectName)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write to a temporary file "fileName.part.minio" before saving.
|
||||||
|
filePartPath := filePath + objectStat.ETag + ".part.minio"
|
||||||
|
|
||||||
|
// If exists, open in append mode. If not create it as a part file.
|
||||||
|
filePart, err := os.OpenFile(filePartPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue Stat to get the current offset.
|
||||||
|
st, err = filePart.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seek to current position for incoming reader.
|
||||||
|
objectReader, objectStat, err := c.getObject(bucketName, objectName, st.Size(), 0)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write to the part file.
|
||||||
|
if _, err = io.CopyN(filePart, objectReader, objectStat.Size); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close the file before rename, this is specifically needed for Windows users.
|
||||||
|
if err = filePart.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Safely completed. Now commit by renaming to actual filename.
|
||||||
|
if err = os.Rename(filePartPath, filePath); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return.
|
||||||
|
return nil
|
||||||
|
}
|
537
vendor/github.com/minio/minio-go/api-get.go
generated
vendored
Normal file
537
vendor/github.com/minio/minio-go/api-get.go
generated
vendored
Normal file
@ -0,0 +1,537 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"math"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GetBucketACL - Get the permissions on an existing bucket.
|
||||||
|
//
|
||||||
|
// Returned values are:
|
||||||
|
//
|
||||||
|
// private - Owner gets full access.
|
||||||
|
// public-read - Owner gets full access, others get read access.
|
||||||
|
// public-read-write - Owner gets full access, others get full access
|
||||||
|
// too.
|
||||||
|
// authenticated-read - Owner gets full access, authenticated users
|
||||||
|
// get read access.
|
||||||
|
func (c Client) GetBucketACL(bucketName string) (BucketACL, error) {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set acl query.
|
||||||
|
urlValues := make(url.Values)
|
||||||
|
urlValues.Set("acl", "")
|
||||||
|
|
||||||
|
// Instantiate a new request.
|
||||||
|
req, err := c.newRequest("GET", requestMetadata{
|
||||||
|
bucketName: bucketName,
|
||||||
|
queryValues: urlValues,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initiate the request.
|
||||||
|
resp, err := c.do(req)
|
||||||
|
defer closeResponse(resp)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if resp != nil {
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return "", httpRespToErrorResponse(resp, bucketName, "")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decode access control policy.
|
||||||
|
policy := accessControlPolicy{}
|
||||||
|
err = xmlDecoder(resp.Body, &policy)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
// We need to avoid following de-serialization check for Google
|
||||||
|
// Cloud Storage. On Google Cloud Storage "private" canned ACL's
|
||||||
|
// policy do not have grant list. Treat it as a valid case, check
|
||||||
|
// for all other vendors.
|
||||||
|
if !isGoogleEndpoint(c.endpointURL) {
|
||||||
|
if policy.AccessControlList.Grant == nil {
|
||||||
|
errorResponse := ErrorResponse{
|
||||||
|
Code: "InternalError",
|
||||||
|
Message: "Access control Grant list is empty. " + reportIssue,
|
||||||
|
BucketName: bucketName,
|
||||||
|
RequestID: resp.Header.Get("x-amz-request-id"),
|
||||||
|
HostID: resp.Header.Get("x-amz-id-2"),
|
||||||
|
AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"),
|
||||||
|
}
|
||||||
|
return "", errorResponse
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Boolean cues to indentify right canned acls.
|
||||||
|
var publicRead, publicWrite, authenticatedRead bool
|
||||||
|
|
||||||
|
// Handle grants.
|
||||||
|
grants := policy.AccessControlList.Grant
|
||||||
|
for _, g := range grants {
|
||||||
|
if g.Grantee.URI == "" && g.Permission == "FULL_CONTROL" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" && g.Permission == "READ" {
|
||||||
|
authenticatedRead = true
|
||||||
|
break
|
||||||
|
} else if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "WRITE" {
|
||||||
|
publicWrite = true
|
||||||
|
} else if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "READ" {
|
||||||
|
publicRead = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify if acl is authenticated read.
|
||||||
|
if authenticatedRead {
|
||||||
|
return BucketACL("authenticated-read"), nil
|
||||||
|
}
|
||||||
|
// Verify if acl is private.
|
||||||
|
if !publicWrite && !publicRead {
|
||||||
|
return BucketACL("private"), nil
|
||||||
|
}
|
||||||
|
// Verify if acl is public-read.
|
||||||
|
if !publicWrite && publicRead {
|
||||||
|
return BucketACL("public-read"), nil
|
||||||
|
}
|
||||||
|
// Verify if acl is public-read-write.
|
||||||
|
if publicRead && publicWrite {
|
||||||
|
return BucketACL("public-read-write"), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return "", ErrorResponse{
|
||||||
|
Code: "NoSuchBucketPolicy",
|
||||||
|
Message: "The specified bucket does not have a bucket policy.",
|
||||||
|
BucketName: bucketName,
|
||||||
|
RequestID: "minio",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetObject - returns an seekable, readable object.
|
||||||
|
func (c Client) GetObject(bucketName, objectName string) (*Object, error) {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := isValidObjectName(objectName); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Send an explicit info to get the actual object size.
|
||||||
|
objectInfo, err := c.StatObject(bucketName, objectName)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create request channel.
|
||||||
|
reqCh := make(chan readRequest)
|
||||||
|
// Create response channel.
|
||||||
|
resCh := make(chan readResponse)
|
||||||
|
// Create done channel.
|
||||||
|
doneCh := make(chan struct{})
|
||||||
|
|
||||||
|
// This routine feeds partial object data as and when the caller
|
||||||
|
// reads.
|
||||||
|
go func() {
|
||||||
|
defer close(reqCh)
|
||||||
|
defer close(resCh)
|
||||||
|
|
||||||
|
// Loop through the incoming control messages and read data.
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
// When the done channel is closed exit our routine.
|
||||||
|
case <-doneCh:
|
||||||
|
return
|
||||||
|
// Request message.
|
||||||
|
case req := <-reqCh:
|
||||||
|
// Get shortest length.
|
||||||
|
// NOTE: Last remaining bytes are usually smaller than
|
||||||
|
// req.Buffer size. Use that as the final length.
|
||||||
|
length := math.Min(float64(len(req.Buffer)), float64(objectInfo.Size-req.Offset))
|
||||||
|
httpReader, _, err := c.getObject(bucketName, objectName, req.Offset, int64(length))
|
||||||
|
if err != nil {
|
||||||
|
resCh <- readResponse{
|
||||||
|
Error: err,
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
size, err := io.ReadFull(httpReader, req.Buffer)
|
||||||
|
if err == io.ErrUnexpectedEOF {
|
||||||
|
// If an EOF happens after reading some but not
|
||||||
|
// all the bytes ReadFull returns ErrUnexpectedEOF
|
||||||
|
err = io.EOF
|
||||||
|
}
|
||||||
|
resCh <- readResponse{
|
||||||
|
Size: int(size),
|
||||||
|
Error: err,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
// Return the readerAt backed by routine.
|
||||||
|
return newObject(reqCh, resCh, doneCh, objectInfo), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read response message container to reply back for the request.
|
||||||
|
type readResponse struct {
|
||||||
|
Size int
|
||||||
|
Error error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read request message container to communicate with internal
|
||||||
|
// go-routine.
|
||||||
|
type readRequest struct {
|
||||||
|
Buffer []byte
|
||||||
|
Offset int64 // readAt offset.
|
||||||
|
}
|
||||||
|
|
||||||
|
// Object represents an open object. It implements Read, ReadAt,
|
||||||
|
// Seeker, Close for a HTTP stream.
|
||||||
|
type Object struct {
|
||||||
|
// Mutex.
|
||||||
|
mutex *sync.Mutex
|
||||||
|
|
||||||
|
// User allocated and defined.
|
||||||
|
reqCh chan<- readRequest
|
||||||
|
resCh <-chan readResponse
|
||||||
|
doneCh chan<- struct{}
|
||||||
|
currOffset int64
|
||||||
|
objectInfo ObjectInfo
|
||||||
|
|
||||||
|
// Keeps track of closed call.
|
||||||
|
isClosed bool
|
||||||
|
|
||||||
|
// Previous error saved for future calls.
|
||||||
|
prevErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read reads up to len(p) bytes into p. It returns the number of
|
||||||
|
// bytes read (0 <= n <= len(p)) and any error encountered. Returns
|
||||||
|
// io.EOF upon end of file.
|
||||||
|
func (o *Object) Read(b []byte) (n int, err error) {
|
||||||
|
if o == nil {
|
||||||
|
return 0, ErrInvalidArgument("Object is nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Locking.
|
||||||
|
o.mutex.Lock()
|
||||||
|
defer o.mutex.Unlock()
|
||||||
|
|
||||||
|
// Previous prevErr is which was saved in previous operation.
|
||||||
|
if o.prevErr != nil || o.isClosed {
|
||||||
|
return 0, o.prevErr
|
||||||
|
}
|
||||||
|
|
||||||
|
// If current offset has reached Size limit, return EOF.
|
||||||
|
if o.currOffset >= o.objectInfo.Size {
|
||||||
|
return 0, io.EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
// Send current information over control channel to indicate we
|
||||||
|
// are ready.
|
||||||
|
reqMsg := readRequest{}
|
||||||
|
|
||||||
|
// Send the offset and pointer to the buffer over the channel.
|
||||||
|
reqMsg.Buffer = b
|
||||||
|
reqMsg.Offset = o.currOffset
|
||||||
|
|
||||||
|
// Send read request over the control channel.
|
||||||
|
o.reqCh <- reqMsg
|
||||||
|
|
||||||
|
// Get data over the response channel.
|
||||||
|
dataMsg := <-o.resCh
|
||||||
|
|
||||||
|
// Bytes read.
|
||||||
|
bytesRead := int64(dataMsg.Size)
|
||||||
|
|
||||||
|
// Update current offset.
|
||||||
|
o.currOffset += bytesRead
|
||||||
|
|
||||||
|
if dataMsg.Error == nil {
|
||||||
|
// If currOffset read is equal to objectSize
|
||||||
|
// We have reached end of file, we return io.EOF.
|
||||||
|
if o.currOffset >= o.objectInfo.Size {
|
||||||
|
return dataMsg.Size, io.EOF
|
||||||
|
}
|
||||||
|
return dataMsg.Size, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save any error.
|
||||||
|
o.prevErr = dataMsg.Error
|
||||||
|
return dataMsg.Size, dataMsg.Error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stat returns the ObjectInfo structure describing object.
|
||||||
|
func (o *Object) Stat() (ObjectInfo, error) {
|
||||||
|
if o == nil {
|
||||||
|
return ObjectInfo{}, ErrInvalidArgument("Object is nil")
|
||||||
|
}
|
||||||
|
// Locking.
|
||||||
|
o.mutex.Lock()
|
||||||
|
defer o.mutex.Unlock()
|
||||||
|
|
||||||
|
if o.prevErr != nil || o.isClosed {
|
||||||
|
return ObjectInfo{}, o.prevErr
|
||||||
|
}
|
||||||
|
|
||||||
|
return o.objectInfo, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadAt reads len(b) bytes from the File starting at byte offset
|
||||||
|
// off. It returns the number of bytes read and the error, if any.
|
||||||
|
// ReadAt always returns a non-nil error when n < len(b). At end of
|
||||||
|
// file, that error is io.EOF.
|
||||||
|
func (o *Object) ReadAt(b []byte, offset int64) (n int, err error) {
|
||||||
|
if o == nil {
|
||||||
|
return 0, ErrInvalidArgument("Object is nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Locking.
|
||||||
|
o.mutex.Lock()
|
||||||
|
defer o.mutex.Unlock()
|
||||||
|
|
||||||
|
// prevErr is which was saved in previous operation.
|
||||||
|
if o.prevErr != nil || o.isClosed {
|
||||||
|
return 0, o.prevErr
|
||||||
|
}
|
||||||
|
|
||||||
|
// If offset is negative and offset is greater than or equal to
|
||||||
|
// object size we return EOF.
|
||||||
|
if offset < 0 || offset >= o.objectInfo.Size {
|
||||||
|
return 0, io.EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
// Send current information over control channel to indicate we
|
||||||
|
// are ready.
|
||||||
|
reqMsg := readRequest{}
|
||||||
|
|
||||||
|
// Send the offset and pointer to the buffer over the channel.
|
||||||
|
reqMsg.Buffer = b
|
||||||
|
reqMsg.Offset = offset
|
||||||
|
|
||||||
|
// Send read request over the control channel.
|
||||||
|
o.reqCh <- reqMsg
|
||||||
|
|
||||||
|
// Get data over the response channel.
|
||||||
|
dataMsg := <-o.resCh
|
||||||
|
|
||||||
|
// Bytes read.
|
||||||
|
bytesRead := int64(dataMsg.Size)
|
||||||
|
|
||||||
|
if dataMsg.Error == nil {
|
||||||
|
// If offset+bytes read is equal to objectSize
|
||||||
|
// we have reached end of file, we return io.EOF.
|
||||||
|
if offset+bytesRead == o.objectInfo.Size {
|
||||||
|
return dataMsg.Size, io.EOF
|
||||||
|
}
|
||||||
|
return dataMsg.Size, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save any error.
|
||||||
|
o.prevErr = dataMsg.Error
|
||||||
|
return dataMsg.Size, dataMsg.Error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seek sets the offset for the next Read or Write to offset,
|
||||||
|
// interpreted according to whence: 0 means relative to the
|
||||||
|
// origin of the file, 1 means relative to the current offset,
|
||||||
|
// and 2 means relative to the end.
|
||||||
|
// Seek returns the new offset and an error, if any.
|
||||||
|
//
|
||||||
|
// Seeking to a negative offset is an error. Seeking to any positive
|
||||||
|
// offset is legal, subsequent io operations succeed until the
|
||||||
|
// underlying object is not closed.
|
||||||
|
func (o *Object) Seek(offset int64, whence int) (n int64, err error) {
|
||||||
|
if o == nil {
|
||||||
|
return 0, ErrInvalidArgument("Object is nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Locking.
|
||||||
|
o.mutex.Lock()
|
||||||
|
defer o.mutex.Unlock()
|
||||||
|
|
||||||
|
if o.prevErr != nil {
|
||||||
|
// At EOF seeking is legal, for any other errors we return.
|
||||||
|
if o.prevErr != io.EOF {
|
||||||
|
return 0, o.prevErr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Negative offset is valid for whence of '2'.
|
||||||
|
if offset < 0 && whence != 2 {
|
||||||
|
return 0, ErrInvalidArgument(fmt.Sprintf("Negative position not allowed for %d.", whence))
|
||||||
|
}
|
||||||
|
switch whence {
|
||||||
|
default:
|
||||||
|
return 0, ErrInvalidArgument(fmt.Sprintf("Invalid whence %d", whence))
|
||||||
|
case 0:
|
||||||
|
if offset > o.objectInfo.Size {
|
||||||
|
return 0, io.EOF
|
||||||
|
}
|
||||||
|
o.currOffset = offset
|
||||||
|
case 1:
|
||||||
|
if o.currOffset+offset > o.objectInfo.Size {
|
||||||
|
return 0, io.EOF
|
||||||
|
}
|
||||||
|
o.currOffset += offset
|
||||||
|
case 2:
|
||||||
|
// Seeking to positive offset is valid for whence '2', but
|
||||||
|
// since we are backing a Reader we have reached 'EOF' if
|
||||||
|
// offset is positive.
|
||||||
|
if offset > 0 {
|
||||||
|
return 0, io.EOF
|
||||||
|
}
|
||||||
|
// Seeking to negative position not allowed for whence.
|
||||||
|
if o.objectInfo.Size+offset < 0 {
|
||||||
|
return 0, ErrInvalidArgument(fmt.Sprintf("Seeking at negative offset not allowed for %d", whence))
|
||||||
|
}
|
||||||
|
o.currOffset += offset
|
||||||
|
}
|
||||||
|
// Return the effective offset.
|
||||||
|
return o.currOffset, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close - The behavior of Close after the first call returns error
|
||||||
|
// for subsequent Close() calls.
|
||||||
|
func (o *Object) Close() (err error) {
|
||||||
|
if o == nil {
|
||||||
|
return ErrInvalidArgument("Object is nil")
|
||||||
|
}
|
||||||
|
// Locking.
|
||||||
|
o.mutex.Lock()
|
||||||
|
defer o.mutex.Unlock()
|
||||||
|
|
||||||
|
// if already closed return an error.
|
||||||
|
if o.isClosed {
|
||||||
|
return o.prevErr
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close successfully.
|
||||||
|
close(o.doneCh)
|
||||||
|
|
||||||
|
// Save for future operations.
|
||||||
|
errMsg := "Object is already closed. Bad file descriptor."
|
||||||
|
o.prevErr = errors.New(errMsg)
|
||||||
|
// Save here that we closed done channel successfully.
|
||||||
|
o.isClosed = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// newObject instantiates a new *minio.Object*
|
||||||
|
func newObject(reqCh chan<- readRequest, resCh <-chan readResponse, doneCh chan<- struct{}, objectInfo ObjectInfo) *Object {
|
||||||
|
return &Object{
|
||||||
|
mutex: &sync.Mutex{},
|
||||||
|
reqCh: reqCh,
|
||||||
|
resCh: resCh,
|
||||||
|
doneCh: doneCh,
|
||||||
|
objectInfo: objectInfo,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// getObject - retrieve object from Object Storage.
|
||||||
|
//
|
||||||
|
// Additionally this function also takes range arguments to download the specified
|
||||||
|
// range bytes of an object. Setting offset and length = 0 will download the full object.
|
||||||
|
//
|
||||||
|
// For more information about the HTTP Range header.
|
||||||
|
// go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35.
|
||||||
|
func (c Client) getObject(bucketName, objectName string, offset, length int64) (io.ReadCloser, ObjectInfo, error) {
|
||||||
|
// Validate input arguments.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return nil, ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
if err := isValidObjectName(objectName); err != nil {
|
||||||
|
return nil, ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
customHeader := make(http.Header)
|
||||||
|
// Set ranges if length and offset are valid.
|
||||||
|
if length > 0 && offset >= 0 {
|
||||||
|
customHeader.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1))
|
||||||
|
} else if offset > 0 && length == 0 {
|
||||||
|
customHeader.Set("Range", fmt.Sprintf("bytes=%d-", offset))
|
||||||
|
} else if length < 0 && offset == 0 {
|
||||||
|
customHeader.Set("Range", fmt.Sprintf("bytes=%d", length))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Instantiate a new request.
|
||||||
|
req, err := c.newRequest("GET", requestMetadata{
|
||||||
|
bucketName: bucketName,
|
||||||
|
objectName: objectName,
|
||||||
|
customHeader: customHeader,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
// Execute the request.
|
||||||
|
resp, err := c.do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
if resp != nil {
|
||||||
|
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
||||||
|
return nil, ObjectInfo{}, httpRespToErrorResponse(resp, bucketName, objectName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Trim off the odd double quotes from ETag in the beginning and end.
|
||||||
|
md5sum := strings.TrimPrefix(resp.Header.Get("ETag"), "\"")
|
||||||
|
md5sum = strings.TrimSuffix(md5sum, "\"")
|
||||||
|
|
||||||
|
// Parse the date.
|
||||||
|
date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified"))
|
||||||
|
if err != nil {
|
||||||
|
msg := "Last-Modified time format not recognized. " + reportIssue
|
||||||
|
return nil, ObjectInfo{}, ErrorResponse{
|
||||||
|
Code: "InternalError",
|
||||||
|
Message: msg,
|
||||||
|
RequestID: resp.Header.Get("x-amz-request-id"),
|
||||||
|
HostID: resp.Header.Get("x-amz-id-2"),
|
||||||
|
AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Get content-type.
|
||||||
|
contentType := strings.TrimSpace(resp.Header.Get("Content-Type"))
|
||||||
|
if contentType == "" {
|
||||||
|
contentType = "application/octet-stream"
|
||||||
|
}
|
||||||
|
var objectStat ObjectInfo
|
||||||
|
objectStat.ETag = md5sum
|
||||||
|
objectStat.Key = objectName
|
||||||
|
objectStat.Size = resp.ContentLength
|
||||||
|
objectStat.LastModified = date
|
||||||
|
objectStat.ContentType = contentType
|
||||||
|
|
||||||
|
// do not close body here, caller will close
|
||||||
|
return resp.Body, objectStat, nil
|
||||||
|
}
|
539
vendor/github.com/minio/minio-go/api-list.go
generated
vendored
Normal file
539
vendor/github.com/minio/minio-go/api-list.go
generated
vendored
Normal file
@ -0,0 +1,539 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ListBuckets list all buckets owned by this authenticated user.
|
||||||
|
//
|
||||||
|
// This call requires explicit authentication, no anonymous requests are
|
||||||
|
// allowed for listing buckets.
|
||||||
|
//
|
||||||
|
// api := client.New(....)
|
||||||
|
// for message := range api.ListBuckets() {
|
||||||
|
// fmt.Println(message)
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
func (c Client) ListBuckets() ([]BucketInfo, error) {
|
||||||
|
// Instantiate a new request.
|
||||||
|
req, err := c.newRequest("GET", requestMetadata{})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Initiate the request.
|
||||||
|
resp, err := c.do(req)
|
||||||
|
defer closeResponse(resp)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if resp != nil {
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return nil, httpRespToErrorResponse(resp, "", "")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
listAllMyBucketsResult := listAllMyBucketsResult{}
|
||||||
|
err = xmlDecoder(resp.Body, &listAllMyBucketsResult)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return listAllMyBucketsResult.Buckets.Bucket, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListObjects - (List Objects) - List some objects or all recursively.
|
||||||
|
//
|
||||||
|
// ListObjects lists all objects matching the objectPrefix from
|
||||||
|
// the specified bucket. If recursion is enabled it would list
|
||||||
|
// all subdirectories and all its contents.
|
||||||
|
//
|
||||||
|
// Your input parameters are just bucketName, objectPrefix, recursive
|
||||||
|
// and a done channel for pro-actively closing the internal go
|
||||||
|
// routine. If you enable recursive as 'true' this function will
|
||||||
|
// return back all the objects in a given bucket name and object
|
||||||
|
// prefix.
|
||||||
|
//
|
||||||
|
// api := client.New(....)
|
||||||
|
// // Create a done channel.
|
||||||
|
// doneCh := make(chan struct{})
|
||||||
|
// defer close(doneCh)
|
||||||
|
// // Recurively list all objects in 'mytestbucket'
|
||||||
|
// recursive := true
|
||||||
|
// for message := range api.ListObjects("mytestbucket", "starthere", recursive, doneCh) {
|
||||||
|
// fmt.Println(message)
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo {
|
||||||
|
// Allocate new list objects channel.
|
||||||
|
objectStatCh := make(chan ObjectInfo, 1000)
|
||||||
|
// Default listing is delimited at "/"
|
||||||
|
delimiter := "/"
|
||||||
|
if recursive {
|
||||||
|
// If recursive we do not delimit.
|
||||||
|
delimiter = ""
|
||||||
|
}
|
||||||
|
// Validate bucket name.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
defer close(objectStatCh)
|
||||||
|
objectStatCh <- ObjectInfo{
|
||||||
|
Err: err,
|
||||||
|
}
|
||||||
|
return objectStatCh
|
||||||
|
}
|
||||||
|
// Validate incoming object prefix.
|
||||||
|
if err := isValidObjectPrefix(objectPrefix); err != nil {
|
||||||
|
defer close(objectStatCh)
|
||||||
|
objectStatCh <- ObjectInfo{
|
||||||
|
Err: err,
|
||||||
|
}
|
||||||
|
return objectStatCh
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initiate list objects goroutine here.
|
||||||
|
go func(objectStatCh chan<- ObjectInfo) {
|
||||||
|
defer close(objectStatCh)
|
||||||
|
// Save marker for next request.
|
||||||
|
var marker string
|
||||||
|
for {
|
||||||
|
// Get list of objects a maximum of 1000 per request.
|
||||||
|
result, err := c.listObjectsQuery(bucketName, objectPrefix, marker, delimiter, 1000)
|
||||||
|
if err != nil {
|
||||||
|
objectStatCh <- ObjectInfo{
|
||||||
|
Err: err,
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// If contents are available loop through and send over channel.
|
||||||
|
for _, object := range result.Contents {
|
||||||
|
// Save the marker.
|
||||||
|
marker = object.Key
|
||||||
|
select {
|
||||||
|
// Send object content.
|
||||||
|
case objectStatCh <- object:
|
||||||
|
// If receives done from the caller, return here.
|
||||||
|
case <-doneCh:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Send all common prefixes if any.
|
||||||
|
// NOTE: prefixes are only present if the request is delimited.
|
||||||
|
for _, obj := range result.CommonPrefixes {
|
||||||
|
object := ObjectInfo{}
|
||||||
|
object.Key = obj.Prefix
|
||||||
|
object.Size = 0
|
||||||
|
select {
|
||||||
|
// Send object prefixes.
|
||||||
|
case objectStatCh <- object:
|
||||||
|
// If receives done from the caller, return here.
|
||||||
|
case <-doneCh:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If next marker present, save it for next request.
|
||||||
|
if result.NextMarker != "" {
|
||||||
|
marker = result.NextMarker
|
||||||
|
}
|
||||||
|
|
||||||
|
// Listing ends result is not truncated, return right here.
|
||||||
|
if !result.IsTruncated {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}(objectStatCh)
|
||||||
|
return objectStatCh
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bucket Read Operations.
|
||||||
|
|
||||||
|
// listObjects - (List Objects) - List some or all (up to 1000) of the objects in a bucket.
|
||||||
|
//
|
||||||
|
// You can use the request parameters as selection criteria to return a subset of the objects in a bucket.
|
||||||
|
// request parameters :-
|
||||||
|
// ---------
|
||||||
|
// ?marker - Specifies the key to start with when listing objects in a bucket.
|
||||||
|
// ?delimiter - A delimiter is a character you use to group keys.
|
||||||
|
// ?prefix - Limits the response to keys that begin with the specified prefix.
|
||||||
|
// ?max-keys - Sets the maximum number of keys returned in the response body.
|
||||||
|
func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimiter string, maxkeys int) (listBucketResult, error) {
|
||||||
|
// Validate bucket name.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return listBucketResult{}, err
|
||||||
|
}
|
||||||
|
// Validate object prefix.
|
||||||
|
if err := isValidObjectPrefix(objectPrefix); err != nil {
|
||||||
|
return listBucketResult{}, err
|
||||||
|
}
|
||||||
|
// Get resources properly escaped and lined up before
|
||||||
|
// using them in http request.
|
||||||
|
urlValues := make(url.Values)
|
||||||
|
// Set object prefix.
|
||||||
|
if objectPrefix != "" {
|
||||||
|
urlValues.Set("prefix", urlEncodePath(objectPrefix))
|
||||||
|
}
|
||||||
|
// Set object marker.
|
||||||
|
if objectMarker != "" {
|
||||||
|
urlValues.Set("marker", urlEncodePath(objectMarker))
|
||||||
|
}
|
||||||
|
// Set delimiter.
|
||||||
|
if delimiter != "" {
|
||||||
|
urlValues.Set("delimiter", delimiter)
|
||||||
|
}
|
||||||
|
|
||||||
|
// maxkeys should default to 1000 or less.
|
||||||
|
if maxkeys == 0 || maxkeys > 1000 {
|
||||||
|
maxkeys = 1000
|
||||||
|
}
|
||||||
|
// Set max keys.
|
||||||
|
urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys))
|
||||||
|
|
||||||
|
// Initialize a new request.
|
||||||
|
req, err := c.newRequest("GET", requestMetadata{
|
||||||
|
bucketName: bucketName,
|
||||||
|
queryValues: urlValues,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return listBucketResult{}, err
|
||||||
|
}
|
||||||
|
// Execute list buckets.
|
||||||
|
resp, err := c.do(req)
|
||||||
|
defer closeResponse(resp)
|
||||||
|
if err != nil {
|
||||||
|
return listBucketResult{}, err
|
||||||
|
}
|
||||||
|
if resp != nil {
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return listBucketResult{}, httpRespToErrorResponse(resp, bucketName, "")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Decode listBuckets XML.
|
||||||
|
listBucketResult := listBucketResult{}
|
||||||
|
err = xmlDecoder(resp.Body, &listBucketResult)
|
||||||
|
if err != nil {
|
||||||
|
return listBucketResult, err
|
||||||
|
}
|
||||||
|
return listBucketResult, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListIncompleteUploads - List incompletely uploaded multipart objects.
|
||||||
|
//
|
||||||
|
// ListIncompleteUploads lists all incompleted objects matching the
|
||||||
|
// objectPrefix from the specified bucket. If recursion is enabled
|
||||||
|
// it would list all subdirectories and all its contents.
|
||||||
|
//
|
||||||
|
// Your input parameters are just bucketName, objectPrefix, recursive
|
||||||
|
// and a done channel to pro-actively close the internal go routine.
|
||||||
|
// If you enable recursive as 'true' this function will return back all
|
||||||
|
// the multipart objects in a given bucket name.
|
||||||
|
//
|
||||||
|
// api := client.New(....)
|
||||||
|
// // Create a done channel.
|
||||||
|
// doneCh := make(chan struct{})
|
||||||
|
// defer close(doneCh)
|
||||||
|
// // Recurively list all objects in 'mytestbucket'
|
||||||
|
// recursive := true
|
||||||
|
// for message := range api.ListIncompleteUploads("mytestbucket", "starthere", recursive) {
|
||||||
|
// fmt.Println(message)
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
func (c Client) ListIncompleteUploads(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo {
|
||||||
|
// Turn on size aggregation of individual parts.
|
||||||
|
isAggregateSize := true
|
||||||
|
return c.listIncompleteUploads(bucketName, objectPrefix, recursive, isAggregateSize, doneCh)
|
||||||
|
}
|
||||||
|
|
||||||
|
// listIncompleteUploads lists all incomplete uploads.
|
||||||
|
func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive, aggregateSize bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo {
|
||||||
|
// Allocate channel for multipart uploads.
|
||||||
|
objectMultipartStatCh := make(chan ObjectMultipartInfo, 1000)
|
||||||
|
// Delimiter is set to "/" by default.
|
||||||
|
delimiter := "/"
|
||||||
|
if recursive {
|
||||||
|
// If recursive do not delimit.
|
||||||
|
delimiter = ""
|
||||||
|
}
|
||||||
|
// Validate bucket name.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
defer close(objectMultipartStatCh)
|
||||||
|
objectMultipartStatCh <- ObjectMultipartInfo{
|
||||||
|
Err: err,
|
||||||
|
}
|
||||||
|
return objectMultipartStatCh
|
||||||
|
}
|
||||||
|
// Validate incoming object prefix.
|
||||||
|
if err := isValidObjectPrefix(objectPrefix); err != nil {
|
||||||
|
defer close(objectMultipartStatCh)
|
||||||
|
objectMultipartStatCh <- ObjectMultipartInfo{
|
||||||
|
Err: err,
|
||||||
|
}
|
||||||
|
return objectMultipartStatCh
|
||||||
|
}
|
||||||
|
go func(objectMultipartStatCh chan<- ObjectMultipartInfo) {
|
||||||
|
defer close(objectMultipartStatCh)
|
||||||
|
// object and upload ID marker for future requests.
|
||||||
|
var objectMarker string
|
||||||
|
var uploadIDMarker string
|
||||||
|
for {
|
||||||
|
// list all multipart uploads.
|
||||||
|
result, err := c.listMultipartUploadsQuery(bucketName, objectMarker, uploadIDMarker, objectPrefix, delimiter, 1000)
|
||||||
|
if err != nil {
|
||||||
|
objectMultipartStatCh <- ObjectMultipartInfo{
|
||||||
|
Err: err,
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Save objectMarker and uploadIDMarker for next request.
|
||||||
|
objectMarker = result.NextKeyMarker
|
||||||
|
uploadIDMarker = result.NextUploadIDMarker
|
||||||
|
// Send all multipart uploads.
|
||||||
|
for _, obj := range result.Uploads {
|
||||||
|
// Calculate total size of the uploaded parts if 'aggregateSize' is enabled.
|
||||||
|
if aggregateSize {
|
||||||
|
// Get total multipart size.
|
||||||
|
obj.Size, err = c.getTotalMultipartSize(bucketName, obj.Key, obj.UploadID)
|
||||||
|
if err != nil {
|
||||||
|
objectMultipartStatCh <- ObjectMultipartInfo{
|
||||||
|
Err: err,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
// Send individual uploads here.
|
||||||
|
case objectMultipartStatCh <- obj:
|
||||||
|
// If done channel return here.
|
||||||
|
case <-doneCh:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Send all common prefixes if any.
|
||||||
|
// NOTE: prefixes are only present if the request is delimited.
|
||||||
|
for _, obj := range result.CommonPrefixes {
|
||||||
|
object := ObjectMultipartInfo{}
|
||||||
|
object.Key = obj.Prefix
|
||||||
|
object.Size = 0
|
||||||
|
select {
|
||||||
|
// Send delimited prefixes here.
|
||||||
|
case objectMultipartStatCh <- object:
|
||||||
|
// If done channel return here.
|
||||||
|
case <-doneCh:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Listing ends if result not truncated, return right here.
|
||||||
|
if !result.IsTruncated {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}(objectMultipartStatCh)
|
||||||
|
// return.
|
||||||
|
return objectMultipartStatCh
|
||||||
|
}
|
||||||
|
|
||||||
|
// listMultipartUploads - (List Multipart Uploads).
|
||||||
|
// - Lists some or all (up to 1000) in-progress multipart uploads in a bucket.
|
||||||
|
//
|
||||||
|
// You can use the request parameters as selection criteria to return a subset of the uploads in a bucket.
|
||||||
|
// request parameters. :-
|
||||||
|
// ---------
|
||||||
|
// ?key-marker - Specifies the multipart upload after which listing should begin.
|
||||||
|
// ?upload-id-marker - Together with key-marker specifies the multipart upload after which listing should begin.
|
||||||
|
// ?delimiter - A delimiter is a character you use to group keys.
|
||||||
|
// ?prefix - Limits the response to keys that begin with the specified prefix.
|
||||||
|
// ?max-uploads - Sets the maximum number of multipart uploads returned in the response body.
|
||||||
|
func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, prefix, delimiter string, maxUploads int) (listMultipartUploadsResult, error) {
|
||||||
|
// Get resources properly escaped and lined up before using them in http request.
|
||||||
|
urlValues := make(url.Values)
|
||||||
|
// Set uploads.
|
||||||
|
urlValues.Set("uploads", "")
|
||||||
|
// Set object key marker.
|
||||||
|
if keyMarker != "" {
|
||||||
|
urlValues.Set("key-marker", urlEncodePath(keyMarker))
|
||||||
|
}
|
||||||
|
// Set upload id marker.
|
||||||
|
if uploadIDMarker != "" {
|
||||||
|
urlValues.Set("upload-id-marker", uploadIDMarker)
|
||||||
|
}
|
||||||
|
// Set prefix marker.
|
||||||
|
if prefix != "" {
|
||||||
|
urlValues.Set("prefix", urlEncodePath(prefix))
|
||||||
|
}
|
||||||
|
// Set delimiter.
|
||||||
|
if delimiter != "" {
|
||||||
|
urlValues.Set("delimiter", delimiter)
|
||||||
|
}
|
||||||
|
|
||||||
|
// maxUploads should be 1000 or less.
|
||||||
|
if maxUploads == 0 || maxUploads > 1000 {
|
||||||
|
maxUploads = 1000
|
||||||
|
}
|
||||||
|
// Set max-uploads.
|
||||||
|
urlValues.Set("max-uploads", fmt.Sprintf("%d", maxUploads))
|
||||||
|
|
||||||
|
// Instantiate a new request.
|
||||||
|
req, err := c.newRequest("GET", requestMetadata{
|
||||||
|
bucketName: bucketName,
|
||||||
|
queryValues: urlValues,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return listMultipartUploadsResult{}, err
|
||||||
|
}
|
||||||
|
// Execute list multipart uploads request.
|
||||||
|
resp, err := c.do(req)
|
||||||
|
defer closeResponse(resp)
|
||||||
|
if err != nil {
|
||||||
|
return listMultipartUploadsResult{}, err
|
||||||
|
}
|
||||||
|
if resp != nil {
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return listMultipartUploadsResult{}, httpRespToErrorResponse(resp, bucketName, "")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Decode response body.
|
||||||
|
listMultipartUploadsResult := listMultipartUploadsResult{}
|
||||||
|
err = xmlDecoder(resp.Body, &listMultipartUploadsResult)
|
||||||
|
if err != nil {
|
||||||
|
return listMultipartUploadsResult, err
|
||||||
|
}
|
||||||
|
return listMultipartUploadsResult, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// listObjectParts list all object parts recursively.
|
||||||
|
func (c Client) listObjectParts(bucketName, objectName, uploadID string) (partsInfo map[int]objectPart, err error) {
|
||||||
|
// Part number marker for the next batch of request.
|
||||||
|
var nextPartNumberMarker int
|
||||||
|
partsInfo = make(map[int]objectPart)
|
||||||
|
for {
|
||||||
|
// Get list of uploaded parts a maximum of 1000 per request.
|
||||||
|
listObjPartsResult, err := c.listObjectPartsQuery(bucketName, objectName, uploadID, nextPartNumberMarker, 1000)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Append to parts info.
|
||||||
|
for _, part := range listObjPartsResult.ObjectParts {
|
||||||
|
// Trim off the odd double quotes from ETag in the beginning and end.
|
||||||
|
part.ETag = strings.TrimPrefix(part.ETag, "\"")
|
||||||
|
part.ETag = strings.TrimSuffix(part.ETag, "\"")
|
||||||
|
partsInfo[part.PartNumber] = part
|
||||||
|
}
|
||||||
|
// Keep part number marker, for the next iteration.
|
||||||
|
nextPartNumberMarker = listObjPartsResult.NextPartNumberMarker
|
||||||
|
// Listing ends result is not truncated, return right here.
|
||||||
|
if !listObjPartsResult.IsTruncated {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return all the parts.
|
||||||
|
return partsInfo, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// findUploadID lists all incomplete uploads and finds the uploadID of the matching object name.
|
||||||
|
func (c Client) findUploadID(bucketName, objectName string) (uploadID string, err error) {
|
||||||
|
// Make list incomplete uploads recursive.
|
||||||
|
isRecursive := true
|
||||||
|
// Turn off size aggregation of individual parts, in this request.
|
||||||
|
isAggregateSize := false
|
||||||
|
// latestUpload to track the latest multipart info for objectName.
|
||||||
|
var latestUpload ObjectMultipartInfo
|
||||||
|
// Create done channel to cleanup the routine.
|
||||||
|
doneCh := make(chan struct{})
|
||||||
|
defer close(doneCh)
|
||||||
|
// List all incomplete uploads.
|
||||||
|
for mpUpload := range c.listIncompleteUploads(bucketName, objectName, isRecursive, isAggregateSize, doneCh) {
|
||||||
|
if mpUpload.Err != nil {
|
||||||
|
return "", mpUpload.Err
|
||||||
|
}
|
||||||
|
if objectName == mpUpload.Key {
|
||||||
|
if mpUpload.Initiated.Sub(latestUpload.Initiated) > 0 {
|
||||||
|
latestUpload = mpUpload
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Return the latest upload id.
|
||||||
|
return latestUpload.UploadID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// getTotalMultipartSize - calculate total uploaded size for the a given multipart object.
|
||||||
|
func (c Client) getTotalMultipartSize(bucketName, objectName, uploadID string) (size int64, err error) {
|
||||||
|
// Iterate over all parts and aggregate the size.
|
||||||
|
partsInfo, err := c.listObjectParts(bucketName, objectName, uploadID)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
for _, partInfo := range partsInfo {
|
||||||
|
size += partInfo.Size
|
||||||
|
}
|
||||||
|
return size, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// listObjectPartsQuery (List Parts query)
|
||||||
|
// - lists some or all (up to 1000) parts that have been uploaded
|
||||||
|
// for a specific multipart upload
|
||||||
|
//
|
||||||
|
// You can use the request parameters as selection criteria to return
|
||||||
|
// a subset of the uploads in a bucket, request parameters :-
|
||||||
|
// ---------
|
||||||
|
// ?part-number-marker - Specifies the part after which listing should
|
||||||
|
// begin.
|
||||||
|
// ?max-parts - Maximum parts to be listed per request.
|
||||||
|
func (c Client) listObjectPartsQuery(bucketName, objectName, uploadID string, partNumberMarker, maxParts int) (listObjectPartsResult, error) {
|
||||||
|
// Get resources properly escaped and lined up before using them in http request.
|
||||||
|
urlValues := make(url.Values)
|
||||||
|
// Set part number marker.
|
||||||
|
urlValues.Set("part-number-marker", fmt.Sprintf("%d", partNumberMarker))
|
||||||
|
// Set upload id.
|
||||||
|
urlValues.Set("uploadId", uploadID)
|
||||||
|
|
||||||
|
// maxParts should be 1000 or less.
|
||||||
|
if maxParts == 0 || maxParts > 1000 {
|
||||||
|
maxParts = 1000
|
||||||
|
}
|
||||||
|
// Set max parts.
|
||||||
|
urlValues.Set("max-parts", fmt.Sprintf("%d", maxParts))
|
||||||
|
|
||||||
|
req, err := c.newRequest("GET", requestMetadata{
|
||||||
|
bucketName: bucketName,
|
||||||
|
objectName: objectName,
|
||||||
|
queryValues: urlValues,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return listObjectPartsResult{}, err
|
||||||
|
}
|
||||||
|
// Exectue list object parts.
|
||||||
|
resp, err := c.do(req)
|
||||||
|
defer closeResponse(resp)
|
||||||
|
if err != nil {
|
||||||
|
return listObjectPartsResult{}, err
|
||||||
|
}
|
||||||
|
if resp != nil {
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return listObjectPartsResult{}, httpRespToErrorResponse(resp, bucketName, objectName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Decode list object parts XML.
|
||||||
|
listObjectPartsResult := listObjectPartsResult{}
|
||||||
|
err = xmlDecoder(resp.Body, &listObjectPartsResult)
|
||||||
|
if err != nil {
|
||||||
|
return listObjectPartsResult, err
|
||||||
|
}
|
||||||
|
return listObjectPartsResult, nil
|
||||||
|
}
|
144
vendor/github.com/minio/minio-go/api-presigned.go
generated
vendored
Normal file
144
vendor/github.com/minio/minio-go/api-presigned.go
generated
vendored
Normal file
@ -0,0 +1,144 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// presignURL - Returns a presigned URL for an input 'method'.
|
||||||
|
// Expires maximum is 7days - ie. 604800 and minimum is 1.
|
||||||
|
func (c Client) presignURL(method string, bucketName string, objectName string, expires time.Duration) (url string, err error) {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := isValidObjectName(objectName); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := isValidExpiry(expires); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
if method == "" {
|
||||||
|
return "", ErrInvalidArgument("method cannot be empty.")
|
||||||
|
}
|
||||||
|
|
||||||
|
expireSeconds := int64(expires / time.Second)
|
||||||
|
// Instantiate a new request.
|
||||||
|
// Since expires is set newRequest will presign the request.
|
||||||
|
req, err := c.newRequest(method, requestMetadata{
|
||||||
|
presignURL: true,
|
||||||
|
bucketName: bucketName,
|
||||||
|
objectName: objectName,
|
||||||
|
expires: expireSeconds,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return req.URL.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// PresignedGetObject - Returns a presigned URL to access an object without credentials.
|
||||||
|
// Expires maximum is 7days - ie. 604800 and minimum is 1.
|
||||||
|
func (c Client) PresignedGetObject(bucketName string, objectName string, expires time.Duration) (url string, err error) {
|
||||||
|
return c.presignURL("GET", bucketName, objectName, expires)
|
||||||
|
}
|
||||||
|
|
||||||
|
// PresignedPutObject - Returns a presigned URL to upload an object without credentials.
|
||||||
|
// Expires maximum is 7days - ie. 604800 and minimum is 1.
|
||||||
|
func (c Client) PresignedPutObject(bucketName string, objectName string, expires time.Duration) (url string, err error) {
|
||||||
|
return c.presignURL("PUT", bucketName, objectName, expires)
|
||||||
|
}
|
||||||
|
|
||||||
|
// PresignedPostPolicy - Returns POST form data to upload an object at a location.
|
||||||
|
func (c Client) PresignedPostPolicy(p *PostPolicy) (map[string]string, error) {
|
||||||
|
// Validate input arguments.
|
||||||
|
if p.expiration.IsZero() {
|
||||||
|
return nil, errors.New("Expiration time must be specified")
|
||||||
|
}
|
||||||
|
if _, ok := p.formData["key"]; !ok {
|
||||||
|
return nil, errors.New("object key must be specified")
|
||||||
|
}
|
||||||
|
if _, ok := p.formData["bucket"]; !ok {
|
||||||
|
return nil, errors.New("bucket name must be specified")
|
||||||
|
}
|
||||||
|
|
||||||
|
bucketName := p.formData["bucket"]
|
||||||
|
// Fetch the bucket location.
|
||||||
|
location, err := c.getBucketLocation(bucketName)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keep time.
|
||||||
|
t := time.Now().UTC()
|
||||||
|
// For signature version '2' handle here.
|
||||||
|
if c.signature.isV2() {
|
||||||
|
policyBase64 := p.base64()
|
||||||
|
p.formData["policy"] = policyBase64
|
||||||
|
// For Google endpoint set this value to be 'GoogleAccessId'.
|
||||||
|
if isGoogleEndpoint(c.endpointURL) {
|
||||||
|
p.formData["GoogleAccessId"] = c.accessKeyID
|
||||||
|
} else {
|
||||||
|
// For all other endpoints set this value to be 'AWSAccessKeyId'.
|
||||||
|
p.formData["AWSAccessKeyId"] = c.accessKeyID
|
||||||
|
}
|
||||||
|
// Sign the policy.
|
||||||
|
p.formData["signature"] = postPresignSignatureV2(policyBase64, c.secretAccessKey)
|
||||||
|
return p.formData, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add date policy.
|
||||||
|
if err = p.addNewPolicy(policyCondition{
|
||||||
|
matchType: "eq",
|
||||||
|
condition: "$x-amz-date",
|
||||||
|
value: t.Format(iso8601DateFormat),
|
||||||
|
}); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add algorithm policy.
|
||||||
|
if err = p.addNewPolicy(policyCondition{
|
||||||
|
matchType: "eq",
|
||||||
|
condition: "$x-amz-algorithm",
|
||||||
|
value: signV4Algorithm,
|
||||||
|
}); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add a credential policy.
|
||||||
|
credential := getCredential(c.accessKeyID, location, t)
|
||||||
|
if err = p.addNewPolicy(policyCondition{
|
||||||
|
matchType: "eq",
|
||||||
|
condition: "$x-amz-credential",
|
||||||
|
value: credential,
|
||||||
|
}); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get base64 encoded policy.
|
||||||
|
policyBase64 := p.base64()
|
||||||
|
// Fill in the form data.
|
||||||
|
p.formData["policy"] = policyBase64
|
||||||
|
p.formData["x-amz-algorithm"] = signV4Algorithm
|
||||||
|
p.formData["x-amz-credential"] = credential
|
||||||
|
p.formData["x-amz-date"] = t.Format(iso8601DateFormat)
|
||||||
|
p.formData["x-amz-signature"] = postPresignSignatureV4(policyBase64, t, c.secretAccessKey, location)
|
||||||
|
return p.formData, nil
|
||||||
|
}
|
214
vendor/github.com/minio/minio-go/api-put-bucket.go
generated
vendored
Normal file
214
vendor/github.com/minio/minio-go/api-put-bucket.go
generated
vendored
Normal file
@ -0,0 +1,214 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/xml"
|
||||||
|
"io/ioutil"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
)
|
||||||
|
|
||||||
|
/// Bucket operations
|
||||||
|
|
||||||
|
// MakeBucket makes a new bucket.
|
||||||
|
//
|
||||||
|
// Optional arguments are acl and location - by default all buckets are created
|
||||||
|
// with ``private`` acl and in US Standard region.
|
||||||
|
//
|
||||||
|
// ACL valid values - http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html
|
||||||
|
//
|
||||||
|
// private - owner gets full access [default].
|
||||||
|
// public-read - owner gets full access, all others get read access.
|
||||||
|
// public-read-write - owner gets full access, all others get full access too.
|
||||||
|
// authenticated-read - owner gets full access, authenticated users get read access.
|
||||||
|
//
|
||||||
|
// For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html
|
||||||
|
// For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations
|
||||||
|
func (c Client) MakeBucket(bucketName string, acl BucketACL, location string) error {
|
||||||
|
// Validate the input arguments.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !acl.isValidBucketACL() {
|
||||||
|
return ErrInvalidArgument("Unrecognized ACL " + acl.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// If location is empty, treat is a default region 'us-east-1'.
|
||||||
|
if location == "" {
|
||||||
|
location = "us-east-1"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Instantiate the request.
|
||||||
|
req, err := c.makeBucketRequest(bucketName, acl, location)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute the request.
|
||||||
|
resp, err := c.do(req)
|
||||||
|
defer closeResponse(resp)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if resp != nil {
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return httpRespToErrorResponse(resp, bucketName, "")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save the location into cache on a successfull makeBucket response.
|
||||||
|
c.bucketLocCache.Set(bucketName, location)
|
||||||
|
|
||||||
|
// Return.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// makeBucketRequest constructs request for makeBucket.
|
||||||
|
func (c Client) makeBucketRequest(bucketName string, acl BucketACL, location string) (*http.Request, error) {
|
||||||
|
// Validate input arguments.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if !acl.isValidBucketACL() {
|
||||||
|
return nil, ErrInvalidArgument("Unrecognized ACL " + acl.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set get bucket location always as path style.
|
||||||
|
targetURL := *c.endpointURL
|
||||||
|
if bucketName != "" {
|
||||||
|
// If endpoint supports virtual host style use that always.
|
||||||
|
// Currently only S3 and Google Cloud Storage would support this.
|
||||||
|
if isVirtualHostSupported(c.endpointURL, bucketName) {
|
||||||
|
targetURL.Host = bucketName + "." + c.endpointURL.Host
|
||||||
|
targetURL.Path = "/"
|
||||||
|
} else {
|
||||||
|
// If not fall back to using path style.
|
||||||
|
targetURL.Path = "/" + bucketName
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// get a new HTTP request for the method.
|
||||||
|
req, err := http.NewRequest("PUT", targetURL.String(), nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// by default bucket acl is set to private.
|
||||||
|
req.Header.Set("x-amz-acl", "private")
|
||||||
|
if acl != "" {
|
||||||
|
req.Header.Set("x-amz-acl", string(acl))
|
||||||
|
}
|
||||||
|
|
||||||
|
// set UserAgent for the request.
|
||||||
|
c.setUserAgent(req)
|
||||||
|
|
||||||
|
// set sha256 sum for signature calculation only with signature version '4'.
|
||||||
|
if c.signature.isV4() {
|
||||||
|
req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256([]byte{})))
|
||||||
|
}
|
||||||
|
|
||||||
|
// If location is not 'us-east-1' create bucket location config.
|
||||||
|
if location != "us-east-1" && location != "" {
|
||||||
|
createBucketConfig := createBucketConfiguration{}
|
||||||
|
createBucketConfig.Location = location
|
||||||
|
var createBucketConfigBytes []byte
|
||||||
|
createBucketConfigBytes, err = xml.Marshal(createBucketConfig)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
createBucketConfigBuffer := bytes.NewBuffer(createBucketConfigBytes)
|
||||||
|
req.Body = ioutil.NopCloser(createBucketConfigBuffer)
|
||||||
|
req.ContentLength = int64(createBucketConfigBuffer.Len())
|
||||||
|
if c.signature.isV4() {
|
||||||
|
req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256(createBucketConfigBuffer.Bytes())))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sign the request.
|
||||||
|
if c.signature.isV4() {
|
||||||
|
// Signature calculated for MakeBucket request should be for 'us-east-1',
|
||||||
|
// regardless of the bucket's location constraint.
|
||||||
|
req = signV4(*req, c.accessKeyID, c.secretAccessKey, "us-east-1")
|
||||||
|
} else if c.signature.isV2() {
|
||||||
|
req = signV2(*req, c.accessKeyID, c.secretAccessKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return signed request.
|
||||||
|
return req, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetBucketACL set the permissions on an existing bucket using access control lists (ACL).
|
||||||
|
//
|
||||||
|
// For example
|
||||||
|
//
|
||||||
|
// private - owner gets full access [default].
|
||||||
|
// public-read - owner gets full access, all others get read access.
|
||||||
|
// public-read-write - owner gets full access, all others get full access too.
|
||||||
|
// authenticated-read - owner gets full access, authenticated users get read access.
|
||||||
|
func (c Client) SetBucketACL(bucketName string, acl BucketACL) error {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !acl.isValidBucketACL() {
|
||||||
|
return ErrInvalidArgument("Unrecognized ACL " + acl.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set acl query.
|
||||||
|
urlValues := make(url.Values)
|
||||||
|
urlValues.Set("acl", "")
|
||||||
|
|
||||||
|
// Add misc headers.
|
||||||
|
customHeader := make(http.Header)
|
||||||
|
|
||||||
|
if acl != "" {
|
||||||
|
customHeader.Set("x-amz-acl", acl.String())
|
||||||
|
} else {
|
||||||
|
customHeader.Set("x-amz-acl", "private")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Instantiate a new request.
|
||||||
|
req, err := c.newRequest("PUT", requestMetadata{
|
||||||
|
bucketName: bucketName,
|
||||||
|
queryValues: urlValues,
|
||||||
|
customHeader: customHeader,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initiate the request.
|
||||||
|
resp, err := c.do(req)
|
||||||
|
defer closeResponse(resp)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if resp != nil {
|
||||||
|
// if error return.
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return httpRespToErrorResponse(resp, bucketName, "")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// return
|
||||||
|
return nil
|
||||||
|
}
|
240
vendor/github.com/minio/minio-go/api-put-object-common.go
generated
vendored
Normal file
240
vendor/github.com/minio/minio-go/api-put-object-common.go
generated
vendored
Normal file
@ -0,0 +1,240 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/md5"
|
||||||
|
"crypto/sha256"
|
||||||
|
"hash"
|
||||||
|
"io"
|
||||||
|
"math"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Verify if reader is *os.File
|
||||||
|
func isFile(reader io.Reader) (ok bool) {
|
||||||
|
_, ok = reader.(*os.File)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify if reader is *minio.Object
|
||||||
|
func isObject(reader io.Reader) (ok bool) {
|
||||||
|
_, ok = reader.(*Object)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify if reader is a generic ReaderAt
|
||||||
|
func isReadAt(reader io.Reader) (ok bool) {
|
||||||
|
_, ok = reader.(io.ReaderAt)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// shouldUploadPart - verify if part should be uploaded.
|
||||||
|
func shouldUploadPart(objPart objectPart, objectParts map[int]objectPart) bool {
|
||||||
|
// If part not found should upload the part.
|
||||||
|
uploadedPart, found := objectParts[objPart.PartNumber]
|
||||||
|
if !found {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
// if size mismatches should upload the part.
|
||||||
|
if objPart.Size != uploadedPart.Size {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
// if md5sum mismatches should upload the part.
|
||||||
|
if objPart.ETag == uploadedPart.ETag {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// optimalPartInfo - calculate the optimal part info for a given
|
||||||
|
// object size.
|
||||||
|
//
|
||||||
|
// NOTE: Assumption here is that for any object to be uploaded to any S3 compatible
|
||||||
|
// object storage it will have the following parameters as constants.
|
||||||
|
//
|
||||||
|
// maxPartsCount - 10000
|
||||||
|
// minPartSize - 5MiB
|
||||||
|
// maxMultipartPutObjectSize - 5TiB
|
||||||
|
//
|
||||||
|
func optimalPartInfo(objectSize int64) (totalPartsCount int, partSize int64, lastPartSize int64, err error) {
|
||||||
|
// object size is '-1' set it to 5TiB.
|
||||||
|
if objectSize == -1 {
|
||||||
|
objectSize = maxMultipartPutObjectSize
|
||||||
|
}
|
||||||
|
// object size is larger than supported maximum.
|
||||||
|
if objectSize > maxMultipartPutObjectSize {
|
||||||
|
err = ErrEntityTooLarge(objectSize, maxMultipartPutObjectSize, "", "")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Use floats for part size for all calculations to avoid
|
||||||
|
// overflows during float64 to int64 conversions.
|
||||||
|
partSizeFlt := math.Ceil(float64(objectSize / maxPartsCount))
|
||||||
|
partSizeFlt = math.Ceil(partSizeFlt/minPartSize) * minPartSize
|
||||||
|
// Total parts count.
|
||||||
|
totalPartsCount = int(math.Ceil(float64(objectSize) / partSizeFlt))
|
||||||
|
// Part size.
|
||||||
|
partSize = int64(partSizeFlt)
|
||||||
|
// Last part size.
|
||||||
|
lastPartSize = objectSize - int64(totalPartsCount-1)*partSize
|
||||||
|
return totalPartsCount, partSize, lastPartSize, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// hashCopyBuffer is identical to hashCopyN except that it stages
|
||||||
|
// through the provided buffer (if one is required) rather than
|
||||||
|
// allocating a temporary one. If buf is nil, one is allocated for 5MiB.
|
||||||
|
func (c Client) hashCopyBuffer(writer io.Writer, reader io.Reader, buf []byte) (md5Sum, sha256Sum []byte, size int64, err error) {
|
||||||
|
// MD5 and SHA256 hasher.
|
||||||
|
var hashMD5, hashSHA256 hash.Hash
|
||||||
|
// MD5 and SHA256 hasher.
|
||||||
|
hashMD5 = md5.New()
|
||||||
|
hashWriter := io.MultiWriter(writer, hashMD5)
|
||||||
|
if c.signature.isV4() {
|
||||||
|
hashSHA256 = sha256.New()
|
||||||
|
hashWriter = io.MultiWriter(writer, hashMD5, hashSHA256)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Allocate buf if not initialized.
|
||||||
|
if buf == nil {
|
||||||
|
buf = make([]byte, optimalReadBufferSize)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Using io.CopyBuffer to copy in large buffers, default buffer
|
||||||
|
// for io.Copy of 32KiB is too small.
|
||||||
|
size, err = io.CopyBuffer(hashWriter, reader, buf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Finalize md5 sum and sha256 sum.
|
||||||
|
md5Sum = hashMD5.Sum(nil)
|
||||||
|
if c.signature.isV4() {
|
||||||
|
sha256Sum = hashSHA256.Sum(nil)
|
||||||
|
}
|
||||||
|
return md5Sum, sha256Sum, size, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// hashCopyN - Calculates Md5sum and SHA256sum for up to partSize amount of bytes.
|
||||||
|
func (c Client) hashCopyN(writer io.Writer, reader io.Reader, partSize int64) (md5Sum, sha256Sum []byte, size int64, err error) {
|
||||||
|
// MD5 and SHA256 hasher.
|
||||||
|
var hashMD5, hashSHA256 hash.Hash
|
||||||
|
// MD5 and SHA256 hasher.
|
||||||
|
hashMD5 = md5.New()
|
||||||
|
hashWriter := io.MultiWriter(writer, hashMD5)
|
||||||
|
if c.signature.isV4() {
|
||||||
|
hashSHA256 = sha256.New()
|
||||||
|
hashWriter = io.MultiWriter(writer, hashMD5, hashSHA256)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Copies to input at writer.
|
||||||
|
size, err = io.CopyN(hashWriter, reader, partSize)
|
||||||
|
if err != nil {
|
||||||
|
// If not EOF return error right here.
|
||||||
|
if err != io.EOF {
|
||||||
|
return nil, nil, 0, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Finalize md5shum and sha256 sum.
|
||||||
|
md5Sum = hashMD5.Sum(nil)
|
||||||
|
if c.signature.isV4() {
|
||||||
|
sha256Sum = hashSHA256.Sum(nil)
|
||||||
|
}
|
||||||
|
return md5Sum, sha256Sum, size, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// getUploadID - fetch upload id if already present for an object name
|
||||||
|
// or initiate a new request to fetch a new upload id.
|
||||||
|
func (c Client) getUploadID(bucketName, objectName, contentType string) (uploadID string, isNew bool, err error) {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
if err := isValidObjectName(objectName); err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set content Type to default if empty string.
|
||||||
|
if contentType == "" {
|
||||||
|
contentType = "application/octet-stream"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find upload id for previous upload for an object.
|
||||||
|
uploadID, err = c.findUploadID(bucketName, objectName)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
if uploadID == "" {
|
||||||
|
// Initiate multipart upload for an object.
|
||||||
|
initMultipartUploadResult, err := c.initiateMultipartUpload(bucketName, objectName, contentType)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
// Save the new upload id.
|
||||||
|
uploadID = initMultipartUploadResult.UploadID
|
||||||
|
// Indicate that this is a new upload id.
|
||||||
|
isNew = true
|
||||||
|
}
|
||||||
|
return uploadID, isNew, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// computeHashBuffer - Calculates MD5 and SHA256 for an input read
|
||||||
|
// Seeker is identical to computeHash except that it stages
|
||||||
|
// through the provided buffer (if one is required) rather than
|
||||||
|
// allocating a temporary one. If buf is nil, it uses a temporary
|
||||||
|
// buffer.
|
||||||
|
func (c Client) computeHashBuffer(reader io.ReadSeeker, buf []byte) (md5Sum, sha256Sum []byte, size int64, err error) {
|
||||||
|
// MD5 and SHA256 hasher.
|
||||||
|
var hashMD5, hashSHA256 hash.Hash
|
||||||
|
// MD5 and SHA256 hasher.
|
||||||
|
hashMD5 = md5.New()
|
||||||
|
hashWriter := io.MultiWriter(hashMD5)
|
||||||
|
if c.signature.isV4() {
|
||||||
|
hashSHA256 = sha256.New()
|
||||||
|
hashWriter = io.MultiWriter(hashMD5, hashSHA256)
|
||||||
|
}
|
||||||
|
|
||||||
|
// If no buffer is provided, no need to allocate just use io.Copy.
|
||||||
|
if buf == nil {
|
||||||
|
size, err = io.Copy(hashWriter, reader)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, 0, err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
size, err = io.CopyBuffer(hashWriter, reader, buf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, 0, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seek back reader to the beginning location.
|
||||||
|
if _, err := reader.Seek(0, 0); err != nil {
|
||||||
|
return nil, nil, 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Finalize md5shum and sha256 sum.
|
||||||
|
md5Sum = hashMD5.Sum(nil)
|
||||||
|
if c.signature.isV4() {
|
||||||
|
sha256Sum = hashSHA256.Sum(nil)
|
||||||
|
}
|
||||||
|
return md5Sum, sha256Sum, size, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// computeHash - Calculates MD5 and SHA256 for an input read Seeker.
|
||||||
|
func (c Client) computeHash(reader io.ReadSeeker) (md5Sum, sha256Sum []byte, size int64, err error) {
|
||||||
|
return c.computeHashBuffer(reader, nil)
|
||||||
|
}
|
240
vendor/github.com/minio/minio-go/api-put-object-file.go
generated
vendored
Normal file
240
vendor/github.com/minio/minio-go/api-put-object-file.go
generated
vendored
Normal file
@ -0,0 +1,240 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FPutObject - Create an object in a bucket, with contents from file at filePath.
|
||||||
|
func (c Client) FPutObject(bucketName, objectName, filePath, contentType string) (n int64, err error) {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if err := isValidObjectName(objectName); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open the referenced file.
|
||||||
|
fileReader, err := os.Open(filePath)
|
||||||
|
// If any error fail quickly here.
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
defer fileReader.Close()
|
||||||
|
|
||||||
|
// Save the file stat.
|
||||||
|
fileStat, err := fileReader.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save the file size.
|
||||||
|
fileSize := fileStat.Size()
|
||||||
|
|
||||||
|
// Check for largest object size allowed.
|
||||||
|
if fileSize > int64(maxMultipartPutObjectSize) {
|
||||||
|
return 0, ErrEntityTooLarge(fileSize, maxMultipartPutObjectSize, bucketName, objectName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NOTE: Google Cloud Storage multipart Put is not compatible with Amazon S3 APIs.
|
||||||
|
// Current implementation will only upload a maximum of 5GiB to Google Cloud Storage servers.
|
||||||
|
if isGoogleEndpoint(c.endpointURL) {
|
||||||
|
if fileSize > int64(maxSinglePutObjectSize) {
|
||||||
|
return 0, ErrorResponse{
|
||||||
|
Code: "NotImplemented",
|
||||||
|
Message: fmt.Sprintf("Invalid Content-Length %d for file uploads to Google Cloud Storage.", fileSize),
|
||||||
|
Key: objectName,
|
||||||
|
BucketName: bucketName,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Do not compute MD5 for Google Cloud Storage. Uploads up to 5GiB in size.
|
||||||
|
return c.putObjectNoChecksum(bucketName, objectName, fileReader, fileSize, contentType, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NOTE: S3 doesn't allow anonymous multipart requests.
|
||||||
|
if isAmazonEndpoint(c.endpointURL) && c.anonymous {
|
||||||
|
if fileSize > int64(maxSinglePutObjectSize) {
|
||||||
|
return 0, ErrorResponse{
|
||||||
|
Code: "NotImplemented",
|
||||||
|
Message: fmt.Sprintf("For anonymous requests Content-Length cannot be %d.", fileSize),
|
||||||
|
Key: objectName,
|
||||||
|
BucketName: bucketName,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Do not compute MD5 for anonymous requests to Amazon
|
||||||
|
// S3. Uploads up to 5GiB in size.
|
||||||
|
return c.putObjectNoChecksum(bucketName, objectName, fileReader, fileSize, contentType, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Small object upload is initiated for uploads for input data size smaller than 5MiB.
|
||||||
|
if fileSize < minPartSize {
|
||||||
|
return c.putObjectSingle(bucketName, objectName, fileReader, fileSize, contentType, nil)
|
||||||
|
}
|
||||||
|
// Upload all large objects as multipart.
|
||||||
|
n, err = c.putObjectMultipartFromFile(bucketName, objectName, fileReader, fileSize, contentType, nil)
|
||||||
|
if err != nil {
|
||||||
|
errResp := ToErrorResponse(err)
|
||||||
|
// Verify if multipart functionality is not available, if not
|
||||||
|
// fall back to single PutObject operation.
|
||||||
|
if errResp.Code == "NotImplemented" {
|
||||||
|
// If size of file is greater than '5GiB' fail.
|
||||||
|
if fileSize > maxSinglePutObjectSize {
|
||||||
|
return 0, ErrEntityTooLarge(fileSize, maxSinglePutObjectSize, bucketName, objectName)
|
||||||
|
}
|
||||||
|
// Fall back to uploading as single PutObject operation.
|
||||||
|
return c.putObjectSingle(bucketName, objectName, fileReader, fileSize, contentType, nil)
|
||||||
|
}
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// putObjectMultipartFromFile - Creates object from contents of *os.File
|
||||||
|
//
|
||||||
|
// NOTE: This function is meant to be used for readers with local
|
||||||
|
// file as in *os.File. This function resumes by skipping all the
|
||||||
|
// necessary parts which were already uploaded by verifying them
|
||||||
|
// against MD5SUM of each individual parts. This function also
|
||||||
|
// effectively utilizes file system capabilities of reading from
|
||||||
|
// specific sections and not having to create temporary files.
|
||||||
|
func (c Client) putObjectMultipartFromFile(bucketName, objectName string, fileReader io.ReaderAt, fileSize int64, contentType string, progress io.Reader) (int64, error) {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if err := isValidObjectName(objectName); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get upload id for an object, initiates a new multipart request
|
||||||
|
// if it cannot find any previously partially uploaded object.
|
||||||
|
uploadID, isNew, err := c.getUploadID(bucketName, objectName, contentType)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Total data read and written to server. should be equal to 'size' at the end of the call.
|
||||||
|
var totalUploadedSize int64
|
||||||
|
|
||||||
|
// Complete multipart upload.
|
||||||
|
var completeMultipartUpload completeMultipartUpload
|
||||||
|
|
||||||
|
// A map of all uploaded parts.
|
||||||
|
var partsInfo = make(map[int]objectPart)
|
||||||
|
|
||||||
|
// If this session is a continuation of a previous session fetch all
|
||||||
|
// previously uploaded parts info.
|
||||||
|
if !isNew {
|
||||||
|
// Fetch previously upload parts and maximum part size.
|
||||||
|
partsInfo, err = c.listObjectParts(bucketName, objectName, uploadID)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate the optimal parts info for a given size.
|
||||||
|
totalPartsCount, partSize, _, err := optimalPartInfo(fileSize)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Part number always starts with '1'.
|
||||||
|
partNumber := 1
|
||||||
|
|
||||||
|
for partNumber <= totalPartsCount {
|
||||||
|
// Get a section reader on a particular offset.
|
||||||
|
sectionReader := io.NewSectionReader(fileReader, totalUploadedSize, partSize)
|
||||||
|
|
||||||
|
// Calculates MD5 and SHA256 sum for a section reader.
|
||||||
|
var md5Sum, sha256Sum []byte
|
||||||
|
var prtSize int64
|
||||||
|
md5Sum, sha256Sum, prtSize, err = c.computeHash(sectionReader)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var reader io.Reader
|
||||||
|
// Update progress reader appropriately to the latest offset
|
||||||
|
// as we read from the source.
|
||||||
|
reader = newHook(sectionReader, progress)
|
||||||
|
|
||||||
|
// Verify if part should be uploaded.
|
||||||
|
if shouldUploadPart(objectPart{
|
||||||
|
ETag: hex.EncodeToString(md5Sum),
|
||||||
|
PartNumber: partNumber,
|
||||||
|
Size: prtSize,
|
||||||
|
}, partsInfo) {
|
||||||
|
// Proceed to upload the part.
|
||||||
|
var objPart objectPart
|
||||||
|
objPart, err = c.uploadPart(bucketName, objectName, uploadID, ioutil.NopCloser(reader), partNumber,
|
||||||
|
md5Sum, sha256Sum, prtSize)
|
||||||
|
if err != nil {
|
||||||
|
return totalUploadedSize, err
|
||||||
|
}
|
||||||
|
// Save successfully uploaded part metadata.
|
||||||
|
partsInfo[partNumber] = objPart
|
||||||
|
} else {
|
||||||
|
// Update the progress reader for the skipped part.
|
||||||
|
if progress != nil {
|
||||||
|
if _, err = io.CopyN(ioutil.Discard, progress, prtSize); err != nil {
|
||||||
|
return totalUploadedSize, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save successfully uploaded size.
|
||||||
|
totalUploadedSize += prtSize
|
||||||
|
|
||||||
|
// Increment part number.
|
||||||
|
partNumber++
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify if we uploaded all data.
|
||||||
|
if totalUploadedSize != fileSize {
|
||||||
|
return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, fileSize, bucketName, objectName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Loop over uploaded parts to save them in a Parts array before completing the multipart request.
|
||||||
|
for _, part := range partsInfo {
|
||||||
|
var complPart completePart
|
||||||
|
complPart.ETag = part.ETag
|
||||||
|
complPart.PartNumber = part.PartNumber
|
||||||
|
completeMultipartUpload.Parts = append(completeMultipartUpload.Parts, complPart)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify if totalPartsCount is not equal to total list of parts.
|
||||||
|
if totalPartsCount != len(completeMultipartUpload.Parts) {
|
||||||
|
return totalUploadedSize, ErrInvalidParts(partNumber, len(completeMultipartUpload.Parts))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort all completed parts.
|
||||||
|
sort.Sort(completedParts(completeMultipartUpload.Parts))
|
||||||
|
_, err = c.completeMultipartUpload(bucketName, objectName, uploadID, completeMultipartUpload)
|
||||||
|
if err != nil {
|
||||||
|
return totalUploadedSize, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return final size.
|
||||||
|
return totalUploadedSize, nil
|
||||||
|
}
|
379
vendor/github.com/minio/minio-go/api-put-object-multipart.go
generated
vendored
Normal file
379
vendor/github.com/minio/minio-go/api-put-object-multipart.go
generated
vendored
Normal file
@ -0,0 +1,379 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/xml"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Comprehensive put object operation involving multipart resumable uploads.
|
||||||
|
//
|
||||||
|
// Following code handles these types of readers.
|
||||||
|
//
|
||||||
|
// - *os.File
|
||||||
|
// - *minio.Object
|
||||||
|
// - Any reader which has a method 'ReadAt()'
|
||||||
|
//
|
||||||
|
// If we exhaust all the known types, code proceeds to use stream as
|
||||||
|
// is where each part is re-downloaded, checksummed and verified
|
||||||
|
// before upload.
|
||||||
|
func (c Client) putObjectMultipart(bucketName, objectName string, reader io.Reader, size int64, contentType string, progress io.Reader) (n int64, err error) {
|
||||||
|
if size > 0 && size >= minPartSize {
|
||||||
|
// Verify if reader is *os.File, then use file system functionalities.
|
||||||
|
if isFile(reader) {
|
||||||
|
return c.putObjectMultipartFromFile(bucketName, objectName, reader.(*os.File), size, contentType, progress)
|
||||||
|
}
|
||||||
|
// Verify if reader is *minio.Object or io.ReaderAt.
|
||||||
|
// NOTE: Verification of object is kept for a specific purpose
|
||||||
|
// while it is going to be duck typed similar to io.ReaderAt.
|
||||||
|
// It is to indicate that *minio.Object implements io.ReaderAt.
|
||||||
|
// and such a functionality is used in the subsequent code
|
||||||
|
// path.
|
||||||
|
if isObject(reader) || isReadAt(reader) {
|
||||||
|
return c.putObjectMultipartFromReadAt(bucketName, objectName, reader.(io.ReaderAt), size, contentType, progress)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// For any other data size and reader type we do generic multipart
|
||||||
|
// approach by staging data in temporary files and uploading them.
|
||||||
|
return c.putObjectMultipartStream(bucketName, objectName, reader, size, contentType, progress)
|
||||||
|
}
|
||||||
|
|
||||||
|
// putObjectStream uploads files bigger than 5MiB, and also supports
|
||||||
|
// special case where size is unknown i.e '-1'.
|
||||||
|
func (c Client) putObjectMultipartStream(bucketName, objectName string, reader io.Reader, size int64, contentType string, progress io.Reader) (n int64, err error) {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if err := isValidObjectName(objectName); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Total data read and written to server. should be equal to 'size' at the end of the call.
|
||||||
|
var totalUploadedSize int64
|
||||||
|
|
||||||
|
// Complete multipart upload.
|
||||||
|
var complMultipartUpload completeMultipartUpload
|
||||||
|
|
||||||
|
// A map of all previously uploaded parts.
|
||||||
|
var partsInfo = make(map[int]objectPart)
|
||||||
|
|
||||||
|
// getUploadID for an object, initiates a new multipart request
|
||||||
|
// if it cannot find any previously partially uploaded object.
|
||||||
|
uploadID, isNew, err := c.getUploadID(bucketName, objectName, contentType)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// If This session is a continuation of a previous session fetch all
|
||||||
|
// previously uploaded parts info.
|
||||||
|
if !isNew {
|
||||||
|
// Fetch previously uploaded parts and maximum part size.
|
||||||
|
partsInfo, err = c.listObjectParts(bucketName, objectName, uploadID)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate the optimal parts info for a given size.
|
||||||
|
totalPartsCount, partSize, _, err := optimalPartInfo(size)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Part number always starts with '1'.
|
||||||
|
partNumber := 1
|
||||||
|
|
||||||
|
// Initialize a temporary buffer.
|
||||||
|
tmpBuffer := new(bytes.Buffer)
|
||||||
|
|
||||||
|
for partNumber <= totalPartsCount {
|
||||||
|
// Calculates MD5 and SHA256 sum while copying partSize bytes
|
||||||
|
// into tmpBuffer.
|
||||||
|
md5Sum, sha256Sum, prtSize, rErr := c.hashCopyN(tmpBuffer, reader, partSize)
|
||||||
|
if rErr != nil {
|
||||||
|
if rErr != io.EOF {
|
||||||
|
return 0, rErr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var reader io.Reader
|
||||||
|
// Update progress reader appropriately to the latest offset
|
||||||
|
// as we read from the source.
|
||||||
|
reader = newHook(tmpBuffer, progress)
|
||||||
|
|
||||||
|
// Verify if part should be uploaded.
|
||||||
|
if shouldUploadPart(objectPart{
|
||||||
|
ETag: hex.EncodeToString(md5Sum),
|
||||||
|
PartNumber: partNumber,
|
||||||
|
Size: prtSize,
|
||||||
|
}, partsInfo) {
|
||||||
|
// Proceed to upload the part.
|
||||||
|
var objPart objectPart
|
||||||
|
objPart, err = c.uploadPart(bucketName, objectName, uploadID, ioutil.NopCloser(reader), partNumber,
|
||||||
|
md5Sum, sha256Sum, prtSize)
|
||||||
|
if err != nil {
|
||||||
|
// Reset the temporary buffer upon any error.
|
||||||
|
tmpBuffer.Reset()
|
||||||
|
return totalUploadedSize, err
|
||||||
|
}
|
||||||
|
// Save successfully uploaded part metadata.
|
||||||
|
partsInfo[partNumber] = objPart
|
||||||
|
} else {
|
||||||
|
// Update the progress reader for the skipped part.
|
||||||
|
if progress != nil {
|
||||||
|
if _, err = io.CopyN(ioutil.Discard, progress, prtSize); err != nil {
|
||||||
|
return totalUploadedSize, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reset the temporary buffer.
|
||||||
|
tmpBuffer.Reset()
|
||||||
|
|
||||||
|
// Save successfully uploaded size.
|
||||||
|
totalUploadedSize += prtSize
|
||||||
|
|
||||||
|
// For unknown size, Read EOF we break away.
|
||||||
|
// We do not have to upload till totalPartsCount.
|
||||||
|
if size < 0 && rErr == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
// Increment part number.
|
||||||
|
partNumber++
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify if we uploaded all the data.
|
||||||
|
if size > 0 {
|
||||||
|
if totalUploadedSize != size {
|
||||||
|
return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Loop over uploaded parts to save them in a Parts array before completing the multipart request.
|
||||||
|
for _, part := range partsInfo {
|
||||||
|
var complPart completePart
|
||||||
|
complPart.ETag = part.ETag
|
||||||
|
complPart.PartNumber = part.PartNumber
|
||||||
|
complMultipartUpload.Parts = append(complMultipartUpload.Parts, complPart)
|
||||||
|
}
|
||||||
|
|
||||||
|
if size > 0 {
|
||||||
|
// Verify if totalPartsCount is not equal to total list of parts.
|
||||||
|
if totalPartsCount != len(complMultipartUpload.Parts) {
|
||||||
|
return totalUploadedSize, ErrInvalidParts(partNumber, len(complMultipartUpload.Parts))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort all completed parts.
|
||||||
|
sort.Sort(completedParts(complMultipartUpload.Parts))
|
||||||
|
_, err = c.completeMultipartUpload(bucketName, objectName, uploadID, complMultipartUpload)
|
||||||
|
if err != nil {
|
||||||
|
return totalUploadedSize, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return final size.
|
||||||
|
return totalUploadedSize, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// initiateMultipartUpload - Initiates a multipart upload and returns an upload ID.
|
||||||
|
func (c Client) initiateMultipartUpload(bucketName, objectName, contentType string) (initiateMultipartUploadResult, error) {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return initiateMultipartUploadResult{}, err
|
||||||
|
}
|
||||||
|
if err := isValidObjectName(objectName); err != nil {
|
||||||
|
return initiateMultipartUploadResult{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize url queries.
|
||||||
|
urlValues := make(url.Values)
|
||||||
|
urlValues.Set("uploads", "")
|
||||||
|
|
||||||
|
if contentType == "" {
|
||||||
|
contentType = "application/octet-stream"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set ContentType header.
|
||||||
|
customHeader := make(http.Header)
|
||||||
|
customHeader.Set("Content-Type", contentType)
|
||||||
|
|
||||||
|
reqMetadata := requestMetadata{
|
||||||
|
bucketName: bucketName,
|
||||||
|
objectName: objectName,
|
||||||
|
queryValues: urlValues,
|
||||||
|
customHeader: customHeader,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Instantiate the request.
|
||||||
|
req, err := c.newRequest("POST", reqMetadata)
|
||||||
|
if err != nil {
|
||||||
|
return initiateMultipartUploadResult{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute the request.
|
||||||
|
resp, err := c.do(req)
|
||||||
|
defer closeResponse(resp)
|
||||||
|
if err != nil {
|
||||||
|
return initiateMultipartUploadResult{}, err
|
||||||
|
}
|
||||||
|
if resp != nil {
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return initiateMultipartUploadResult{}, httpRespToErrorResponse(resp, bucketName, objectName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Decode xml for new multipart upload.
|
||||||
|
initiateMultipartUploadResult := initiateMultipartUploadResult{}
|
||||||
|
err = xmlDecoder(resp.Body, &initiateMultipartUploadResult)
|
||||||
|
if err != nil {
|
||||||
|
return initiateMultipartUploadResult, err
|
||||||
|
}
|
||||||
|
return initiateMultipartUploadResult, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// uploadPart - Uploads a part in a multipart upload.
|
||||||
|
func (c Client) uploadPart(bucketName, objectName, uploadID string, reader io.ReadCloser, partNumber int, md5Sum, sha256Sum []byte, size int64) (objectPart, error) {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return objectPart{}, err
|
||||||
|
}
|
||||||
|
if err := isValidObjectName(objectName); err != nil {
|
||||||
|
return objectPart{}, err
|
||||||
|
}
|
||||||
|
if size > maxPartSize {
|
||||||
|
return objectPart{}, ErrEntityTooLarge(size, maxPartSize, bucketName, objectName)
|
||||||
|
}
|
||||||
|
if size <= -1 {
|
||||||
|
return objectPart{}, ErrEntityTooSmall(size, bucketName, objectName)
|
||||||
|
}
|
||||||
|
if partNumber <= 0 {
|
||||||
|
return objectPart{}, ErrInvalidArgument("Part number cannot be negative or equal to zero.")
|
||||||
|
}
|
||||||
|
if uploadID == "" {
|
||||||
|
return objectPart{}, ErrInvalidArgument("UploadID cannot be empty.")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get resources properly escaped and lined up before using them in http request.
|
||||||
|
urlValues := make(url.Values)
|
||||||
|
// Set part number.
|
||||||
|
urlValues.Set("partNumber", strconv.Itoa(partNumber))
|
||||||
|
// Set upload id.
|
||||||
|
urlValues.Set("uploadId", uploadID)
|
||||||
|
|
||||||
|
reqMetadata := requestMetadata{
|
||||||
|
bucketName: bucketName,
|
||||||
|
objectName: objectName,
|
||||||
|
queryValues: urlValues,
|
||||||
|
contentBody: reader,
|
||||||
|
contentLength: size,
|
||||||
|
contentMD5Bytes: md5Sum,
|
||||||
|
contentSHA256Bytes: sha256Sum,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Instantiate a request.
|
||||||
|
req, err := c.newRequest("PUT", reqMetadata)
|
||||||
|
if err != nil {
|
||||||
|
return objectPart{}, err
|
||||||
|
}
|
||||||
|
// Execute the request.
|
||||||
|
resp, err := c.do(req)
|
||||||
|
defer closeResponse(resp)
|
||||||
|
if err != nil {
|
||||||
|
return objectPart{}, err
|
||||||
|
}
|
||||||
|
if resp != nil {
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return objectPart{}, httpRespToErrorResponse(resp, bucketName, objectName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Once successfully uploaded, return completed part.
|
||||||
|
objPart := objectPart{}
|
||||||
|
objPart.Size = size
|
||||||
|
objPart.PartNumber = partNumber
|
||||||
|
// Trim off the odd double quotes from ETag in the beginning and end.
|
||||||
|
objPart.ETag = strings.TrimPrefix(resp.Header.Get("ETag"), "\"")
|
||||||
|
objPart.ETag = strings.TrimSuffix(objPart.ETag, "\"")
|
||||||
|
return objPart, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// completeMultipartUpload - Completes a multipart upload by assembling previously uploaded parts.
|
||||||
|
func (c Client) completeMultipartUpload(bucketName, objectName, uploadID string, complete completeMultipartUpload) (completeMultipartUploadResult, error) {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return completeMultipartUploadResult{}, err
|
||||||
|
}
|
||||||
|
if err := isValidObjectName(objectName); err != nil {
|
||||||
|
return completeMultipartUploadResult{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize url queries.
|
||||||
|
urlValues := make(url.Values)
|
||||||
|
urlValues.Set("uploadId", uploadID)
|
||||||
|
|
||||||
|
// Marshal complete multipart body.
|
||||||
|
completeMultipartUploadBytes, err := xml.Marshal(complete)
|
||||||
|
if err != nil {
|
||||||
|
return completeMultipartUploadResult{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Instantiate all the complete multipart buffer.
|
||||||
|
completeMultipartUploadBuffer := bytes.NewBuffer(completeMultipartUploadBytes)
|
||||||
|
reqMetadata := requestMetadata{
|
||||||
|
bucketName: bucketName,
|
||||||
|
objectName: objectName,
|
||||||
|
queryValues: urlValues,
|
||||||
|
contentBody: ioutil.NopCloser(completeMultipartUploadBuffer),
|
||||||
|
contentLength: int64(completeMultipartUploadBuffer.Len()),
|
||||||
|
contentSHA256Bytes: sum256(completeMultipartUploadBuffer.Bytes()),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Instantiate the request.
|
||||||
|
req, err := c.newRequest("POST", reqMetadata)
|
||||||
|
if err != nil {
|
||||||
|
return completeMultipartUploadResult{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute the request.
|
||||||
|
resp, err := c.do(req)
|
||||||
|
defer closeResponse(resp)
|
||||||
|
if err != nil {
|
||||||
|
return completeMultipartUploadResult{}, err
|
||||||
|
}
|
||||||
|
if resp != nil {
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return completeMultipartUploadResult{}, httpRespToErrorResponse(resp, bucketName, objectName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Decode completed multipart upload response on success.
|
||||||
|
completeMultipartUploadResult := completeMultipartUploadResult{}
|
||||||
|
err = xmlDecoder(resp.Body, &completeMultipartUploadResult)
|
||||||
|
if err != nil {
|
||||||
|
return completeMultipartUploadResult, err
|
||||||
|
}
|
||||||
|
return completeMultipartUploadResult, nil
|
||||||
|
}
|
105
vendor/github.com/minio/minio-go/api-put-object-progress.go
generated
vendored
Normal file
105
vendor/github.com/minio/minio-go/api-put-object-progress.go
generated
vendored
Normal file
@ -0,0 +1,105 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import "io"
|
||||||
|
|
||||||
|
// PutObjectWithProgress - With progress.
|
||||||
|
func (c Client) PutObjectWithProgress(bucketName, objectName string, reader io.Reader, contentType string, progress io.Reader) (n int64, err error) {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if err := isValidObjectName(objectName); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if reader == nil {
|
||||||
|
return 0, ErrInvalidArgument("Input reader is invalid, cannot be nil.")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Size of the object.
|
||||||
|
var size int64
|
||||||
|
|
||||||
|
// Get reader size.
|
||||||
|
size, err = getReaderSize(reader)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for largest object size allowed.
|
||||||
|
if size > int64(maxMultipartPutObjectSize) {
|
||||||
|
return 0, ErrEntityTooLarge(size, maxMultipartPutObjectSize, bucketName, objectName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NOTE: Google Cloud Storage does not implement Amazon S3 Compatible multipart PUT.
|
||||||
|
// So we fall back to single PUT operation with the maximum limit of 5GiB.
|
||||||
|
if isGoogleEndpoint(c.endpointURL) {
|
||||||
|
if size <= -1 {
|
||||||
|
return 0, ErrorResponse{
|
||||||
|
Code: "NotImplemented",
|
||||||
|
Message: "Content-Length cannot be negative for file uploads to Google Cloud Storage.",
|
||||||
|
Key: objectName,
|
||||||
|
BucketName: bucketName,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if size > maxSinglePutObjectSize {
|
||||||
|
return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName)
|
||||||
|
}
|
||||||
|
// Do not compute MD5 for Google Cloud Storage. Uploads up to 5GiB in size.
|
||||||
|
return c.putObjectNoChecksum(bucketName, objectName, reader, size, contentType, progress)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NOTE: S3 doesn't allow anonymous multipart requests.
|
||||||
|
if isAmazonEndpoint(c.endpointURL) && c.anonymous {
|
||||||
|
if size <= -1 {
|
||||||
|
return 0, ErrorResponse{
|
||||||
|
Code: "NotImplemented",
|
||||||
|
Message: "Content-Length cannot be negative for anonymous requests.",
|
||||||
|
Key: objectName,
|
||||||
|
BucketName: bucketName,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if size > maxSinglePutObjectSize {
|
||||||
|
return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName)
|
||||||
|
}
|
||||||
|
// Do not compute MD5 for anonymous requests to Amazon
|
||||||
|
// S3. Uploads up to 5GiB in size.
|
||||||
|
return c.putObjectNoChecksum(bucketName, objectName, reader, size, contentType, progress)
|
||||||
|
}
|
||||||
|
|
||||||
|
// putSmall object.
|
||||||
|
if size < minPartSize && size > 0 {
|
||||||
|
return c.putObjectSingle(bucketName, objectName, reader, size, contentType, progress)
|
||||||
|
}
|
||||||
|
// For all sizes greater than 5MiB do multipart.
|
||||||
|
n, err = c.putObjectMultipart(bucketName, objectName, reader, size, contentType, progress)
|
||||||
|
if err != nil {
|
||||||
|
errResp := ToErrorResponse(err)
|
||||||
|
// Verify if multipart functionality is not available, if not
|
||||||
|
// fall back to single PutObject operation.
|
||||||
|
if errResp.Code == "NotImplemented" {
|
||||||
|
// Verify if size of reader is greater than '5GiB'.
|
||||||
|
if size > maxSinglePutObjectSize {
|
||||||
|
return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName)
|
||||||
|
}
|
||||||
|
// Fall back to uploading as single PutObject operation.
|
||||||
|
return c.putObjectSingle(bucketName, objectName, reader, size, contentType, progress)
|
||||||
|
}
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
208
vendor/github.com/minio/minio-go/api-put-object-readat.go
generated
vendored
Normal file
208
vendor/github.com/minio/minio-go/api-put-object-readat.go
generated
vendored
Normal file
@ -0,0 +1,208 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
"sort"
|
||||||
|
)
|
||||||
|
|
||||||
|
// shouldUploadPartReadAt - verify if part should be uploaded.
|
||||||
|
func shouldUploadPartReadAt(objPart objectPart, objectParts map[int]objectPart) bool {
|
||||||
|
// If part not found part should be uploaded.
|
||||||
|
uploadedPart, found := objectParts[objPart.PartNumber]
|
||||||
|
if !found {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
// if size mismatches part should be uploaded.
|
||||||
|
if uploadedPart.Size != objPart.Size {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// putObjectMultipartFromReadAt - Uploads files bigger than 5MiB. Supports reader
|
||||||
|
// of type which implements io.ReaderAt interface (ReadAt method).
|
||||||
|
//
|
||||||
|
// NOTE: This function is meant to be used for all readers which
|
||||||
|
// implement io.ReaderAt which allows us for resuming multipart
|
||||||
|
// uploads but reading at an offset, which would avoid re-read the
|
||||||
|
// data which was already uploaded. Internally this function uses
|
||||||
|
// temporary files for staging all the data, these temporary files are
|
||||||
|
// cleaned automatically when the caller i.e http client closes the
|
||||||
|
// stream after uploading all the contents successfully.
|
||||||
|
func (c Client) putObjectMultipartFromReadAt(bucketName, objectName string, reader io.ReaderAt, size int64, contentType string, progress io.Reader) (n int64, err error) {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if err := isValidObjectName(objectName); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get upload id for an object, initiates a new multipart request
|
||||||
|
// if it cannot find any previously partially uploaded object.
|
||||||
|
uploadID, isNew, err := c.getUploadID(bucketName, objectName, contentType)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Total data read and written to server. should be equal to 'size' at the end of the call.
|
||||||
|
var totalUploadedSize int64
|
||||||
|
|
||||||
|
// Complete multipart upload.
|
||||||
|
var complMultipartUpload completeMultipartUpload
|
||||||
|
|
||||||
|
// A map of all uploaded parts.
|
||||||
|
var partsInfo = make(map[int]objectPart)
|
||||||
|
|
||||||
|
// Fetch all parts info previously uploaded.
|
||||||
|
if !isNew {
|
||||||
|
partsInfo, err = c.listObjectParts(bucketName, objectName, uploadID)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate the optimal parts info for a given size.
|
||||||
|
totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Used for readability, lastPartNumber is always
|
||||||
|
// totalPartsCount.
|
||||||
|
lastPartNumber := totalPartsCount
|
||||||
|
|
||||||
|
// partNumber always starts with '1'.
|
||||||
|
partNumber := 1
|
||||||
|
|
||||||
|
// Initialize a temporary buffer.
|
||||||
|
tmpBuffer := new(bytes.Buffer)
|
||||||
|
|
||||||
|
// Read defaults to reading at 5MiB buffer.
|
||||||
|
readBuffer := make([]byte, optimalReadBufferSize)
|
||||||
|
|
||||||
|
// Upload all the missing parts.
|
||||||
|
for partNumber <= lastPartNumber {
|
||||||
|
// Verify object if its uploaded.
|
||||||
|
verifyObjPart := objectPart{
|
||||||
|
PartNumber: partNumber,
|
||||||
|
Size: partSize,
|
||||||
|
}
|
||||||
|
// Special case if we see a last part number, save last part
|
||||||
|
// size as the proper part size.
|
||||||
|
if partNumber == lastPartNumber {
|
||||||
|
verifyObjPart = objectPart{
|
||||||
|
PartNumber: lastPartNumber,
|
||||||
|
Size: lastPartSize,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify if part should be uploaded.
|
||||||
|
if !shouldUploadPartReadAt(verifyObjPart, partsInfo) {
|
||||||
|
// Increment part number when not uploaded.
|
||||||
|
partNumber++
|
||||||
|
if progress != nil {
|
||||||
|
// Update the progress reader for the skipped part.
|
||||||
|
if _, err = io.CopyN(ioutil.Discard, progress, verifyObjPart.Size); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// If partNumber was not uploaded we calculate the missing
|
||||||
|
// part offset and size. For all other part numbers we
|
||||||
|
// calculate offset based on multiples of partSize.
|
||||||
|
readOffset := int64(partNumber-1) * partSize
|
||||||
|
missingPartSize := partSize
|
||||||
|
|
||||||
|
// As a special case if partNumber is lastPartNumber, we
|
||||||
|
// calculate the offset based on the last part size.
|
||||||
|
if partNumber == lastPartNumber {
|
||||||
|
readOffset = (size - lastPartSize)
|
||||||
|
missingPartSize = lastPartSize
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get a section reader on a particular offset.
|
||||||
|
sectionReader := io.NewSectionReader(reader, readOffset, missingPartSize)
|
||||||
|
|
||||||
|
// Calculates MD5 and SHA256 sum for a section reader.
|
||||||
|
var md5Sum, sha256Sum []byte
|
||||||
|
var prtSize int64
|
||||||
|
md5Sum, sha256Sum, prtSize, err = c.hashCopyBuffer(tmpBuffer, sectionReader, readBuffer)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var reader io.Reader
|
||||||
|
// Update progress reader appropriately to the latest offset
|
||||||
|
// as we read from the source.
|
||||||
|
reader = newHook(tmpBuffer, progress)
|
||||||
|
|
||||||
|
// Proceed to upload the part.
|
||||||
|
var objPart objectPart
|
||||||
|
objPart, err = c.uploadPart(bucketName, objectName, uploadID, ioutil.NopCloser(reader),
|
||||||
|
partNumber, md5Sum, sha256Sum, prtSize)
|
||||||
|
if err != nil {
|
||||||
|
// Reset the buffer upon any error.
|
||||||
|
tmpBuffer.Reset()
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save successfully uploaded part metadata.
|
||||||
|
partsInfo[partNumber] = objPart
|
||||||
|
|
||||||
|
// Increment part number here after successful part upload.
|
||||||
|
partNumber++
|
||||||
|
|
||||||
|
// Reset the buffer.
|
||||||
|
tmpBuffer.Reset()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Loop over uploaded parts to save them in a Parts array before completing the multipart request.
|
||||||
|
for _, part := range partsInfo {
|
||||||
|
var complPart completePart
|
||||||
|
complPart.ETag = part.ETag
|
||||||
|
complPart.PartNumber = part.PartNumber
|
||||||
|
totalUploadedSize += part.Size
|
||||||
|
complMultipartUpload.Parts = append(complMultipartUpload.Parts, complPart)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify if we uploaded all the data.
|
||||||
|
if totalUploadedSize != size {
|
||||||
|
return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify if totalPartsCount is not equal to total list of parts.
|
||||||
|
if totalPartsCount != len(complMultipartUpload.Parts) {
|
||||||
|
return totalUploadedSize, ErrInvalidParts(totalPartsCount, len(complMultipartUpload.Parts))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort all completed parts.
|
||||||
|
sort.Sort(completedParts(complMultipartUpload.Parts))
|
||||||
|
_, err = c.completeMultipartUpload(bucketName, objectName, uploadID, complMultipartUpload)
|
||||||
|
if err != nil {
|
||||||
|
return totalUploadedSize, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return final size.
|
||||||
|
return totalUploadedSize, nil
|
||||||
|
}
|
287
vendor/github.com/minio/minio-go/api-put-object.go
generated
vendored
Normal file
287
vendor/github.com/minio/minio-go/api-put-object.go
generated
vendored
Normal file
@ -0,0 +1,287 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// getReaderSize - Determine the size of Reader if available.
|
||||||
|
func getReaderSize(reader io.Reader) (size int64, err error) {
|
||||||
|
var result []reflect.Value
|
||||||
|
size = -1
|
||||||
|
if reader != nil {
|
||||||
|
// Verify if there is a method by name 'Size'.
|
||||||
|
lenFn := reflect.ValueOf(reader).MethodByName("Size")
|
||||||
|
if lenFn.IsValid() {
|
||||||
|
if lenFn.Kind() == reflect.Func {
|
||||||
|
// Call the 'Size' function and save its return value.
|
||||||
|
result = lenFn.Call([]reflect.Value{})
|
||||||
|
if result != nil && len(result) == 1 {
|
||||||
|
lenValue := result[0]
|
||||||
|
if lenValue.IsValid() {
|
||||||
|
switch lenValue.Kind() {
|
||||||
|
case reflect.Int:
|
||||||
|
fallthrough
|
||||||
|
case reflect.Int8:
|
||||||
|
fallthrough
|
||||||
|
case reflect.Int16:
|
||||||
|
fallthrough
|
||||||
|
case reflect.Int32:
|
||||||
|
fallthrough
|
||||||
|
case reflect.Int64:
|
||||||
|
size = lenValue.Int()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Fallback to Stat() method, two possible Stat() structs
|
||||||
|
// exist.
|
||||||
|
switch v := reader.(type) {
|
||||||
|
case *os.File:
|
||||||
|
var st os.FileInfo
|
||||||
|
st, err = v.Stat()
|
||||||
|
if err != nil {
|
||||||
|
// Handle this case specially for "windows",
|
||||||
|
// certain files for example 'Stdin', 'Stdout' and
|
||||||
|
// 'Stderr' it is not allowed to fetch file information.
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
if strings.Contains(err.Error(), "GetFileInformationByHandle") {
|
||||||
|
return -1, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Ignore if input is a directory, throw an error.
|
||||||
|
if st.Mode().IsDir() {
|
||||||
|
return -1, ErrInvalidArgument("Input file cannot be a directory.")
|
||||||
|
}
|
||||||
|
// Ignore 'Stdin', 'Stdout' and 'Stderr', since they
|
||||||
|
// represent *os.File type but internally do not
|
||||||
|
// implement Seekable calls. Ignore them and treat
|
||||||
|
// them like a stream with unknown length.
|
||||||
|
switch st.Name() {
|
||||||
|
case "stdin":
|
||||||
|
fallthrough
|
||||||
|
case "stdout":
|
||||||
|
fallthrough
|
||||||
|
case "stderr":
|
||||||
|
return
|
||||||
|
}
|
||||||
|
size = st.Size()
|
||||||
|
case *Object:
|
||||||
|
var st ObjectInfo
|
||||||
|
st, err = v.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
size = st.Size
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Returns the size here.
|
||||||
|
return size, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// completedParts is a collection of parts sortable by their part numbers.
|
||||||
|
// used for sorting the uploaded parts before completing the multipart request.
|
||||||
|
type completedParts []completePart
|
||||||
|
|
||||||
|
func (a completedParts) Len() int { return len(a) }
|
||||||
|
func (a completedParts) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||||
|
func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
|
||||||
|
|
||||||
|
// PutObject creates an object in a bucket.
|
||||||
|
//
|
||||||
|
// You must have WRITE permissions on a bucket to create an object.
|
||||||
|
//
|
||||||
|
// - For size smaller than 5MiB PutObject automatically does a single atomic Put operation.
|
||||||
|
// - For size larger than 5MiB PutObject automatically does a resumable multipart Put operation.
|
||||||
|
// - For size input as -1 PutObject does a multipart Put operation until input stream reaches EOF.
|
||||||
|
// Maximum object size that can be uploaded through this operation will be 5TiB.
|
||||||
|
//
|
||||||
|
// NOTE: Google Cloud Storage does not implement Amazon S3 Compatible multipart PUT.
|
||||||
|
// So we fall back to single PUT operation with the maximum limit of 5GiB.
|
||||||
|
//
|
||||||
|
// NOTE: For anonymous requests Amazon S3 doesn't allow multipart upload. So we fall back to single PUT operation.
|
||||||
|
func (c Client) PutObject(bucketName, objectName string, reader io.Reader, contentType string) (n int64, err error) {
|
||||||
|
return c.PutObjectWithProgress(bucketName, objectName, reader, contentType, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// putObjectNoChecksum special function used Google Cloud Storage. This special function
|
||||||
|
// is used for Google Cloud Storage since Google's multipart API is not S3 compatible.
|
||||||
|
func (c Client) putObjectNoChecksum(bucketName, objectName string, reader io.Reader, size int64, contentType string, progress io.Reader) (n int64, err error) {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if err := isValidObjectName(objectName); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if size > maxSinglePutObjectSize {
|
||||||
|
return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update progress reader appropriately to the latest offset as we
|
||||||
|
// read from the source.
|
||||||
|
reader = newHook(reader, progress)
|
||||||
|
|
||||||
|
// This function does not calculate sha256 and md5sum for payload.
|
||||||
|
// Execute put object.
|
||||||
|
st, err := c.putObjectDo(bucketName, objectName, ioutil.NopCloser(reader), nil, nil, size, contentType)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if st.Size != size {
|
||||||
|
return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName)
|
||||||
|
}
|
||||||
|
return size, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// putObjectSingle is a special function for uploading single put object request.
|
||||||
|
// This special function is used as a fallback when multipart upload fails.
|
||||||
|
func (c Client) putObjectSingle(bucketName, objectName string, reader io.Reader, size int64, contentType string, progress io.Reader) (n int64, err error) {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if err := isValidObjectName(objectName); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if size > maxSinglePutObjectSize {
|
||||||
|
return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName)
|
||||||
|
}
|
||||||
|
// If size is a stream, upload up to 5GiB.
|
||||||
|
if size <= -1 {
|
||||||
|
size = maxSinglePutObjectSize
|
||||||
|
}
|
||||||
|
var md5Sum, sha256Sum []byte
|
||||||
|
var readCloser io.ReadCloser
|
||||||
|
if size <= minPartSize {
|
||||||
|
// Initialize a new temporary buffer.
|
||||||
|
tmpBuffer := new(bytes.Buffer)
|
||||||
|
md5Sum, sha256Sum, size, err = c.hashCopyN(tmpBuffer, reader, size)
|
||||||
|
readCloser = ioutil.NopCloser(tmpBuffer)
|
||||||
|
} else {
|
||||||
|
// Initialize a new temporary file.
|
||||||
|
var tmpFile *tempFile
|
||||||
|
tmpFile, err = newTempFile("single$-putobject-single")
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
md5Sum, sha256Sum, size, err = c.hashCopyN(tmpFile, reader, size)
|
||||||
|
// Seek back to beginning of the temporary file.
|
||||||
|
if _, err = tmpFile.Seek(0, 0); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
readCloser = tmpFile
|
||||||
|
}
|
||||||
|
// Return error if its not io.EOF.
|
||||||
|
if err != nil {
|
||||||
|
if err != io.EOF {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Progress the reader to the size.
|
||||||
|
if progress != nil {
|
||||||
|
if _, err = io.CopyN(ioutil.Discard, progress, size); err != nil {
|
||||||
|
return size, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Execute put object.
|
||||||
|
st, err := c.putObjectDo(bucketName, objectName, readCloser, md5Sum, sha256Sum, size, contentType)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if st.Size != size {
|
||||||
|
return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName)
|
||||||
|
}
|
||||||
|
return size, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// putObjectDo - executes the put object http operation.
|
||||||
|
// NOTE: You must have WRITE permissions on a bucket to add an object to it.
|
||||||
|
func (c Client) putObjectDo(bucketName, objectName string, reader io.ReadCloser, md5Sum []byte, sha256Sum []byte, size int64, contentType string) (ObjectInfo, error) {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
if err := isValidObjectName(objectName); err != nil {
|
||||||
|
return ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if size <= -1 {
|
||||||
|
return ObjectInfo{}, ErrEntityTooSmall(size, bucketName, objectName)
|
||||||
|
}
|
||||||
|
|
||||||
|
if size > maxSinglePutObjectSize {
|
||||||
|
return ObjectInfo{}, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName)
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.TrimSpace(contentType) == "" {
|
||||||
|
contentType = "application/octet-stream"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set headers.
|
||||||
|
customHeader := make(http.Header)
|
||||||
|
customHeader.Set("Content-Type", contentType)
|
||||||
|
|
||||||
|
// Populate request metadata.
|
||||||
|
reqMetadata := requestMetadata{
|
||||||
|
bucketName: bucketName,
|
||||||
|
objectName: objectName,
|
||||||
|
customHeader: customHeader,
|
||||||
|
contentBody: reader,
|
||||||
|
contentLength: size,
|
||||||
|
contentMD5Bytes: md5Sum,
|
||||||
|
contentSHA256Bytes: sha256Sum,
|
||||||
|
}
|
||||||
|
// Initiate new request.
|
||||||
|
req, err := c.newRequest("PUT", reqMetadata)
|
||||||
|
if err != nil {
|
||||||
|
return ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
// Execute the request.
|
||||||
|
resp, err := c.do(req)
|
||||||
|
defer closeResponse(resp)
|
||||||
|
if err != nil {
|
||||||
|
return ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
if resp != nil {
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return ObjectInfo{}, httpRespToErrorResponse(resp, bucketName, objectName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var metadata ObjectInfo
|
||||||
|
// Trim off the odd double quotes from ETag in the beginning and end.
|
||||||
|
metadata.ETag = strings.TrimPrefix(resp.Header.Get("ETag"), "\"")
|
||||||
|
metadata.ETag = strings.TrimSuffix(metadata.ETag, "\"")
|
||||||
|
// A success here means data was written to server successfully.
|
||||||
|
metadata.Size = size
|
||||||
|
|
||||||
|
// Return here.
|
||||||
|
return metadata, nil
|
||||||
|
}
|
167
vendor/github.com/minio/minio-go/api-remove.go
generated
vendored
Normal file
167
vendor/github.com/minio/minio-go/api-remove.go
generated
vendored
Normal file
@ -0,0 +1,167 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RemoveBucket deletes the bucket name.
|
||||||
|
//
|
||||||
|
// All objects (including all object versions and delete markers).
|
||||||
|
// in the bucket must be deleted before successfully attempting this request.
|
||||||
|
func (c Client) RemoveBucket(bucketName string) error {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Instantiate a new request.
|
||||||
|
req, err := c.newRequest("DELETE", requestMetadata{
|
||||||
|
bucketName: bucketName,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Initiate the request.
|
||||||
|
resp, err := c.do(req)
|
||||||
|
defer closeResponse(resp)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if resp != nil {
|
||||||
|
if resp.StatusCode != http.StatusNoContent {
|
||||||
|
return httpRespToErrorResponse(resp, bucketName, "")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove the location from cache on a successful delete.
|
||||||
|
c.bucketLocCache.Delete(bucketName)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// RemoveObject remove an object from a bucket.
|
||||||
|
func (c Client) RemoveObject(bucketName, objectName string) error {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := isValidObjectName(objectName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Instantiate the request.
|
||||||
|
req, err := c.newRequest("DELETE", requestMetadata{
|
||||||
|
bucketName: bucketName,
|
||||||
|
objectName: objectName,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Initiate the request.
|
||||||
|
resp, err := c.do(req)
|
||||||
|
defer closeResponse(resp)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// DeleteObject always responds with http '204' even for
|
||||||
|
// objects which do not exist. So no need to handle them
|
||||||
|
// specifically.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// RemoveIncompleteUpload aborts an partially uploaded object.
|
||||||
|
// Requires explicit authentication, no anonymous requests are allowed for multipart API.
|
||||||
|
func (c Client) RemoveIncompleteUpload(bucketName, objectName string) error {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := isValidObjectName(objectName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Find multipart upload id of the object to be aborted.
|
||||||
|
uploadID, err := c.findUploadID(bucketName, objectName)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if uploadID != "" {
|
||||||
|
// Upload id found, abort the incomplete multipart upload.
|
||||||
|
err := c.abortMultipartUpload(bucketName, objectName, uploadID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// abortMultipartUpload aborts a multipart upload for the given
|
||||||
|
// uploadID, all previously uploaded parts are deleted.
|
||||||
|
func (c Client) abortMultipartUpload(bucketName, objectName, uploadID string) error {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := isValidObjectName(objectName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize url queries.
|
||||||
|
urlValues := make(url.Values)
|
||||||
|
urlValues.Set("uploadId", uploadID)
|
||||||
|
|
||||||
|
// Instantiate a new DELETE request.
|
||||||
|
req, err := c.newRequest("DELETE", requestMetadata{
|
||||||
|
bucketName: bucketName,
|
||||||
|
objectName: objectName,
|
||||||
|
queryValues: urlValues,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initiate the request.
|
||||||
|
resp, err := c.do(req)
|
||||||
|
defer closeResponse(resp)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if resp != nil {
|
||||||
|
if resp.StatusCode != http.StatusNoContent {
|
||||||
|
// Abort has no response body, handle it for any errors.
|
||||||
|
var errorResponse ErrorResponse
|
||||||
|
switch resp.StatusCode {
|
||||||
|
case http.StatusNotFound:
|
||||||
|
// This is needed specifically for abort and it cannot
|
||||||
|
// be converged into default case.
|
||||||
|
errorResponse = ErrorResponse{
|
||||||
|
Code: "NoSuchUpload",
|
||||||
|
Message: "The specified multipart upload does not exist.",
|
||||||
|
BucketName: bucketName,
|
||||||
|
Key: objectName,
|
||||||
|
RequestID: resp.Header.Get("x-amz-request-id"),
|
||||||
|
HostID: resp.Header.Get("x-amz-id-2"),
|
||||||
|
AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"),
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return httpRespToErrorResponse(resp, bucketName, objectName)
|
||||||
|
}
|
||||||
|
return errorResponse
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
197
vendor/github.com/minio/minio-go/api-s3-definitions.go
generated
vendored
Normal file
197
vendor/github.com/minio/minio-go/api-s3-definitions.go
generated
vendored
Normal file
@ -0,0 +1,197 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/xml"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// listAllMyBucketsResult container for listBuckets response.
|
||||||
|
type listAllMyBucketsResult struct {
|
||||||
|
// Container for one or more buckets.
|
||||||
|
Buckets struct {
|
||||||
|
Bucket []BucketInfo
|
||||||
|
}
|
||||||
|
Owner owner
|
||||||
|
}
|
||||||
|
|
||||||
|
// owner container for bucket owner information.
|
||||||
|
type owner struct {
|
||||||
|
DisplayName string
|
||||||
|
ID string
|
||||||
|
}
|
||||||
|
|
||||||
|
// commonPrefix container for prefix response.
|
||||||
|
type commonPrefix struct {
|
||||||
|
Prefix string
|
||||||
|
}
|
||||||
|
|
||||||
|
// listBucketResult container for listObjects response.
|
||||||
|
type listBucketResult struct {
|
||||||
|
// A response can contain CommonPrefixes only if you have
|
||||||
|
// specified a delimiter.
|
||||||
|
CommonPrefixes []commonPrefix
|
||||||
|
// Metadata about each object returned.
|
||||||
|
Contents []ObjectInfo
|
||||||
|
Delimiter string
|
||||||
|
|
||||||
|
// Encoding type used to encode object keys in the response.
|
||||||
|
EncodingType string
|
||||||
|
|
||||||
|
// A flag that indicates whether or not ListObjects returned all of the results
|
||||||
|
// that satisfied the search criteria.
|
||||||
|
IsTruncated bool
|
||||||
|
Marker string
|
||||||
|
MaxKeys int64
|
||||||
|
Name string
|
||||||
|
|
||||||
|
// When response is truncated (the IsTruncated element value in
|
||||||
|
// the response is true), you can use the key name in this field
|
||||||
|
// as marker in the subsequent request to get next set of objects.
|
||||||
|
// Object storage lists objects in alphabetical order Note: This
|
||||||
|
// element is returned only if you have delimiter request
|
||||||
|
// parameter specified. If response does not include the NextMaker
|
||||||
|
// and it is truncated, you can use the value of the last Key in
|
||||||
|
// the response as the marker in the subsequent request to get the
|
||||||
|
// next set of object keys.
|
||||||
|
NextMarker string
|
||||||
|
Prefix string
|
||||||
|
}
|
||||||
|
|
||||||
|
// listMultipartUploadsResult container for ListMultipartUploads response
|
||||||
|
type listMultipartUploadsResult struct {
|
||||||
|
Bucket string
|
||||||
|
KeyMarker string
|
||||||
|
UploadIDMarker string `xml:"UploadIdMarker"`
|
||||||
|
NextKeyMarker string
|
||||||
|
NextUploadIDMarker string `xml:"NextUploadIdMarker"`
|
||||||
|
EncodingType string
|
||||||
|
MaxUploads int64
|
||||||
|
IsTruncated bool
|
||||||
|
Uploads []ObjectMultipartInfo `xml:"Upload"`
|
||||||
|
Prefix string
|
||||||
|
Delimiter string
|
||||||
|
// A response can contain CommonPrefixes only if you specify a delimiter.
|
||||||
|
CommonPrefixes []commonPrefix
|
||||||
|
}
|
||||||
|
|
||||||
|
// initiator container for who initiated multipart upload.
|
||||||
|
type initiator struct {
|
||||||
|
ID string
|
||||||
|
DisplayName string
|
||||||
|
}
|
||||||
|
|
||||||
|
// objectPart container for particular part of an object.
|
||||||
|
type objectPart struct {
|
||||||
|
// Part number identifies the part.
|
||||||
|
PartNumber int
|
||||||
|
|
||||||
|
// Date and time the part was uploaded.
|
||||||
|
LastModified time.Time
|
||||||
|
|
||||||
|
// Entity tag returned when the part was uploaded, usually md5sum
|
||||||
|
// of the part.
|
||||||
|
ETag string
|
||||||
|
|
||||||
|
// Size of the uploaded part data.
|
||||||
|
Size int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// listObjectPartsResult container for ListObjectParts response.
|
||||||
|
type listObjectPartsResult struct {
|
||||||
|
Bucket string
|
||||||
|
Key string
|
||||||
|
UploadID string `xml:"UploadId"`
|
||||||
|
|
||||||
|
Initiator initiator
|
||||||
|
Owner owner
|
||||||
|
|
||||||
|
StorageClass string
|
||||||
|
PartNumberMarker int
|
||||||
|
NextPartNumberMarker int
|
||||||
|
MaxParts int
|
||||||
|
|
||||||
|
// Indicates whether the returned list of parts is truncated.
|
||||||
|
IsTruncated bool
|
||||||
|
ObjectParts []objectPart `xml:"Part"`
|
||||||
|
|
||||||
|
EncodingType string
|
||||||
|
}
|
||||||
|
|
||||||
|
// initiateMultipartUploadResult container for InitiateMultiPartUpload
|
||||||
|
// response.
|
||||||
|
type initiateMultipartUploadResult struct {
|
||||||
|
Bucket string
|
||||||
|
Key string
|
||||||
|
UploadID string `xml:"UploadId"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// completeMultipartUploadResult container for completed multipart
|
||||||
|
// upload response.
|
||||||
|
type completeMultipartUploadResult struct {
|
||||||
|
Location string
|
||||||
|
Bucket string
|
||||||
|
Key string
|
||||||
|
ETag string
|
||||||
|
}
|
||||||
|
|
||||||
|
// completePart sub container lists individual part numbers and their
|
||||||
|
// md5sum, part of completeMultipartUpload.
|
||||||
|
type completePart struct {
|
||||||
|
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Part" json:"-"`
|
||||||
|
|
||||||
|
// Part number identifies the part.
|
||||||
|
PartNumber int
|
||||||
|
ETag string
|
||||||
|
}
|
||||||
|
|
||||||
|
// completeMultipartUpload container for completing multipart upload.
|
||||||
|
type completeMultipartUpload struct {
|
||||||
|
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUpload" json:"-"`
|
||||||
|
Parts []completePart `xml:"Part"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// createBucketConfiguration container for bucket configuration.
|
||||||
|
type createBucketConfiguration struct {
|
||||||
|
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreateBucketConfiguration" json:"-"`
|
||||||
|
Location string `xml:"LocationConstraint"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// grant container for the grantee and his or her permissions.
|
||||||
|
type grant struct {
|
||||||
|
// grantee container for DisplayName and ID of the person being
|
||||||
|
// granted permissions.
|
||||||
|
Grantee struct {
|
||||||
|
ID string
|
||||||
|
DisplayName string
|
||||||
|
EmailAddress string
|
||||||
|
Type string
|
||||||
|
URI string
|
||||||
|
}
|
||||||
|
Permission string
|
||||||
|
}
|
||||||
|
|
||||||
|
// accessControlPolicy contains the elements providing ACL permissions
|
||||||
|
// for a bucket.
|
||||||
|
type accessControlPolicy struct {
|
||||||
|
// accessControlList container for ACL information.
|
||||||
|
AccessControlList struct {
|
||||||
|
Grant []grant
|
||||||
|
}
|
||||||
|
Owner owner
|
||||||
|
}
|
125
vendor/github.com/minio/minio-go/api-stat.go
generated
vendored
Normal file
125
vendor/github.com/minio/minio-go/api-stat.go
generated
vendored
Normal file
@ -0,0 +1,125 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// BucketExists verify if bucket exists and you have permission to access it.
|
||||||
|
func (c Client) BucketExists(bucketName string) error {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Instantiate a new request.
|
||||||
|
req, err := c.newRequest("HEAD", requestMetadata{
|
||||||
|
bucketName: bucketName,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Initiate the request.
|
||||||
|
resp, err := c.do(req)
|
||||||
|
defer closeResponse(resp)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if resp != nil {
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return httpRespToErrorResponse(resp, bucketName, "")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// StatObject verifies if object exists and you have permission to access.
|
||||||
|
func (c Client) StatObject(bucketName, objectName string) (ObjectInfo, error) {
|
||||||
|
// Input validation.
|
||||||
|
if err := isValidBucketName(bucketName); err != nil {
|
||||||
|
return ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
if err := isValidObjectName(objectName); err != nil {
|
||||||
|
return ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
// Instantiate a new request.
|
||||||
|
req, err := c.newRequest("HEAD", requestMetadata{
|
||||||
|
bucketName: bucketName,
|
||||||
|
objectName: objectName,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
// Initiate the request.
|
||||||
|
resp, err := c.do(req)
|
||||||
|
defer closeResponse(resp)
|
||||||
|
if err != nil {
|
||||||
|
return ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
if resp != nil {
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return ObjectInfo{}, httpRespToErrorResponse(resp, bucketName, objectName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Trim off the odd double quotes from ETag in the beginning and end.
|
||||||
|
md5sum := strings.TrimPrefix(resp.Header.Get("ETag"), "\"")
|
||||||
|
md5sum = strings.TrimSuffix(md5sum, "\"")
|
||||||
|
|
||||||
|
// Parse content length.
|
||||||
|
size, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return ObjectInfo{}, ErrorResponse{
|
||||||
|
Code: "InternalError",
|
||||||
|
Message: "Content-Length is invalid. " + reportIssue,
|
||||||
|
BucketName: bucketName,
|
||||||
|
Key: objectName,
|
||||||
|
RequestID: resp.Header.Get("x-amz-request-id"),
|
||||||
|
HostID: resp.Header.Get("x-amz-id-2"),
|
||||||
|
AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Parse Last-Modified has http time format.
|
||||||
|
date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified"))
|
||||||
|
if err != nil {
|
||||||
|
return ObjectInfo{}, ErrorResponse{
|
||||||
|
Code: "InternalError",
|
||||||
|
Message: "Last-Modified time format is invalid. " + reportIssue,
|
||||||
|
BucketName: bucketName,
|
||||||
|
Key: objectName,
|
||||||
|
RequestID: resp.Header.Get("x-amz-request-id"),
|
||||||
|
HostID: resp.Header.Get("x-amz-id-2"),
|
||||||
|
AmzBucketRegion: resp.Header.Get("x-amz-bucket-region"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Fetch content type if any present.
|
||||||
|
contentType := strings.TrimSpace(resp.Header.Get("Content-Type"))
|
||||||
|
if contentType == "" {
|
||||||
|
contentType = "application/octet-stream"
|
||||||
|
}
|
||||||
|
// Save object metadata info.
|
||||||
|
var objectStat ObjectInfo
|
||||||
|
objectStat.ETag = md5sum
|
||||||
|
objectStat.Key = objectName
|
||||||
|
objectStat.Size = size
|
||||||
|
objectStat.LastModified = date
|
||||||
|
objectStat.ContentType = contentType
|
||||||
|
return objectStat, nil
|
||||||
|
}
|
533
vendor/github.com/minio/minio-go/api.go
generated
vendored
Normal file
533
vendor/github.com/minio/minio-go/api.go
generated
vendored
Normal file
@ -0,0 +1,533 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httputil"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Client implements Amazon S3 compatible methods.
|
||||||
|
type Client struct {
|
||||||
|
/// Standard options.
|
||||||
|
|
||||||
|
// AccessKeyID required for authorized requests.
|
||||||
|
accessKeyID string
|
||||||
|
// SecretAccessKey required for authorized requests.
|
||||||
|
secretAccessKey string
|
||||||
|
// Choose a signature type if necessary.
|
||||||
|
signature SignatureType
|
||||||
|
// Set to 'true' if Client has no access and secret keys.
|
||||||
|
anonymous bool
|
||||||
|
|
||||||
|
// User supplied.
|
||||||
|
appInfo struct {
|
||||||
|
appName string
|
||||||
|
appVersion string
|
||||||
|
}
|
||||||
|
endpointURL *url.URL
|
||||||
|
|
||||||
|
// Needs allocation.
|
||||||
|
httpClient *http.Client
|
||||||
|
bucketLocCache *bucketLocationCache
|
||||||
|
|
||||||
|
// Advanced functionality
|
||||||
|
isTraceEnabled bool
|
||||||
|
traceOutput io.Writer
|
||||||
|
}
|
||||||
|
|
||||||
|
// Global constants.
|
||||||
|
const (
|
||||||
|
libraryName = "minio-go"
|
||||||
|
libraryVersion = "0.2.5"
|
||||||
|
)
|
||||||
|
|
||||||
|
// User Agent should always following the below style.
|
||||||
|
// Please open an issue to discuss any new changes here.
|
||||||
|
//
|
||||||
|
// Minio (OS; ARCH) LIB/VER APP/VER
|
||||||
|
const (
|
||||||
|
libraryUserAgentPrefix = "Minio (" + runtime.GOOS + "; " + runtime.GOARCH + ") "
|
||||||
|
libraryUserAgent = libraryUserAgentPrefix + libraryName + "/" + libraryVersion
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewV2 - instantiate minio client with Amazon S3 signature version
|
||||||
|
// '2' compatibility.
|
||||||
|
func NewV2(endpoint string, accessKeyID, secretAccessKey string, insecure bool) (CloudStorageClient, error) {
|
||||||
|
clnt, err := privateNew(endpoint, accessKeyID, secretAccessKey, insecure)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Set to use signature version '2'.
|
||||||
|
clnt.signature = SignatureV2
|
||||||
|
return clnt, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewV4 - instantiate minio client with Amazon S3 signature version
|
||||||
|
// '4' compatibility.
|
||||||
|
func NewV4(endpoint string, accessKeyID, secretAccessKey string, insecure bool) (CloudStorageClient, error) {
|
||||||
|
clnt, err := privateNew(endpoint, accessKeyID, secretAccessKey, insecure)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Set to use signature version '4'.
|
||||||
|
clnt.signature = SignatureV4
|
||||||
|
return clnt, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// New - instantiate minio client Client, adds automatic verification
|
||||||
|
// of signature.
|
||||||
|
func New(endpoint string, accessKeyID, secretAccessKey string, insecure bool) (CloudStorageClient, error) {
|
||||||
|
clnt, err := privateNew(endpoint, accessKeyID, secretAccessKey, insecure)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Google cloud storage should be set to signature V2, force it if
|
||||||
|
// not.
|
||||||
|
if isGoogleEndpoint(clnt.endpointURL) {
|
||||||
|
clnt.signature = SignatureV2
|
||||||
|
}
|
||||||
|
// If Amazon S3 set to signature v2.
|
||||||
|
if isAmazonEndpoint(clnt.endpointURL) {
|
||||||
|
clnt.signature = SignatureV4
|
||||||
|
}
|
||||||
|
return clnt, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func privateNew(endpoint, accessKeyID, secretAccessKey string, insecure bool) (*Client, error) {
|
||||||
|
// construct endpoint.
|
||||||
|
endpointURL, err := getEndpointURL(endpoint, insecure)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// instantiate new Client.
|
||||||
|
clnt := new(Client)
|
||||||
|
clnt.accessKeyID = accessKeyID
|
||||||
|
clnt.secretAccessKey = secretAccessKey
|
||||||
|
if clnt.accessKeyID == "" || clnt.secretAccessKey == "" {
|
||||||
|
clnt.anonymous = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save endpoint URL, user agent for future uses.
|
||||||
|
clnt.endpointURL = endpointURL
|
||||||
|
|
||||||
|
// Instantiate http client and bucket location cache.
|
||||||
|
clnt.httpClient = &http.Client{}
|
||||||
|
clnt.bucketLocCache = newBucketLocationCache()
|
||||||
|
|
||||||
|
// Return.
|
||||||
|
return clnt, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetAppInfo - add application details to user agent.
|
||||||
|
func (c *Client) SetAppInfo(appName string, appVersion string) {
|
||||||
|
// if app name and version is not set, we do not a new user
|
||||||
|
// agent.
|
||||||
|
if appName != "" && appVersion != "" {
|
||||||
|
c.appInfo = struct {
|
||||||
|
appName string
|
||||||
|
appVersion string
|
||||||
|
}{}
|
||||||
|
c.appInfo.appName = appName
|
||||||
|
c.appInfo.appVersion = appVersion
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetCustomTransport - set new custom transport.
|
||||||
|
func (c *Client) SetCustomTransport(customHTTPTransport http.RoundTripper) {
|
||||||
|
// Set this to override default transport
|
||||||
|
// ``http.DefaultTransport``.
|
||||||
|
//
|
||||||
|
// This transport is usually needed for debugging OR to add your
|
||||||
|
// own custom TLS certificates on the client transport, for custom
|
||||||
|
// CA's and certs which are not part of standard certificate
|
||||||
|
// authority follow this example :-
|
||||||
|
//
|
||||||
|
// tr := &http.Transport{
|
||||||
|
// TLSClientConfig: &tls.Config{RootCAs: pool},
|
||||||
|
// DisableCompression: true,
|
||||||
|
// }
|
||||||
|
// api.SetTransport(tr)
|
||||||
|
//
|
||||||
|
if c.httpClient != nil {
|
||||||
|
c.httpClient.Transport = customHTTPTransport
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TraceOn - enable HTTP tracing.
|
||||||
|
func (c *Client) TraceOn(outputStream io.Writer) error {
|
||||||
|
// if outputStream is nil then default to os.Stdout.
|
||||||
|
if outputStream == nil {
|
||||||
|
outputStream = os.Stdout
|
||||||
|
}
|
||||||
|
// Sets a new output stream.
|
||||||
|
c.traceOutput = outputStream
|
||||||
|
|
||||||
|
// Enable tracing.
|
||||||
|
c.isTraceEnabled = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TraceOff - disable HTTP tracing.
|
||||||
|
func (c *Client) TraceOff() {
|
||||||
|
// Disable tracing.
|
||||||
|
c.isTraceEnabled = false
|
||||||
|
}
|
||||||
|
|
||||||
|
// requestMetadata - is container for all the values to make a
|
||||||
|
// request.
|
||||||
|
type requestMetadata struct {
|
||||||
|
// If set newRequest presigns the URL.
|
||||||
|
presignURL bool
|
||||||
|
|
||||||
|
// User supplied.
|
||||||
|
bucketName string
|
||||||
|
objectName string
|
||||||
|
queryValues url.Values
|
||||||
|
customHeader http.Header
|
||||||
|
expires int64
|
||||||
|
|
||||||
|
// Generated by our internal code.
|
||||||
|
bucketLocation string
|
||||||
|
contentBody io.ReadCloser
|
||||||
|
contentLength int64
|
||||||
|
contentSHA256Bytes []byte
|
||||||
|
contentMD5Bytes []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
// Filter out signature value from Authorization header.
|
||||||
|
func (c Client) filterSignature(req *http.Request) {
|
||||||
|
// For anonymous requests, no need to filter.
|
||||||
|
if c.anonymous {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Handle if Signature V2.
|
||||||
|
if c.signature.isV2() {
|
||||||
|
// Set a temporary redacted auth
|
||||||
|
req.Header.Set("Authorization", "AWS **REDACTED**:**REDACTED**")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Signature V4 authorization header.
|
||||||
|
|
||||||
|
// Save the original auth.
|
||||||
|
origAuth := req.Header.Get("Authorization")
|
||||||
|
// Strip out accessKeyID from:
|
||||||
|
// Credential=<access-key-id>/<date>/<aws-region>/<aws-service>/aws4_request
|
||||||
|
regCred := regexp.MustCompile("Credential=([A-Z0-9]+)/")
|
||||||
|
newAuth := regCred.ReplaceAllString(origAuth, "Credential=**REDACTED**/")
|
||||||
|
|
||||||
|
// Strip out 256-bit signature from: Signature=<256-bit signature>
|
||||||
|
regSign := regexp.MustCompile("Signature=([[0-9a-f]+)")
|
||||||
|
newAuth = regSign.ReplaceAllString(newAuth, "Signature=**REDACTED**")
|
||||||
|
|
||||||
|
// Set a temporary redacted auth
|
||||||
|
req.Header.Set("Authorization", newAuth)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// dumpHTTP - dump HTTP request and response.
|
||||||
|
func (c Client) dumpHTTP(req *http.Request, resp *http.Response) error {
|
||||||
|
// Starts http dump.
|
||||||
|
_, err := fmt.Fprintln(c.traceOutput, "---------START-HTTP---------")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Filter out Signature field from Authorization header.
|
||||||
|
c.filterSignature(req)
|
||||||
|
|
||||||
|
// Only display request header.
|
||||||
|
reqTrace, err := httputil.DumpRequestOut(req, false)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write request to trace output.
|
||||||
|
_, err = fmt.Fprint(c.traceOutput, string(reqTrace))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only display response header.
|
||||||
|
var respTrace []byte
|
||||||
|
|
||||||
|
// For errors we make sure to dump response body as well.
|
||||||
|
if resp.StatusCode != http.StatusOK &&
|
||||||
|
resp.StatusCode != http.StatusPartialContent &&
|
||||||
|
resp.StatusCode != http.StatusNoContent {
|
||||||
|
respTrace, err = httputil.DumpResponse(resp, true)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// WORKAROUND for https://github.com/golang/go/issues/13942.
|
||||||
|
// httputil.DumpResponse does not print response headers for
|
||||||
|
// all successful calls which have response ContentLength set
|
||||||
|
// to zero. Keep this workaround until the above bug is fixed.
|
||||||
|
if resp.ContentLength == 0 {
|
||||||
|
var buffer bytes.Buffer
|
||||||
|
if err := resp.Header.Write(&buffer); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
respTrace = buffer.Bytes()
|
||||||
|
respTrace = append(respTrace, []byte("\r\n")...)
|
||||||
|
} else {
|
||||||
|
respTrace, err = httputil.DumpResponse(resp, false)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Write response to trace output.
|
||||||
|
_, err = fmt.Fprint(c.traceOutput, strings.TrimSuffix(string(respTrace), "\r\n"))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ends the http dump.
|
||||||
|
_, err = fmt.Fprintln(c.traceOutput, "---------END-HTTP---------")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Returns success.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// do - execute http request.
|
||||||
|
func (c Client) do(req *http.Request) (*http.Response, error) {
|
||||||
|
// execute the request.
|
||||||
|
resp, err := c.httpClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return resp, err
|
||||||
|
}
|
||||||
|
// If trace is enabled, dump http request and response.
|
||||||
|
if c.isTraceEnabled {
|
||||||
|
err = c.dumpHTTP(req, resp)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// newRequest - instantiate a new HTTP request for a given method.
|
||||||
|
func (c Client) newRequest(method string, metadata requestMetadata) (req *http.Request, err error) {
|
||||||
|
// If no method is supplied default to 'POST'.
|
||||||
|
if method == "" {
|
||||||
|
method = "POST"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Gather location only if bucketName is present.
|
||||||
|
location := "us-east-1" // Default all other requests to "us-east-1".
|
||||||
|
if metadata.bucketName != "" {
|
||||||
|
location, err = c.getBucketLocation(metadata.bucketName)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save location.
|
||||||
|
metadata.bucketLocation = location
|
||||||
|
|
||||||
|
// Construct a new target URL.
|
||||||
|
targetURL, err := c.makeTargetURL(metadata.bucketName, metadata.objectName, metadata.bucketLocation, metadata.queryValues)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize a new HTTP request for the method.
|
||||||
|
req, err = http.NewRequest(method, targetURL.String(), nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate presign url if needed, return right here.
|
||||||
|
if metadata.expires != 0 && metadata.presignURL {
|
||||||
|
if c.anonymous {
|
||||||
|
return nil, ErrInvalidArgument("Requests cannot be presigned with anonymous credentials.")
|
||||||
|
}
|
||||||
|
if c.signature.isV2() {
|
||||||
|
// Presign URL with signature v2.
|
||||||
|
req = preSignV2(*req, c.accessKeyID, c.secretAccessKey, metadata.expires)
|
||||||
|
} else {
|
||||||
|
// Presign URL with signature v4.
|
||||||
|
req = preSignV4(*req, c.accessKeyID, c.secretAccessKey, location, metadata.expires)
|
||||||
|
}
|
||||||
|
return req, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set content body if available.
|
||||||
|
if metadata.contentBody != nil {
|
||||||
|
req.Body = metadata.contentBody
|
||||||
|
}
|
||||||
|
|
||||||
|
// set UserAgent for the request.
|
||||||
|
c.setUserAgent(req)
|
||||||
|
|
||||||
|
// Set all headers.
|
||||||
|
for k, v := range metadata.customHeader {
|
||||||
|
req.Header.Set(k, v[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
// set incoming content-length.
|
||||||
|
if metadata.contentLength > 0 {
|
||||||
|
req.ContentLength = metadata.contentLength
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set sha256 sum only for non anonymous credentials.
|
||||||
|
if !c.anonymous {
|
||||||
|
// set sha256 sum for signature calculation only with
|
||||||
|
// signature version '4'.
|
||||||
|
if c.signature.isV4() {
|
||||||
|
req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256([]byte{})))
|
||||||
|
if metadata.contentSHA256Bytes != nil {
|
||||||
|
req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(metadata.contentSHA256Bytes))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// set md5Sum for content protection.
|
||||||
|
if metadata.contentMD5Bytes != nil {
|
||||||
|
req.Header.Set("Content-MD5", base64.StdEncoding.EncodeToString(metadata.contentMD5Bytes))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sign the request for all authenticated requests.
|
||||||
|
if !c.anonymous {
|
||||||
|
if c.signature.isV2() {
|
||||||
|
// Add signature version '2' authorization header.
|
||||||
|
req = signV2(*req, c.accessKeyID, c.secretAccessKey)
|
||||||
|
} else if c.signature.isV4() {
|
||||||
|
// Add signature version '4' authorization header.
|
||||||
|
req = signV4(*req, c.accessKeyID, c.secretAccessKey, location)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return request.
|
||||||
|
return req, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// set User agent.
|
||||||
|
func (c Client) setUserAgent(req *http.Request) {
|
||||||
|
req.Header.Set("User-Agent", libraryUserAgent)
|
||||||
|
if c.appInfo.appName != "" && c.appInfo.appVersion != "" {
|
||||||
|
req.Header.Set("User-Agent", libraryUserAgent+" "+c.appInfo.appName+"/"+c.appInfo.appVersion)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// makeTargetURL make a new target url.
|
||||||
|
func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, queryValues url.Values) (*url.URL, error) {
|
||||||
|
// Save host.
|
||||||
|
host := c.endpointURL.Host
|
||||||
|
// For Amazon S3 endpoint, try to fetch location based endpoint.
|
||||||
|
if isAmazonEndpoint(c.endpointURL) {
|
||||||
|
// Fetch new host based on the bucket location.
|
||||||
|
host = getS3Endpoint(bucketLocation)
|
||||||
|
}
|
||||||
|
// Save scheme.
|
||||||
|
scheme := c.endpointURL.Scheme
|
||||||
|
|
||||||
|
urlStr := scheme + "://" + host + "/"
|
||||||
|
// Make URL only if bucketName is available, otherwise use the
|
||||||
|
// endpoint URL.
|
||||||
|
if bucketName != "" {
|
||||||
|
// Save if target url will have buckets which suppport virtual host.
|
||||||
|
isVirtualHostStyle := isVirtualHostSupported(c.endpointURL, bucketName)
|
||||||
|
|
||||||
|
// If endpoint supports virtual host style use that always.
|
||||||
|
// Currently only S3 and Google Cloud Storage would support
|
||||||
|
// virtual host style.
|
||||||
|
if isVirtualHostStyle {
|
||||||
|
urlStr = scheme + "://" + bucketName + "." + host + "/"
|
||||||
|
if objectName != "" {
|
||||||
|
urlStr = urlStr + urlEncodePath(objectName)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// If not fall back to using path style.
|
||||||
|
urlStr = urlStr + bucketName
|
||||||
|
if objectName != "" {
|
||||||
|
urlStr = urlStr + "/" + urlEncodePath(objectName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// If there are any query values, add them to the end.
|
||||||
|
if len(queryValues) > 0 {
|
||||||
|
urlStr = urlStr + "?" + queryValues.Encode()
|
||||||
|
}
|
||||||
|
u, err := url.Parse(urlStr)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return u, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CloudStorageClient - Cloud Storage Client interface.
|
||||||
|
type CloudStorageClient interface {
|
||||||
|
// Bucket Read/Write/Stat operations.
|
||||||
|
MakeBucket(bucketName string, cannedACL BucketACL, location string) error
|
||||||
|
BucketExists(bucketName string) error
|
||||||
|
RemoveBucket(bucketName string) error
|
||||||
|
SetBucketACL(bucketName string, cannedACL BucketACL) error
|
||||||
|
GetBucketACL(bucketName string) (BucketACL, error)
|
||||||
|
|
||||||
|
ListBuckets() ([]BucketInfo, error)
|
||||||
|
ListObjects(bucket, prefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo
|
||||||
|
ListIncompleteUploads(bucket, prefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo
|
||||||
|
|
||||||
|
// Object Read/Write/Stat operations.
|
||||||
|
GetObject(bucketName, objectName string) (reader *Object, err error)
|
||||||
|
PutObject(bucketName, objectName string, reader io.Reader, contentType string) (n int64, err error)
|
||||||
|
StatObject(bucketName, objectName string) (ObjectInfo, error)
|
||||||
|
RemoveObject(bucketName, objectName string) error
|
||||||
|
RemoveIncompleteUpload(bucketName, objectName string) error
|
||||||
|
|
||||||
|
// File to Object API.
|
||||||
|
FPutObject(bucketName, objectName, filePath, contentType string) (n int64, err error)
|
||||||
|
FGetObject(bucketName, objectName, filePath string) error
|
||||||
|
|
||||||
|
// PutObjectWithProgress for progress.
|
||||||
|
PutObjectWithProgress(bucketName, objectName string, reader io.Reader, contentType string, progress io.Reader) (n int64, err error)
|
||||||
|
|
||||||
|
// Presigned operations.
|
||||||
|
PresignedGetObject(bucketName, objectName string, expires time.Duration) (presignedURL string, err error)
|
||||||
|
PresignedPutObject(bucketName, objectName string, expires time.Duration) (presignedURL string, err error)
|
||||||
|
PresignedPostPolicy(*PostPolicy) (formData map[string]string, err error)
|
||||||
|
|
||||||
|
// Application info.
|
||||||
|
SetAppInfo(appName, appVersion string)
|
||||||
|
|
||||||
|
// Set custom transport.
|
||||||
|
SetCustomTransport(customTransport http.RoundTripper)
|
||||||
|
|
||||||
|
// HTTP tracing methods.
|
||||||
|
TraceOn(traceOutput io.Writer) error
|
||||||
|
TraceOff()
|
||||||
|
}
|
945
vendor/github.com/minio/minio-go/api_functional_v2_test.go
generated
vendored
Normal file
945
vendor/github.com/minio/minio-go/api_functional_v2_test.go
generated
vendored
Normal file
@ -0,0 +1,945 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
crand "crypto/rand"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
"math/rand"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio-go"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGetObjectClosedTwiceV2(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("skipping functional tests for short runs")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seed random based on current time.
|
||||||
|
rand.Seed(time.Now().Unix())
|
||||||
|
|
||||||
|
// Connect and make sure bucket exists.
|
||||||
|
c, err := minio.New(
|
||||||
|
"s3.amazonaws.com",
|
||||||
|
os.Getenv("ACCESS_KEY"),
|
||||||
|
os.Getenv("SECRET_KEY"),
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enable tracing, write to stderr.
|
||||||
|
// c.TraceOn(os.Stderr)
|
||||||
|
|
||||||
|
// Set user agent.
|
||||||
|
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||||
|
|
||||||
|
// Generate a new random bucket name.
|
||||||
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||||
|
|
||||||
|
// Make a new bucket.
|
||||||
|
err = c.MakeBucket(bucketName, "private", "us-east-1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, bucketName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate data more than 32K
|
||||||
|
buf := make([]byte, rand.Intn(1<<20)+32*1024)
|
||||||
|
|
||||||
|
_, err = io.ReadFull(crand.Reader, buf)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save the data
|
||||||
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||||
|
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, bucketName, objectName)
|
||||||
|
}
|
||||||
|
|
||||||
|
if n != int64(len(buf)) {
|
||||||
|
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read the data back
|
||||||
|
r, err := c.GetObject(bucketName, objectName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, bucketName, objectName)
|
||||||
|
}
|
||||||
|
|
||||||
|
st, err := r.Stat()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, bucketName, objectName)
|
||||||
|
}
|
||||||
|
if st.Size != int64(len(buf)) {
|
||||||
|
t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n",
|
||||||
|
len(buf), st.Size)
|
||||||
|
}
|
||||||
|
if err := r.Close(); err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if err := r.Close(); err == nil {
|
||||||
|
t.Fatal("Error: object is already closed, should return error")
|
||||||
|
}
|
||||||
|
|
||||||
|
err = c.RemoveObject(bucketName, objectName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
err = c.RemoveBucket(bucketName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tests removing partially uploaded objects.
|
||||||
|
func TestRemovePartiallyUploadedV2(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("skipping function tests for short runs")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seed random based on current time.
|
||||||
|
rand.Seed(time.Now().Unix())
|
||||||
|
|
||||||
|
// Connect and make sure bucket exists.
|
||||||
|
c, err := minio.NewV2(
|
||||||
|
"s3.amazonaws.com",
|
||||||
|
os.Getenv("ACCESS_KEY"),
|
||||||
|
os.Getenv("SECRET_KEY"),
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set user agent.
|
||||||
|
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||||
|
|
||||||
|
// Enable tracing, write to stdout.
|
||||||
|
// c.TraceOn(os.Stderr)
|
||||||
|
|
||||||
|
// Generate a new random bucket name.
|
||||||
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||||
|
|
||||||
|
// make a new bucket.
|
||||||
|
err = c.MakeBucket(bucketName, "private", "us-east-1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, bucketName)
|
||||||
|
}
|
||||||
|
|
||||||
|
reader, writer := io.Pipe()
|
||||||
|
go func() {
|
||||||
|
i := 0
|
||||||
|
for i < 25 {
|
||||||
|
_, err = io.CopyN(writer, crand.Reader, 128*1024)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, bucketName)
|
||||||
|
}
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
writer.CloseWithError(errors.New("Proactively closed to be verified later."))
|
||||||
|
}()
|
||||||
|
|
||||||
|
objectName := bucketName + "-resumable"
|
||||||
|
_, err = c.PutObject(bucketName, objectName, reader, "application/octet-stream")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Error: PutObject should fail.")
|
||||||
|
}
|
||||||
|
if err.Error() != "Proactively closed to be verified later." {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
err = c.RemoveIncompleteUpload(bucketName, objectName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
err = c.RemoveBucket(bucketName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tests resumable put object cloud to cloud.
|
||||||
|
func TestResumbalePutObjectV2(t *testing.T) {
|
||||||
|
// By passing 'go test -short' skips these tests.
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("skipping functional tests for the short runs")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seed random based on current time.
|
||||||
|
rand.Seed(time.Now().Unix())
|
||||||
|
|
||||||
|
// Connect and make sure bucket exists.
|
||||||
|
c, err := minio.New(
|
||||||
|
"s3.amazonaws.com",
|
||||||
|
os.Getenv("ACCESS_KEY"),
|
||||||
|
os.Getenv("SECRET_KEY"),
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set user agent.
|
||||||
|
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||||
|
|
||||||
|
// Enable tracing, write to stdout.
|
||||||
|
// c.TraceOn(os.Stderr)
|
||||||
|
|
||||||
|
// Generate a new random bucket name.
|
||||||
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||||
|
|
||||||
|
// Make a new bucket.
|
||||||
|
err = c.MakeBucket(bucketName, "private", "us-east-1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, bucketName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a temporary file.
|
||||||
|
file, err := ioutil.TempFile(os.TempDir(), "resumable")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Copy 11MiB worth of random data.
|
||||||
|
n, err := io.CopyN(file, crand.Reader, 11*1024*1024)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if n != int64(11*1024*1024) {
|
||||||
|
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close the file pro-actively for windows.
|
||||||
|
if err = file.Close(); err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// New object name.
|
||||||
|
objectName := bucketName + "-resumable"
|
||||||
|
|
||||||
|
// Upload the file.
|
||||||
|
n, err = c.FPutObject(bucketName, objectName, file.Name(), "application/octet-stream")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if n != int64(11*1024*1024) {
|
||||||
|
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the uploaded object.
|
||||||
|
reader, err := c.GetObject(bucketName, objectName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upload now cloud to cloud.
|
||||||
|
n, err = c.PutObject(bucketName, objectName+"-put", reader, "application/octest-stream")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get object info.
|
||||||
|
objInfo, err := reader.Stat()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if n != objInfo.Size {
|
||||||
|
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", objInfo.Size, n)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove all temp files, objects and bucket.
|
||||||
|
err = c.RemoveObject(bucketName, objectName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = c.RemoveObject(bucketName, objectName+"-put")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = c.RemoveBucket(bucketName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = os.Remove(file.Name())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tests resumable file based put object multipart upload.
|
||||||
|
func TestResumableFPutObjectV2(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("skipping functional tests for the short runs")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seed random based on current time.
|
||||||
|
rand.Seed(time.Now().Unix())
|
||||||
|
|
||||||
|
// Connect and make sure bucket exists.
|
||||||
|
c, err := minio.New(
|
||||||
|
"s3.amazonaws.com",
|
||||||
|
os.Getenv("ACCESS_KEY"),
|
||||||
|
os.Getenv("SECRET_KEY"),
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set user agent.
|
||||||
|
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||||
|
|
||||||
|
// Enable tracing, write to stdout.
|
||||||
|
// c.TraceOn(os.Stderr)
|
||||||
|
|
||||||
|
// Generate a new random bucket name.
|
||||||
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||||
|
|
||||||
|
// make a new bucket.
|
||||||
|
err = c.MakeBucket(bucketName, "private", "us-east-1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, bucketName)
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := ioutil.TempFile(os.TempDir(), "resumable")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
n, err := io.CopyN(file, crand.Reader, 11*1024*1024)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if n != int64(11*1024*1024) {
|
||||||
|
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n)
|
||||||
|
}
|
||||||
|
|
||||||
|
objectName := bucketName + "-resumable"
|
||||||
|
|
||||||
|
n, err = c.FPutObject(bucketName, objectName, file.Name(), "application/octet-stream")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if n != int64(11*1024*1024) {
|
||||||
|
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close the file pro-actively for windows.
|
||||||
|
file.Close()
|
||||||
|
|
||||||
|
err = c.RemoveObject(bucketName, objectName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = c.RemoveBucket(bucketName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = os.Remove(file.Name())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tests resumable put object multipart upload.
|
||||||
|
func TestResumablePutObjectV2(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("skipping functional tests for the short runs")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seed random based on current time.
|
||||||
|
rand.Seed(time.Now().Unix())
|
||||||
|
|
||||||
|
// Connect and make sure bucket exists.
|
||||||
|
c, err := minio.New(
|
||||||
|
"s3.amazonaws.com",
|
||||||
|
os.Getenv("ACCESS_KEY"),
|
||||||
|
os.Getenv("SECRET_KEY"),
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enable tracing, write to stderr.
|
||||||
|
// c.TraceOn(os.Stderr)
|
||||||
|
|
||||||
|
// Set user agent.
|
||||||
|
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||||
|
|
||||||
|
// Generate a new random bucket name.
|
||||||
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||||
|
|
||||||
|
// make a new bucket.
|
||||||
|
err = c.MakeBucket(bucketName, "private", "us-east-1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, bucketName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// generate 11MB
|
||||||
|
buf := make([]byte, 11*1024*1024)
|
||||||
|
|
||||||
|
_, err = io.ReadFull(crand.Reader, buf)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
objectName := bucketName + "-resumable"
|
||||||
|
reader := bytes.NewReader(buf)
|
||||||
|
n, err := c.PutObject(bucketName, objectName, reader, "application/octet-stream")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, bucketName, objectName)
|
||||||
|
}
|
||||||
|
if n != int64(len(buf)) {
|
||||||
|
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = c.RemoveObject(bucketName, objectName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = c.RemoveBucket(bucketName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tests get object ReaderSeeker interface methods.
|
||||||
|
func TestGetObjectReadSeekFunctionalV2(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("skipping functional tests for short runs")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seed random based on current time.
|
||||||
|
rand.Seed(time.Now().Unix())
|
||||||
|
|
||||||
|
// Connect and make sure bucket exists.
|
||||||
|
c, err := minio.New(
|
||||||
|
"s3.amazonaws.com",
|
||||||
|
os.Getenv("ACCESS_KEY"),
|
||||||
|
os.Getenv("SECRET_KEY"),
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enable tracing, write to stderr.
|
||||||
|
// c.TraceOn(os.Stderr)
|
||||||
|
|
||||||
|
// Set user agent.
|
||||||
|
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||||
|
|
||||||
|
// Generate a new random bucket name.
|
||||||
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||||
|
|
||||||
|
// Make a new bucket.
|
||||||
|
err = c.MakeBucket(bucketName, "private", "us-east-1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, bucketName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate data more than 32K
|
||||||
|
buf := make([]byte, rand.Intn(1<<20)+32*1024)
|
||||||
|
|
||||||
|
_, err = io.ReadFull(crand.Reader, buf)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save the data
|
||||||
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||||
|
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, bucketName, objectName)
|
||||||
|
}
|
||||||
|
|
||||||
|
if n != int64(len(buf)) {
|
||||||
|
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read the data back
|
||||||
|
r, err := c.GetObject(bucketName, objectName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, bucketName, objectName)
|
||||||
|
}
|
||||||
|
|
||||||
|
st, err := r.Stat()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, bucketName, objectName)
|
||||||
|
}
|
||||||
|
if st.Size != int64(len(buf)) {
|
||||||
|
t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n",
|
||||||
|
len(buf), st.Size)
|
||||||
|
}
|
||||||
|
|
||||||
|
offset := int64(2048)
|
||||||
|
n, err = r.Seek(offset, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, offset)
|
||||||
|
}
|
||||||
|
if n != offset {
|
||||||
|
t.Fatalf("Error: number of bytes seeked does not match, want %v, got %v\n",
|
||||||
|
offset, n)
|
||||||
|
}
|
||||||
|
n, err = r.Seek(0, 1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if n != offset {
|
||||||
|
t.Fatalf("Error: number of current seek does not match, want %v, got %v\n",
|
||||||
|
offset, n)
|
||||||
|
}
|
||||||
|
_, err = r.Seek(offset, 2)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Error: seek on positive offset for whence '2' should error out")
|
||||||
|
}
|
||||||
|
n, err = r.Seek(-offset, 2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if n != 0 {
|
||||||
|
t.Fatalf("Error: number of bytes seeked back does not match, want 0, got %v\n", n)
|
||||||
|
}
|
||||||
|
var buffer bytes.Buffer
|
||||||
|
if _, err = io.CopyN(&buffer, r, st.Size); err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(buf, buffer.Bytes()) {
|
||||||
|
t.Fatal("Error: Incorrect read bytes v/s original buffer.")
|
||||||
|
}
|
||||||
|
err = c.RemoveObject(bucketName, objectName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
err = c.RemoveBucket(bucketName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tests get object ReaderAt interface methods.
|
||||||
|
func TestGetObjectReadAtFunctionalV2(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("skipping functional tests for the short runs")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seed random based on current time.
|
||||||
|
rand.Seed(time.Now().Unix())
|
||||||
|
|
||||||
|
// Connect and make sure bucket exists.
|
||||||
|
c, err := minio.New(
|
||||||
|
"s3.amazonaws.com",
|
||||||
|
os.Getenv("ACCESS_KEY"),
|
||||||
|
os.Getenv("SECRET_KEY"),
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enable tracing, write to stderr.
|
||||||
|
// c.TraceOn(os.Stderr)
|
||||||
|
|
||||||
|
// Set user agent.
|
||||||
|
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||||
|
|
||||||
|
// Generate a new random bucket name.
|
||||||
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||||
|
|
||||||
|
// Make a new bucket.
|
||||||
|
err = c.MakeBucket(bucketName, "private", "us-east-1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, bucketName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate data more than 32K
|
||||||
|
buf := make([]byte, rand.Intn(1<<20)+32*1024)
|
||||||
|
|
||||||
|
_, err = io.ReadFull(crand.Reader, buf)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save the data
|
||||||
|
objectName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||||
|
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, bucketName, objectName)
|
||||||
|
}
|
||||||
|
|
||||||
|
if n != int64(len(buf)) {
|
||||||
|
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read the data back
|
||||||
|
r, err := c.GetObject(bucketName, objectName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, bucketName, objectName)
|
||||||
|
}
|
||||||
|
|
||||||
|
st, err := r.Stat()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, bucketName, objectName)
|
||||||
|
}
|
||||||
|
if st.Size != int64(len(buf)) {
|
||||||
|
t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n",
|
||||||
|
len(buf), st.Size)
|
||||||
|
}
|
||||||
|
|
||||||
|
offset := int64(2048)
|
||||||
|
|
||||||
|
// Read directly
|
||||||
|
buf2 := make([]byte, 512)
|
||||||
|
buf3 := make([]byte, 512)
|
||||||
|
buf4 := make([]byte, 512)
|
||||||
|
|
||||||
|
m, err := r.ReadAt(buf2, offset)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, st.Size, len(buf2), offset)
|
||||||
|
}
|
||||||
|
if m != len(buf2) {
|
||||||
|
t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf2))
|
||||||
|
}
|
||||||
|
if !bytes.Equal(buf2, buf[offset:offset+512]) {
|
||||||
|
t.Fatal("Error: Incorrect read between two ReadAt from same offset.")
|
||||||
|
}
|
||||||
|
offset += 512
|
||||||
|
m, err = r.ReadAt(buf3, offset)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, st.Size, len(buf3), offset)
|
||||||
|
}
|
||||||
|
if m != len(buf3) {
|
||||||
|
t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf3))
|
||||||
|
}
|
||||||
|
if !bytes.Equal(buf3, buf[offset:offset+512]) {
|
||||||
|
t.Fatal("Error: Incorrect read between two ReadAt from same offset.")
|
||||||
|
}
|
||||||
|
offset += 512
|
||||||
|
m, err = r.ReadAt(buf4, offset)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, st.Size, len(buf4), offset)
|
||||||
|
}
|
||||||
|
if m != len(buf4) {
|
||||||
|
t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf4))
|
||||||
|
}
|
||||||
|
if !bytes.Equal(buf4, buf[offset:offset+512]) {
|
||||||
|
t.Fatal("Error: Incorrect read between two ReadAt from same offset.")
|
||||||
|
}
|
||||||
|
|
||||||
|
buf5 := make([]byte, n)
|
||||||
|
// Read the whole object.
|
||||||
|
m, err = r.ReadAt(buf5, 0)
|
||||||
|
if err != nil {
|
||||||
|
if err != io.EOF {
|
||||||
|
t.Fatal("Error:", err, len(buf5))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if m != len(buf5) {
|
||||||
|
t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf5))
|
||||||
|
}
|
||||||
|
if !bytes.Equal(buf, buf5) {
|
||||||
|
t.Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.")
|
||||||
|
}
|
||||||
|
|
||||||
|
buf6 := make([]byte, n+1)
|
||||||
|
// Read the whole object and beyond.
|
||||||
|
_, err = r.ReadAt(buf6, 0)
|
||||||
|
if err != nil {
|
||||||
|
if err != io.EOF {
|
||||||
|
t.Fatal("Error:", err, len(buf6))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
err = c.RemoveObject(bucketName, objectName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
err = c.RemoveBucket(bucketName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tests comprehensive list of all methods.
|
||||||
|
func TestFunctionalV2(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("skipping functional tests for the short runs")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seed random based on current time.
|
||||||
|
rand.Seed(time.Now().Unix())
|
||||||
|
|
||||||
|
c, err := minio.New(
|
||||||
|
"s3.amazonaws.com",
|
||||||
|
os.Getenv("ACCESS_KEY"),
|
||||||
|
os.Getenv("SECRET_KEY"),
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enable to debug
|
||||||
|
// c.TraceOn(os.Stderr)
|
||||||
|
|
||||||
|
// Set user agent.
|
||||||
|
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||||
|
|
||||||
|
// Generate a new random bucket name.
|
||||||
|
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||||
|
|
||||||
|
// Make a new bucket.
|
||||||
|
err = c.MakeBucket(bucketName, "private", "us-east-1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, bucketName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate a random file name.
|
||||||
|
fileName := randString(60, rand.NewSource(time.Now().UnixNano()))
|
||||||
|
file, err := os.Create(fileName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
buf := make([]byte, rand.Intn(1<<19))
|
||||||
|
_, err = file.Write(buf)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
file.Close()
|
||||||
|
|
||||||
|
// Verify if bucket exits and you have access.
|
||||||
|
err = c.BucketExists(bucketName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, bucketName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Make the bucket 'public read/write'.
|
||||||
|
err = c.SetBucketACL(bucketName, "public-read-write")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the previously set acl.
|
||||||
|
acl, err := c.GetBucketACL(bucketName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ACL must be 'public read/write'.
|
||||||
|
if acl != minio.BucketACL("public-read-write") {
|
||||||
|
t.Fatal("Error:", acl)
|
||||||
|
}
|
||||||
|
|
||||||
|
// List all buckets.
|
||||||
|
buckets, err := c.ListBuckets()
|
||||||
|
if len(buckets) == 0 {
|
||||||
|
t.Fatal("Error: list buckets cannot be empty", buckets)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify if previously created bucket is listed in list buckets.
|
||||||
|
bucketFound := false
|
||||||
|
for _, bucket := range buckets {
|
||||||
|
if bucket.Name == bucketName {
|
||||||
|
bucketFound = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If bucket not found error out.
|
||||||
|
if !bucketFound {
|
||||||
|
t.Fatal("Error: bucket ", bucketName, "not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
objectName := bucketName + "unique"
|
||||||
|
|
||||||
|
// Generate data
|
||||||
|
buf := make([]byte, rand.Intn(1<<19))
|
||||||
|
_, err = io.ReadFull(crand.Reader, buf)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
if n != int64(len(buf)) {
|
||||||
|
t.Fatal("Error: bad length ", n, len(buf))
|
||||||
|
}
|
||||||
|
|
||||||
|
n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), "binary/octet-stream")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err, bucketName, objectName+"-nolength")
|
||||||
|
}
|
||||||
|
|
||||||
|
if n != int64(len(buf)) {
|
||||||
|
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Instantiate a done channel to close all listing.
|
||||||
|
doneCh := make(chan struct{})
|
||||||
|
defer close(doneCh)
|
||||||
|
|
||||||
|
objFound := false
|
||||||
|
isRecursive := true // Recursive is true.
|
||||||
|
for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) {
|
||||||
|
if obj.Key == objectName {
|
||||||
|
objFound = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !objFound {
|
||||||
|
t.Fatal("Error: object " + objectName + " not found.")
|
||||||
|
}
|
||||||
|
|
||||||
|
incompObjNotFound := true
|
||||||
|
for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) {
|
||||||
|
if objIncompl.Key != "" {
|
||||||
|
incompObjNotFound = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !incompObjNotFound {
|
||||||
|
t.Fatal("Error: unexpected dangling incomplete upload found.")
|
||||||
|
}
|
||||||
|
|
||||||
|
newReader, err := c.GetObject(bucketName, objectName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
newReadBytes, err := ioutil.ReadAll(newReader)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !bytes.Equal(newReadBytes, buf) {
|
||||||
|
t.Fatal("Error: bytes mismatch.")
|
||||||
|
}
|
||||||
|
|
||||||
|
err = c.FGetObject(bucketName, objectName, fileName+"-f")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := http.Get(presignedGetURL)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatal("Error: ", resp.Status)
|
||||||
|
}
|
||||||
|
newPresignedBytes, err := ioutil.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(newPresignedBytes, buf) {
|
||||||
|
t.Fatal("Error: bytes mismatch.")
|
||||||
|
}
|
||||||
|
|
||||||
|
presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
buf = make([]byte, rand.Intn(1<<20))
|
||||||
|
_, err = io.ReadFull(crand.Reader, buf)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
req, err := http.NewRequest("PUT", presignedPutURL, bytes.NewReader(buf))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
httpClient := &http.Client{}
|
||||||
|
resp, err = httpClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
newReader, err = c.GetObject(bucketName, objectName+"-presigned")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
newReadBytes, err = ioutil.ReadAll(newReader)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !bytes.Equal(newReadBytes, buf) {
|
||||||
|
t.Fatal("Error: bytes mismatch.")
|
||||||
|
}
|
||||||
|
|
||||||
|
err = c.RemoveObject(bucketName, objectName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
err = c.RemoveObject(bucketName, objectName+"-f")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
err = c.RemoveObject(bucketName, objectName+"-nolength")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
err = c.RemoveObject(bucketName, objectName+"-presigned")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
err = c.RemoveBucket(bucketName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
err = c.RemoveBucket(bucketName)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Error:")
|
||||||
|
}
|
||||||
|
if err.Error() != "The specified bucket does not exist" {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
if err = os.Remove(fileName); err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
if err = os.Remove(fileName + "-f"); err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
}
|
1026
vendor/github.com/minio/minio-go/api_functional_v4_test.go
generated
vendored
Normal file
1026
vendor/github.com/minio/minio-go/api_functional_v4_test.go
generated
vendored
Normal file
File diff suppressed because it is too large
Load Diff
583
vendor/github.com/minio/minio-go/api_unit_test.go
generated
vendored
Normal file
583
vendor/github.com/minio/minio-go/api_unit_test.go
generated
vendored
Normal file
@ -0,0 +1,583 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type customReader struct{}
|
||||||
|
|
||||||
|
func (c *customReader) Read(p []byte) (n int, err error) {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *customReader) Size() (n int64) {
|
||||||
|
return 10
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tests getReaderSize() for various Reader types.
|
||||||
|
func TestGetReaderSize(t *testing.T) {
|
||||||
|
var reader io.Reader
|
||||||
|
size, err := getReaderSize(reader)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if size != -1 {
|
||||||
|
t.Fatal("Reader shouldn't have any length.")
|
||||||
|
}
|
||||||
|
|
||||||
|
bytesReader := bytes.NewReader([]byte("Hello World"))
|
||||||
|
size, err = getReaderSize(bytesReader)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if size != int64(len("Hello World")) {
|
||||||
|
t.Fatalf("Reader length doesn't match got: %v, want: %v", size, len("Hello World"))
|
||||||
|
}
|
||||||
|
|
||||||
|
size, err = getReaderSize(new(customReader))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if size != int64(10) {
|
||||||
|
t.Fatalf("Reader length doesn't match got: %v, want: %v", size, 10)
|
||||||
|
}
|
||||||
|
|
||||||
|
stringsReader := strings.NewReader("Hello World")
|
||||||
|
size, err = getReaderSize(stringsReader)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if size != int64(len("Hello World")) {
|
||||||
|
t.Fatalf("Reader length doesn't match got: %v, want: %v", size, len("Hello World"))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create request channel.
|
||||||
|
reqCh := make(chan readRequest)
|
||||||
|
// Create response channel.
|
||||||
|
resCh := make(chan readResponse)
|
||||||
|
// Create done channel.
|
||||||
|
doneCh := make(chan struct{})
|
||||||
|
// objectInfo.
|
||||||
|
objectInfo := ObjectInfo{Size: 10}
|
||||||
|
objectReader := newObject(reqCh, resCh, doneCh, objectInfo)
|
||||||
|
defer objectReader.Close()
|
||||||
|
|
||||||
|
size, err = getReaderSize(objectReader)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if size != int64(10) {
|
||||||
|
t.Fatalf("Reader length doesn't match got: %v, want: %v", size, 10)
|
||||||
|
}
|
||||||
|
|
||||||
|
fileReader, err := ioutil.TempFile(os.TempDir(), "prefix")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
defer fileReader.Close()
|
||||||
|
defer os.RemoveAll(fileReader.Name())
|
||||||
|
|
||||||
|
size, err = getReaderSize(fileReader)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if size == -1 {
|
||||||
|
t.Fatal("Reader length for file cannot be -1.")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify for standard input, output and error file descriptors.
|
||||||
|
size, err = getReaderSize(os.Stdin)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if size != -1 {
|
||||||
|
t.Fatal("Stdin should have length of -1.")
|
||||||
|
}
|
||||||
|
size, err = getReaderSize(os.Stdout)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if size != -1 {
|
||||||
|
t.Fatal("Stdout should have length of -1.")
|
||||||
|
}
|
||||||
|
size, err = getReaderSize(os.Stderr)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if size != -1 {
|
||||||
|
t.Fatal("Stderr should have length of -1.")
|
||||||
|
}
|
||||||
|
file, err := os.Open(os.TempDir())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
_, err = getReaderSize(file)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Input file as directory should throw an error.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tests valid hosts for location.
|
||||||
|
func TestValidBucketLocation(t *testing.T) {
|
||||||
|
s3Hosts := []struct {
|
||||||
|
bucketLocation string
|
||||||
|
endpoint string
|
||||||
|
}{
|
||||||
|
{"us-east-1", "s3.amazonaws.com"},
|
||||||
|
{"unknown", "s3.amazonaws.com"},
|
||||||
|
{"ap-southeast-1", "s3-ap-southeast-1.amazonaws.com"},
|
||||||
|
}
|
||||||
|
for _, s3Host := range s3Hosts {
|
||||||
|
endpoint := getS3Endpoint(s3Host.bucketLocation)
|
||||||
|
if endpoint != s3Host.endpoint {
|
||||||
|
t.Fatal("Error: invalid bucket location", endpoint)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tests valid bucket names.
|
||||||
|
func TestBucketNames(t *testing.T) {
|
||||||
|
buckets := []struct {
|
||||||
|
name string
|
||||||
|
valid error
|
||||||
|
}{
|
||||||
|
{".mybucket", ErrInvalidBucketName("Bucket name cannot start or end with a '.' dot.")},
|
||||||
|
{"mybucket.", ErrInvalidBucketName("Bucket name cannot start or end with a '.' dot.")},
|
||||||
|
{"mybucket-", ErrInvalidBucketName("Bucket name contains invalid characters.")},
|
||||||
|
{"my", ErrInvalidBucketName("Bucket name cannot be smaller than 3 characters.")},
|
||||||
|
{"", ErrInvalidBucketName("Bucket name cannot be empty.")},
|
||||||
|
{"my..bucket", ErrInvalidBucketName("Bucket name cannot have successive periods.")},
|
||||||
|
{"my.bucket.com", nil},
|
||||||
|
{"my-bucket", nil},
|
||||||
|
{"123my-bucket", nil},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, b := range buckets {
|
||||||
|
err := isValidBucketName(b.name)
|
||||||
|
if err != b.valid {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tests temp file.
|
||||||
|
func TestTempFile(t *testing.T) {
|
||||||
|
tmpFile, err := newTempFile("testing")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
fileName := tmpFile.Name()
|
||||||
|
// Closing temporary file purges the file.
|
||||||
|
err = tmpFile.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
st, err := os.Stat(fileName)
|
||||||
|
if err != nil && !os.IsNotExist(err) {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if err == nil && st != nil {
|
||||||
|
t.Fatal("Error: file should be deleted and should not exist.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tests url encoding.
|
||||||
|
func TestEncodeURL2Path(t *testing.T) {
|
||||||
|
type urlStrings struct {
|
||||||
|
objName string
|
||||||
|
encodedObjName string
|
||||||
|
}
|
||||||
|
|
||||||
|
bucketName := "bucketName"
|
||||||
|
want := []urlStrings{
|
||||||
|
{
|
||||||
|
objName: "本語",
|
||||||
|
encodedObjName: "%E6%9C%AC%E8%AA%9E",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
objName: "本語.1",
|
||||||
|
encodedObjName: "%E6%9C%AC%E8%AA%9E.1",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
objName: ">123>3123123",
|
||||||
|
encodedObjName: "%3E123%3E3123123",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
objName: "test 1 2.txt",
|
||||||
|
encodedObjName: "test%201%202.txt",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
objName: "test++ 1.txt",
|
||||||
|
encodedObjName: "test%2B%2B%201.txt",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, o := range want {
|
||||||
|
u, err := url.Parse(fmt.Sprintf("https://%s.s3.amazonaws.com/%s", bucketName, o.objName))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
urlPath := "/" + bucketName + "/" + o.encodedObjName
|
||||||
|
if urlPath != encodeURL2Path(u) {
|
||||||
|
t.Fatal("Error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tests error response structure.
|
||||||
|
func TestErrorResponse(t *testing.T) {
|
||||||
|
var err error
|
||||||
|
err = ErrorResponse{
|
||||||
|
Code: "Testing",
|
||||||
|
}
|
||||||
|
errResp := ToErrorResponse(err)
|
||||||
|
if errResp.Code != "Testing" {
|
||||||
|
t.Fatal("Type conversion failed, we have an empty struct.")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test http response decoding.
|
||||||
|
var httpResponse *http.Response
|
||||||
|
// Set empty variables
|
||||||
|
httpResponse = nil
|
||||||
|
var bucketName, objectName string
|
||||||
|
|
||||||
|
// Should fail with invalid argument.
|
||||||
|
err = httpRespToErrorResponse(httpResponse, bucketName, objectName)
|
||||||
|
errResp = ToErrorResponse(err)
|
||||||
|
if errResp.Code != "InvalidArgument" {
|
||||||
|
t.Fatal("Empty response input should return invalid argument.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tests signature calculation.
|
||||||
|
func TestSignatureCalculation(t *testing.T) {
|
||||||
|
req, err := http.NewRequest("GET", "https://s3.amazonaws.com", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
req = signV4(*req, "", "", "us-east-1")
|
||||||
|
if req.Header.Get("Authorization") != "" {
|
||||||
|
t.Fatal("Error: anonymous credentials should not have Authorization header.")
|
||||||
|
}
|
||||||
|
|
||||||
|
req = preSignV4(*req, "", "", "us-east-1", 0)
|
||||||
|
if strings.Contains(req.URL.RawQuery, "X-Amz-Signature") {
|
||||||
|
t.Fatal("Error: anonymous credentials should not have Signature query resource.")
|
||||||
|
}
|
||||||
|
|
||||||
|
req = signV2(*req, "", "")
|
||||||
|
if req.Header.Get("Authorization") != "" {
|
||||||
|
t.Fatal("Error: anonymous credentials should not have Authorization header.")
|
||||||
|
}
|
||||||
|
|
||||||
|
req = preSignV2(*req, "", "", 0)
|
||||||
|
if strings.Contains(req.URL.RawQuery, "Signature") {
|
||||||
|
t.Fatal("Error: anonymous credentials should not have Signature query resource.")
|
||||||
|
}
|
||||||
|
|
||||||
|
req = signV4(*req, "ACCESS-KEY", "SECRET-KEY", "us-east-1")
|
||||||
|
if req.Header.Get("Authorization") == "" {
|
||||||
|
t.Fatal("Error: normal credentials should have Authorization header.")
|
||||||
|
}
|
||||||
|
|
||||||
|
req = preSignV4(*req, "ACCESS-KEY", "SECRET-KEY", "us-east-1", 0)
|
||||||
|
if !strings.Contains(req.URL.RawQuery, "X-Amz-Signature") {
|
||||||
|
t.Fatal("Error: normal credentials should have Signature query resource.")
|
||||||
|
}
|
||||||
|
|
||||||
|
req = signV2(*req, "ACCESS-KEY", "SECRET-KEY")
|
||||||
|
if req.Header.Get("Authorization") == "" {
|
||||||
|
t.Fatal("Error: normal credentials should have Authorization header.")
|
||||||
|
}
|
||||||
|
|
||||||
|
req = preSignV2(*req, "ACCESS-KEY", "SECRET-KEY", 0)
|
||||||
|
if !strings.Contains(req.URL.RawQuery, "Signature") {
|
||||||
|
t.Fatal("Error: normal credentials should not have Signature query resource.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tests signature type.
|
||||||
|
func TestSignatureType(t *testing.T) {
|
||||||
|
clnt := Client{}
|
||||||
|
if !clnt.signature.isV4() {
|
||||||
|
t.Fatal("Error")
|
||||||
|
}
|
||||||
|
clnt.signature = SignatureV2
|
||||||
|
if !clnt.signature.isV2() {
|
||||||
|
t.Fatal("Error")
|
||||||
|
}
|
||||||
|
if clnt.signature.isV4() {
|
||||||
|
t.Fatal("Error")
|
||||||
|
}
|
||||||
|
clnt.signature = SignatureV4
|
||||||
|
if !clnt.signature.isV4() {
|
||||||
|
t.Fatal("Error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tests bucket acl types.
|
||||||
|
func TestBucketACLTypes(t *testing.T) {
|
||||||
|
want := map[string]bool{
|
||||||
|
"private": true,
|
||||||
|
"public-read": true,
|
||||||
|
"public-read-write": true,
|
||||||
|
"authenticated-read": true,
|
||||||
|
"invalid": false,
|
||||||
|
}
|
||||||
|
for acl, ok := range want {
|
||||||
|
if BucketACL(acl).isValidBucketACL() != ok {
|
||||||
|
t.Fatal("Error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tests optimal part size.
|
||||||
|
func TestPartSize(t *testing.T) {
|
||||||
|
totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(5000000000000000000)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Error: should fail")
|
||||||
|
}
|
||||||
|
totalPartsCount, partSize, lastPartSize, err = optimalPartInfo(5497558138880)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error: ", err)
|
||||||
|
}
|
||||||
|
if totalPartsCount != 9987 {
|
||||||
|
t.Fatalf("Error: expecting total parts count of 9987: got %v instead", totalPartsCount)
|
||||||
|
}
|
||||||
|
if partSize != 550502400 {
|
||||||
|
t.Fatalf("Error: expecting part size of 550502400: got %v instead", partSize)
|
||||||
|
}
|
||||||
|
if lastPartSize != 241172480 {
|
||||||
|
t.Fatalf("Error: expecting last part size of 241172480: got %v instead", lastPartSize)
|
||||||
|
}
|
||||||
|
totalPartsCount, partSize, lastPartSize, err = optimalPartInfo(5000000000)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if partSize != minPartSize {
|
||||||
|
t.Fatalf("Error: expecting part size of %v: got %v instead", minPartSize, partSize)
|
||||||
|
}
|
||||||
|
totalPartsCount, partSize, lastPartSize, err = optimalPartInfo(-1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if totalPartsCount != 9987 {
|
||||||
|
t.Fatalf("Error: expecting total parts count of 9987: got %v instead", totalPartsCount)
|
||||||
|
}
|
||||||
|
if partSize != 550502400 {
|
||||||
|
t.Fatalf("Error: expecting part size of 550502400: got %v instead", partSize)
|
||||||
|
}
|
||||||
|
if lastPartSize != 241172480 {
|
||||||
|
t.Fatalf("Error: expecting last part size of 241172480: got %v instead", lastPartSize)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tests url encoding.
|
||||||
|
func TestURLEncoding(t *testing.T) {
|
||||||
|
type urlStrings struct {
|
||||||
|
name string
|
||||||
|
encodedName string
|
||||||
|
}
|
||||||
|
|
||||||
|
want := []urlStrings{
|
||||||
|
{
|
||||||
|
name: "bigfile-1._%",
|
||||||
|
encodedName: "bigfile-1._%25",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "本語",
|
||||||
|
encodedName: "%E6%9C%AC%E8%AA%9E",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "本語.1",
|
||||||
|
encodedName: "%E6%9C%AC%E8%AA%9E.1",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: ">123>3123123",
|
||||||
|
encodedName: "%3E123%3E3123123",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "test 1 2.txt",
|
||||||
|
encodedName: "test%201%202.txt",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "test++ 1.txt",
|
||||||
|
encodedName: "test%2B%2B%201.txt",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, u := range want {
|
||||||
|
if u.encodedName != urlEncodePath(u.name) {
|
||||||
|
t.Fatal("Error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tests constructing valid endpoint url.
|
||||||
|
func TestGetEndpointURL(t *testing.T) {
|
||||||
|
if _, err := getEndpointURL("s3.amazonaws.com", false); err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if _, err := getEndpointURL("192.168.1.1", false); err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
if _, err := getEndpointURL("13333.123123.-", false); err == nil {
|
||||||
|
t.Fatal("Error")
|
||||||
|
}
|
||||||
|
if _, err := getEndpointURL("s3.aamzza.-", false); err == nil {
|
||||||
|
t.Fatal("Error")
|
||||||
|
}
|
||||||
|
if _, err := getEndpointURL("s3.amazonaws.com:443", false); err == nil {
|
||||||
|
t.Fatal("Error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tests valid ip address.
|
||||||
|
func TestValidIPAddr(t *testing.T) {
|
||||||
|
type validIP struct {
|
||||||
|
ip string
|
||||||
|
valid bool
|
||||||
|
}
|
||||||
|
|
||||||
|
want := []validIP{
|
||||||
|
{
|
||||||
|
ip: "192.168.1.1",
|
||||||
|
valid: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ip: "192.1.8",
|
||||||
|
valid: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ip: "..192.",
|
||||||
|
valid: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ip: "192.168.1.1.1",
|
||||||
|
valid: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, w := range want {
|
||||||
|
valid := isValidIP(w.ip)
|
||||||
|
if valid != w.valid {
|
||||||
|
t.Fatal("Error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tests valid endpoint domain.
|
||||||
|
func TestValidEndpointDomain(t *testing.T) {
|
||||||
|
type validEndpoint struct {
|
||||||
|
endpointDomain string
|
||||||
|
valid bool
|
||||||
|
}
|
||||||
|
|
||||||
|
want := []validEndpoint{
|
||||||
|
{
|
||||||
|
endpointDomain: "s3.amazonaws.com",
|
||||||
|
valid: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
endpointDomain: "s3.amazonaws.com_",
|
||||||
|
valid: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
endpointDomain: "%$$$",
|
||||||
|
valid: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
endpointDomain: "s3.amz.test.com",
|
||||||
|
valid: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
endpointDomain: "s3.%%",
|
||||||
|
valid: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
endpointDomain: "localhost",
|
||||||
|
valid: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
endpointDomain: "-localhost",
|
||||||
|
valid: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
endpointDomain: "",
|
||||||
|
valid: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
endpointDomain: "\n \t",
|
||||||
|
valid: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
endpointDomain: " ",
|
||||||
|
valid: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, w := range want {
|
||||||
|
valid := isValidDomain(w.endpointDomain)
|
||||||
|
if valid != w.valid {
|
||||||
|
t.Fatal("Error:", w.endpointDomain)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tests valid endpoint url.
|
||||||
|
func TestValidEndpointURL(t *testing.T) {
|
||||||
|
type validURL struct {
|
||||||
|
url string
|
||||||
|
valid bool
|
||||||
|
}
|
||||||
|
want := []validURL{
|
||||||
|
{
|
||||||
|
url: "https://s3.amazonaws.com",
|
||||||
|
valid: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
url: "https://s3.amazonaws.com/bucket/object",
|
||||||
|
valid: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
url: "192.168.1.1",
|
||||||
|
valid: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, w := range want {
|
||||||
|
u, err := url.Parse(w.url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Error:", err)
|
||||||
|
}
|
||||||
|
valid := false
|
||||||
|
if err := isValidEndpointURL(u); err == nil {
|
||||||
|
valid = true
|
||||||
|
}
|
||||||
|
if valid != w.valid {
|
||||||
|
t.Fatal("Error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
36
vendor/github.com/minio/minio-go/appveyor.yml
generated
vendored
Normal file
36
vendor/github.com/minio/minio-go/appveyor.yml
generated
vendored
Normal file
@ -0,0 +1,36 @@
|
|||||||
|
# version format
|
||||||
|
version: "{build}"
|
||||||
|
|
||||||
|
# Operating system (build VM template)
|
||||||
|
os: Windows Server 2012 R2
|
||||||
|
|
||||||
|
clone_folder: c:\gopath\src\github.com\minio\minio-go
|
||||||
|
|
||||||
|
# environment variables
|
||||||
|
environment:
|
||||||
|
GOPATH: c:\gopath
|
||||||
|
GO15VENDOREXPERIMENT: 1
|
||||||
|
|
||||||
|
# scripts that run after cloning repository
|
||||||
|
install:
|
||||||
|
- set PATH=%GOPATH%\bin;c:\go\bin;%PATH%
|
||||||
|
- go version
|
||||||
|
- go env
|
||||||
|
- go get -u github.com/golang/lint/golint
|
||||||
|
- go get -u golang.org/x/tools/cmd/vet
|
||||||
|
- go get -u github.com/remyoudompheng/go-misc/deadcode
|
||||||
|
|
||||||
|
# to run your custom scripts instead of automatic MSBuild
|
||||||
|
build_script:
|
||||||
|
- go vet ./...
|
||||||
|
- gofmt -s -l .
|
||||||
|
- golint github.com/minio/minio-go...
|
||||||
|
- deadcode
|
||||||
|
- go test -short -v
|
||||||
|
- go test -short -race -v
|
||||||
|
|
||||||
|
# to disable automatic tests
|
||||||
|
test: off
|
||||||
|
|
||||||
|
# to disable deployment
|
||||||
|
deploy: off
|
75
vendor/github.com/minio/minio-go/bucket-acl.go
generated
vendored
Normal file
75
vendor/github.com/minio/minio-go/bucket-acl.go
generated
vendored
Normal file
@ -0,0 +1,75 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
// BucketACL - Bucket level access control.
|
||||||
|
type BucketACL string
|
||||||
|
|
||||||
|
// Different types of ACL's currently supported for buckets.
|
||||||
|
const (
|
||||||
|
bucketPrivate = BucketACL("private")
|
||||||
|
bucketReadOnly = BucketACL("public-read")
|
||||||
|
bucketPublic = BucketACL("public-read-write")
|
||||||
|
bucketAuthenticated = BucketACL("authenticated-read")
|
||||||
|
)
|
||||||
|
|
||||||
|
// Stringify acl.
|
||||||
|
func (b BucketACL) String() string {
|
||||||
|
if string(b) == "" {
|
||||||
|
return "private"
|
||||||
|
}
|
||||||
|
return string(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
// isValidBucketACL - Is provided acl string supported.
|
||||||
|
func (b BucketACL) isValidBucketACL() bool {
|
||||||
|
switch true {
|
||||||
|
case b.isPrivate():
|
||||||
|
fallthrough
|
||||||
|
case b.isReadOnly():
|
||||||
|
fallthrough
|
||||||
|
case b.isPublic():
|
||||||
|
fallthrough
|
||||||
|
case b.isAuthenticated():
|
||||||
|
return true
|
||||||
|
case b.String() == "private":
|
||||||
|
// By default its "private"
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// isPrivate - Is acl Private.
|
||||||
|
func (b BucketACL) isPrivate() bool {
|
||||||
|
return b == bucketPrivate
|
||||||
|
}
|
||||||
|
|
||||||
|
// isPublicRead - Is acl PublicRead.
|
||||||
|
func (b BucketACL) isReadOnly() bool {
|
||||||
|
return b == bucketReadOnly
|
||||||
|
}
|
||||||
|
|
||||||
|
// isPublicReadWrite - Is acl PublicReadWrite.
|
||||||
|
func (b BucketACL) isPublic() bool {
|
||||||
|
return b == bucketPublic
|
||||||
|
}
|
||||||
|
|
||||||
|
// isAuthenticated - Is acl AuthenticatedRead.
|
||||||
|
func (b BucketACL) isAuthenticated() bool {
|
||||||
|
return b == bucketAuthenticated
|
||||||
|
}
|
154
vendor/github.com/minio/minio-go/bucket-cache.go
generated
vendored
Normal file
154
vendor/github.com/minio/minio-go/bucket-cache.go
generated
vendored
Normal file
@ -0,0 +1,154 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/hex"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
// bucketLocationCache - Provides simple mechansim to hold bucket
|
||||||
|
// locations in memory.
|
||||||
|
type bucketLocationCache struct {
|
||||||
|
// mutex is used for handling the concurrent
|
||||||
|
// read/write requests for cache.
|
||||||
|
sync.RWMutex
|
||||||
|
|
||||||
|
// items holds the cached bucket locations.
|
||||||
|
items map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
// newBucketLocationCache - Provides a new bucket location cache to be
|
||||||
|
// used internally with the client object.
|
||||||
|
func newBucketLocationCache() *bucketLocationCache {
|
||||||
|
return &bucketLocationCache{
|
||||||
|
items: make(map[string]string),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get - Returns a value of a given key if it exists.
|
||||||
|
func (r *bucketLocationCache) Get(bucketName string) (location string, ok bool) {
|
||||||
|
r.RLock()
|
||||||
|
defer r.RUnlock()
|
||||||
|
location, ok = r.items[bucketName]
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set - Will persist a value into cache.
|
||||||
|
func (r *bucketLocationCache) Set(bucketName string, location string) {
|
||||||
|
r.Lock()
|
||||||
|
defer r.Unlock()
|
||||||
|
r.items[bucketName] = location
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete - Deletes a bucket name from cache.
|
||||||
|
func (r *bucketLocationCache) Delete(bucketName string) {
|
||||||
|
r.Lock()
|
||||||
|
defer r.Unlock()
|
||||||
|
delete(r.items, bucketName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// getBucketLocation - Get location for the bucketName from location map cache.
|
||||||
|
func (c Client) getBucketLocation(bucketName string) (string, error) {
|
||||||
|
// For anonymous requests, default to "us-east-1" and let other calls
|
||||||
|
// move forward.
|
||||||
|
if c.anonymous {
|
||||||
|
return "us-east-1", nil
|
||||||
|
}
|
||||||
|
if location, ok := c.bucketLocCache.Get(bucketName); ok {
|
||||||
|
return location, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize a new request.
|
||||||
|
req, err := c.getBucketLocationRequest(bucketName)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initiate the request.
|
||||||
|
resp, err := c.do(req)
|
||||||
|
defer closeResponse(resp)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if resp != nil {
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return "", httpRespToErrorResponse(resp, bucketName, "")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract location.
|
||||||
|
var locationConstraint string
|
||||||
|
err = xmlDecoder(resp.Body, &locationConstraint)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
location := locationConstraint
|
||||||
|
// Location is empty will be 'us-east-1'.
|
||||||
|
if location == "" {
|
||||||
|
location = "us-east-1"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Location can be 'EU' convert it to meaningful 'eu-west-1'.
|
||||||
|
if location == "EU" {
|
||||||
|
location = "eu-west-1"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save the location into cache.
|
||||||
|
c.bucketLocCache.Set(bucketName, location)
|
||||||
|
|
||||||
|
// Return.
|
||||||
|
return location, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// getBucketLocationRequest - Wrapper creates a new getBucketLocation request.
|
||||||
|
func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, error) {
|
||||||
|
// Set location query.
|
||||||
|
urlValues := make(url.Values)
|
||||||
|
urlValues.Set("location", "")
|
||||||
|
|
||||||
|
// Set get bucket location always as path style.
|
||||||
|
targetURL := c.endpointURL
|
||||||
|
targetURL.Path = filepath.Join(bucketName, "")
|
||||||
|
targetURL.RawQuery = urlValues.Encode()
|
||||||
|
|
||||||
|
// Get a new HTTP request for the method.
|
||||||
|
req, err := http.NewRequest("GET", targetURL.String(), nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set UserAgent for the request.
|
||||||
|
c.setUserAgent(req)
|
||||||
|
|
||||||
|
// Set sha256 sum for signature calculation only with signature version '4'.
|
||||||
|
if c.signature.isV4() {
|
||||||
|
req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256([]byte{})))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sign the request.
|
||||||
|
if c.signature.isV4() {
|
||||||
|
req = signV4(*req, c.accessKeyID, c.secretAccessKey, "us-east-1")
|
||||||
|
} else if c.signature.isV2() {
|
||||||
|
req = signV2(*req, c.accessKeyID, c.secretAccessKey)
|
||||||
|
}
|
||||||
|
return req, nil
|
||||||
|
}
|
42
vendor/github.com/minio/minio-go/constants.go
generated
vendored
Normal file
42
vendor/github.com/minio/minio-go/constants.go
generated
vendored
Normal file
@ -0,0 +1,42 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
/// Multipart upload defaults.
|
||||||
|
|
||||||
|
// miniPartSize - minimum part size 5MiB per object after which
|
||||||
|
// putObject behaves internally as multipart.
|
||||||
|
const minPartSize = 1024 * 1024 * 5
|
||||||
|
|
||||||
|
// maxPartsCount - maximum number of parts for a single multipart session.
|
||||||
|
const maxPartsCount = 10000
|
||||||
|
|
||||||
|
// maxPartSize - maximum part size 5GiB for a single multipart upload
|
||||||
|
// operation.
|
||||||
|
const maxPartSize = 1024 * 1024 * 1024 * 5
|
||||||
|
|
||||||
|
// maxSinglePutObjectSize - maximum size 5GiB of object per PUT
|
||||||
|
// operation.
|
||||||
|
const maxSinglePutObjectSize = 1024 * 1024 * 1024 * 5
|
||||||
|
|
||||||
|
// maxMultipartPutObjectSize - maximum size 5TiB of object for
|
||||||
|
// Multipart operation.
|
||||||
|
const maxMultipartPutObjectSize = 1024 * 1024 * 1024 * 1024 * 5
|
||||||
|
|
||||||
|
// optimalReadBufferSize - optimal buffer 5MiB used for reading
|
||||||
|
// through Read operation.
|
||||||
|
const optimalReadBufferSize = 1024 * 1024 * 5
|
54
vendor/github.com/minio/minio-go/hook-reader.go
generated
vendored
Normal file
54
vendor/github.com/minio/minio-go/hook-reader.go
generated
vendored
Normal file
@ -0,0 +1,54 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import "io"
|
||||||
|
|
||||||
|
// hookReader hooks additional reader in the source stream. It is
|
||||||
|
// useful for making progress bars. Second reader is appropriately
|
||||||
|
// notified about the exact number of bytes read from the primary
|
||||||
|
// source on each Read operation.
|
||||||
|
type hookReader struct {
|
||||||
|
source io.Reader
|
||||||
|
hook io.Reader
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read implements io.Reader. Always reads from the source, the return
|
||||||
|
// value 'n' number of bytes are reported through the hook. Returns
|
||||||
|
// error for all non io.EOF conditions.
|
||||||
|
func (hr *hookReader) Read(b []byte) (n int, err error) {
|
||||||
|
n, err = hr.source.Read(b)
|
||||||
|
if err != nil && err != io.EOF {
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
// Progress the hook with the total read bytes from the source.
|
||||||
|
if _, herr := hr.hook.Read(b[:n]); herr != nil {
|
||||||
|
if herr != io.EOF {
|
||||||
|
return n, herr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// newHook returns a io.Reader which implements hookReader that
|
||||||
|
// reports the data read from the source to the hook.
|
||||||
|
func newHook(source, hook io.Reader) io.Reader {
|
||||||
|
if hook == nil {
|
||||||
|
return source
|
||||||
|
}
|
||||||
|
return &hookReader{source, hook}
|
||||||
|
}
|
191
vendor/github.com/minio/minio-go/post-policy.go
generated
vendored
Normal file
191
vendor/github.com/minio/minio-go/post-policy.go
generated
vendored
Normal file
@ -0,0 +1,191 @@
|
|||||||
|
package minio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/base64"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// expirationDateFormat date format for expiration key in json policy.
|
||||||
|
const expirationDateFormat = "2006-01-02T15:04:05.999Z"
|
||||||
|
|
||||||
|
// policyCondition explanation:
|
||||||
|
// http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html
|
||||||
|
//
|
||||||
|
// Example:
|
||||||
|
//
|
||||||
|
// policyCondition {
|
||||||
|
// matchType: "$eq",
|
||||||
|
// key: "$Content-Type",
|
||||||
|
// value: "image/png",
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
type policyCondition struct {
|
||||||
|
matchType string
|
||||||
|
condition string
|
||||||
|
value string
|
||||||
|
}
|
||||||
|
|
||||||
|
// PostPolicy - Provides strict static type conversion and validation
|
||||||
|
// for Amazon S3's POST policy JSON string.
|
||||||
|
type PostPolicy struct {
|
||||||
|
// Expiration date and time of the POST policy.
|
||||||
|
expiration time.Time
|
||||||
|
// Collection of different policy conditions.
|
||||||
|
conditions []policyCondition
|
||||||
|
// ContentLengthRange minimum and maximum allowable size for the
|
||||||
|
// uploaded content.
|
||||||
|
contentLengthRange struct {
|
||||||
|
min int64
|
||||||
|
max int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Post form data.
|
||||||
|
formData map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewPostPolicy - Instantiate new post policy.
|
||||||
|
func NewPostPolicy() *PostPolicy {
|
||||||
|
p := &PostPolicy{}
|
||||||
|
p.conditions = make([]policyCondition, 0)
|
||||||
|
p.formData = make(map[string]string)
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetExpires - Sets expiration time for the new policy.
|
||||||
|
func (p *PostPolicy) SetExpires(t time.Time) error {
|
||||||
|
if t.IsZero() {
|
||||||
|
return ErrInvalidArgument("No expiry time set.")
|
||||||
|
}
|
||||||
|
p.expiration = t
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetKey - Sets an object name for the policy based upload.
|
||||||
|
func (p *PostPolicy) SetKey(key string) error {
|
||||||
|
if strings.TrimSpace(key) == "" || key == "" {
|
||||||
|
return ErrInvalidArgument("Object name is empty.")
|
||||||
|
}
|
||||||
|
policyCond := policyCondition{
|
||||||
|
matchType: "eq",
|
||||||
|
condition: "$key",
|
||||||
|
value: key,
|
||||||
|
}
|
||||||
|
if err := p.addNewPolicy(policyCond); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p.formData["key"] = key
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetKeyStartsWith - Sets an object name that an policy based upload
|
||||||
|
// can start with.
|
||||||
|
func (p *PostPolicy) SetKeyStartsWith(keyStartsWith string) error {
|
||||||
|
if strings.TrimSpace(keyStartsWith) == "" || keyStartsWith == "" {
|
||||||
|
return ErrInvalidArgument("Object prefix is empty.")
|
||||||
|
}
|
||||||
|
policyCond := policyCondition{
|
||||||
|
matchType: "starts-with",
|
||||||
|
condition: "$key",
|
||||||
|
value: keyStartsWith,
|
||||||
|
}
|
||||||
|
if err := p.addNewPolicy(policyCond); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p.formData["key"] = keyStartsWith
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetBucket - Sets bucket at which objects will be uploaded to.
|
||||||
|
func (p *PostPolicy) SetBucket(bucketName string) error {
|
||||||
|
if strings.TrimSpace(bucketName) == "" || bucketName == "" {
|
||||||
|
return ErrInvalidArgument("Bucket name is empty.")
|
||||||
|
}
|
||||||
|
policyCond := policyCondition{
|
||||||
|
matchType: "eq",
|
||||||
|
condition: "$bucket",
|
||||||
|
value: bucketName,
|
||||||
|
}
|
||||||
|
if err := p.addNewPolicy(policyCond); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p.formData["bucket"] = bucketName
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetContentType - Sets content-type of the object for this policy
|
||||||
|
// based upload.
|
||||||
|
func (p *PostPolicy) SetContentType(contentType string) error {
|
||||||
|
if strings.TrimSpace(contentType) == "" || contentType == "" {
|
||||||
|
return ErrInvalidArgument("No content type specified.")
|
||||||
|
}
|
||||||
|
policyCond := policyCondition{
|
||||||
|
matchType: "eq",
|
||||||
|
condition: "$Content-Type",
|
||||||
|
value: contentType,
|
||||||
|
}
|
||||||
|
if err := p.addNewPolicy(policyCond); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p.formData["Content-Type"] = contentType
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetContentLengthRange - Set new min and max content length
|
||||||
|
// condition for all incoming uploads.
|
||||||
|
func (p *PostPolicy) SetContentLengthRange(min, max int64) error {
|
||||||
|
if min > max {
|
||||||
|
return ErrInvalidArgument("Minimum limit is larger than maximum limit.")
|
||||||
|
}
|
||||||
|
if min < 0 {
|
||||||
|
return ErrInvalidArgument("Minimum limit cannot be negative.")
|
||||||
|
}
|
||||||
|
if max < 0 {
|
||||||
|
return ErrInvalidArgument("Maximum limit cannot be negative.")
|
||||||
|
}
|
||||||
|
p.contentLengthRange.min = min
|
||||||
|
p.contentLengthRange.max = max
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// addNewPolicy - internal helper to validate adding new policies.
|
||||||
|
func (p *PostPolicy) addNewPolicy(policyCond policyCondition) error {
|
||||||
|
if policyCond.matchType == "" || policyCond.condition == "" || policyCond.value == "" {
|
||||||
|
return ErrInvalidArgument("Policy fields are empty.")
|
||||||
|
}
|
||||||
|
p.conditions = append(p.conditions, policyCond)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stringer interface for printing policy in json formatted string.
|
||||||
|
func (p PostPolicy) String() string {
|
||||||
|
return string(p.marshalJSON())
|
||||||
|
}
|
||||||
|
|
||||||
|
// marshalJSON - Provides Marshalled JSON in bytes.
|
||||||
|
func (p PostPolicy) marshalJSON() []byte {
|
||||||
|
expirationStr := `"expiration":"` + p.expiration.Format(expirationDateFormat) + `"`
|
||||||
|
var conditionsStr string
|
||||||
|
conditions := []string{}
|
||||||
|
for _, po := range p.conditions {
|
||||||
|
conditions = append(conditions, fmt.Sprintf("[\"%s\",\"%s\",\"%s\"]", po.matchType, po.condition, po.value))
|
||||||
|
}
|
||||||
|
if p.contentLengthRange.min != 0 || p.contentLengthRange.max != 0 {
|
||||||
|
conditions = append(conditions, fmt.Sprintf("[\"content-length-range\", %d, %d]",
|
||||||
|
p.contentLengthRange.min, p.contentLengthRange.max))
|
||||||
|
}
|
||||||
|
if len(conditions) > 0 {
|
||||||
|
conditionsStr = `"conditions":[` + strings.Join(conditions, ",") + "]"
|
||||||
|
}
|
||||||
|
retStr := "{"
|
||||||
|
retStr = retStr + expirationStr + ","
|
||||||
|
retStr = retStr + conditionsStr
|
||||||
|
retStr = retStr + "}"
|
||||||
|
return []byte(retStr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// base64 - Produces base64 of PostPolicy's Marshalled json.
|
||||||
|
func (p PostPolicy) base64() string {
|
||||||
|
return base64.StdEncoding.EncodeToString(p.marshalJSON())
|
||||||
|
}
|
289
vendor/github.com/minio/minio-go/request-signature-v2.go
generated
vendored
Normal file
289
vendor/github.com/minio/minio-go/request-signature-v2.go
generated
vendored
Normal file
@ -0,0 +1,289 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"crypto/hmac"
|
||||||
|
"crypto/sha1"
|
||||||
|
"encoding/base64"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Signature and API related constants.
|
||||||
|
const (
|
||||||
|
signV2Algorithm = "AWS"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Encode input URL path to URL encoded path.
|
||||||
|
func encodeURL2Path(u *url.URL) (path string) {
|
||||||
|
// Encode URL path.
|
||||||
|
if strings.HasSuffix(u.Host, ".s3.amazonaws.com") {
|
||||||
|
path = "/" + strings.TrimSuffix(u.Host, ".s3.amazonaws.com")
|
||||||
|
path += u.Path
|
||||||
|
path = urlEncodePath(path)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if strings.HasSuffix(u.Host, ".storage.googleapis.com") {
|
||||||
|
path = "/" + strings.TrimSuffix(u.Host, ".storage.googleapis.com")
|
||||||
|
path += u.Path
|
||||||
|
path = urlEncodePath(path)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
path = urlEncodePath(u.Path)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// preSignV2 - presign the request in following style.
|
||||||
|
// https://${S3_BUCKET}.s3.amazonaws.com/${S3_OBJECT}?AWSAccessKeyId=${S3_ACCESS_KEY}&Expires=${TIMESTAMP}&Signature=${SIGNATURE}.
|
||||||
|
func preSignV2(req http.Request, accessKeyID, secretAccessKey string, expires int64) *http.Request {
|
||||||
|
// Presign is not needed for anonymous credentials.
|
||||||
|
if accessKeyID == "" || secretAccessKey == "" {
|
||||||
|
return &req
|
||||||
|
}
|
||||||
|
d := time.Now().UTC()
|
||||||
|
// Add date if not present.
|
||||||
|
if date := req.Header.Get("Date"); date == "" {
|
||||||
|
req.Header.Set("Date", d.Format(http.TimeFormat))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get encoded URL path.
|
||||||
|
path := encodeURL2Path(req.URL)
|
||||||
|
|
||||||
|
// Find epoch expires when the request will expire.
|
||||||
|
epochExpires := d.Unix() + expires
|
||||||
|
|
||||||
|
// Get string to sign.
|
||||||
|
stringToSign := fmt.Sprintf("%s\n\n\n%d\n%s", req.Method, epochExpires, path)
|
||||||
|
hm := hmac.New(sha1.New, []byte(secretAccessKey))
|
||||||
|
hm.Write([]byte(stringToSign))
|
||||||
|
|
||||||
|
// Calculate signature.
|
||||||
|
signature := base64.StdEncoding.EncodeToString(hm.Sum(nil))
|
||||||
|
|
||||||
|
query := req.URL.Query()
|
||||||
|
// Handle specially for Google Cloud Storage.
|
||||||
|
if strings.Contains(req.URL.Host, ".storage.googleapis.com") {
|
||||||
|
query.Set("GoogleAccessId", accessKeyID)
|
||||||
|
} else {
|
||||||
|
query.Set("AWSAccessKeyId", accessKeyID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fill in Expires and Signature for presigned query.
|
||||||
|
query.Set("Expires", strconv.FormatInt(epochExpires, 10))
|
||||||
|
query.Set("Signature", signature)
|
||||||
|
|
||||||
|
// Encode query and save.
|
||||||
|
req.URL.RawQuery = query.Encode()
|
||||||
|
return &req
|
||||||
|
}
|
||||||
|
|
||||||
|
// postPresignSignatureV2 - presigned signature for PostPolicy
|
||||||
|
// request.
|
||||||
|
func postPresignSignatureV2(policyBase64, secretAccessKey string) string {
|
||||||
|
hm := hmac.New(sha1.New, []byte(secretAccessKey))
|
||||||
|
hm.Write([]byte(policyBase64))
|
||||||
|
signature := base64.StdEncoding.EncodeToString(hm.Sum(nil))
|
||||||
|
return signature
|
||||||
|
}
|
||||||
|
|
||||||
|
// Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature;
|
||||||
|
// Signature = Base64( HMAC-SHA1( YourSecretAccessKeyID, UTF-8-Encoding-Of( StringToSign ) ) );
|
||||||
|
//
|
||||||
|
// StringToSign = HTTP-Verb + "\n" +
|
||||||
|
// Content-MD5 + "\n" +
|
||||||
|
// Content-Type + "\n" +
|
||||||
|
// Date + "\n" +
|
||||||
|
// CanonicalizedProtocolHeaders +
|
||||||
|
// CanonicalizedResource;
|
||||||
|
//
|
||||||
|
// CanonicalizedResource = [ "/" + Bucket ] +
|
||||||
|
// <HTTP-Request-URI, from the protocol name up to the query string> +
|
||||||
|
// [ subresource, if present. For example "?acl", "?location", "?logging", or "?torrent"];
|
||||||
|
//
|
||||||
|
// CanonicalizedProtocolHeaders = <described below>
|
||||||
|
|
||||||
|
// signV2 sign the request before Do() (AWS Signature Version 2).
|
||||||
|
func signV2(req http.Request, accessKeyID, secretAccessKey string) *http.Request {
|
||||||
|
// Signature calculation is not needed for anonymous credentials.
|
||||||
|
if accessKeyID == "" || secretAccessKey == "" {
|
||||||
|
return &req
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initial time.
|
||||||
|
d := time.Now().UTC()
|
||||||
|
|
||||||
|
// Add date if not present.
|
||||||
|
if date := req.Header.Get("Date"); date == "" {
|
||||||
|
req.Header.Set("Date", d.Format(http.TimeFormat))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate HMAC for secretAccessKey.
|
||||||
|
stringToSign := getStringToSignV2(req)
|
||||||
|
hm := hmac.New(sha1.New, []byte(secretAccessKey))
|
||||||
|
hm.Write([]byte(stringToSign))
|
||||||
|
|
||||||
|
// Prepare auth header.
|
||||||
|
authHeader := new(bytes.Buffer)
|
||||||
|
authHeader.WriteString(fmt.Sprintf("%s %s:", signV2Algorithm, accessKeyID))
|
||||||
|
encoder := base64.NewEncoder(base64.StdEncoding, authHeader)
|
||||||
|
encoder.Write(hm.Sum(nil))
|
||||||
|
encoder.Close()
|
||||||
|
|
||||||
|
// Set Authorization header.
|
||||||
|
req.Header.Set("Authorization", authHeader.String())
|
||||||
|
|
||||||
|
return &req
|
||||||
|
}
|
||||||
|
|
||||||
|
// From the Amazon docs:
|
||||||
|
//
|
||||||
|
// StringToSign = HTTP-Verb + "\n" +
|
||||||
|
// Content-MD5 + "\n" +
|
||||||
|
// Content-Type + "\n" +
|
||||||
|
// Date + "\n" +
|
||||||
|
// CanonicalizedProtocolHeaders +
|
||||||
|
// CanonicalizedResource;
|
||||||
|
func getStringToSignV2(req http.Request) string {
|
||||||
|
buf := new(bytes.Buffer)
|
||||||
|
// Write standard headers.
|
||||||
|
writeDefaultHeaders(buf, req)
|
||||||
|
// Write canonicalized protocol headers if any.
|
||||||
|
writeCanonicalizedHeaders(buf, req)
|
||||||
|
// Write canonicalized Query resources if any.
|
||||||
|
writeCanonicalizedResource(buf, req)
|
||||||
|
return buf.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeDefaultHeader - write all default necessary headers
|
||||||
|
func writeDefaultHeaders(buf *bytes.Buffer, req http.Request) {
|
||||||
|
buf.WriteString(req.Method)
|
||||||
|
buf.WriteByte('\n')
|
||||||
|
buf.WriteString(req.Header.Get("Content-MD5"))
|
||||||
|
buf.WriteByte('\n')
|
||||||
|
buf.WriteString(req.Header.Get("Content-Type"))
|
||||||
|
buf.WriteByte('\n')
|
||||||
|
buf.WriteString(req.Header.Get("Date"))
|
||||||
|
buf.WriteByte('\n')
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeCanonicalizedHeaders - write canonicalized headers.
|
||||||
|
func writeCanonicalizedHeaders(buf *bytes.Buffer, req http.Request) {
|
||||||
|
var protoHeaders []string
|
||||||
|
vals := make(map[string][]string)
|
||||||
|
for k, vv := range req.Header {
|
||||||
|
// All the AMZ headers should be lowercase
|
||||||
|
lk := strings.ToLower(k)
|
||||||
|
if strings.HasPrefix(lk, "x-amz") {
|
||||||
|
protoHeaders = append(protoHeaders, lk)
|
||||||
|
vals[lk] = vv
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(protoHeaders)
|
||||||
|
for _, k := range protoHeaders {
|
||||||
|
buf.WriteString(k)
|
||||||
|
buf.WriteByte(':')
|
||||||
|
for idx, v := range vals[k] {
|
||||||
|
if idx > 0 {
|
||||||
|
buf.WriteByte(',')
|
||||||
|
}
|
||||||
|
if strings.Contains(v, "\n") {
|
||||||
|
// TODO: "Unfold" long headers that
|
||||||
|
// span multiple lines (as allowed by
|
||||||
|
// RFC 2616, section 4.2) by replacing
|
||||||
|
// the folding white-space (including
|
||||||
|
// new-line) by a single space.
|
||||||
|
buf.WriteString(v)
|
||||||
|
} else {
|
||||||
|
buf.WriteString(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
buf.WriteByte('\n')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Must be sorted:
|
||||||
|
var resourceList = []string{
|
||||||
|
"acl",
|
||||||
|
"location",
|
||||||
|
"logging",
|
||||||
|
"notification",
|
||||||
|
"partNumber",
|
||||||
|
"policy",
|
||||||
|
"response-content-type",
|
||||||
|
"response-content-language",
|
||||||
|
"response-expires",
|
||||||
|
"response-cache-control",
|
||||||
|
"response-content-disposition",
|
||||||
|
"response-content-encoding",
|
||||||
|
"requestPayment",
|
||||||
|
"torrent",
|
||||||
|
"uploadId",
|
||||||
|
"uploads",
|
||||||
|
"versionId",
|
||||||
|
"versioning",
|
||||||
|
"versions",
|
||||||
|
"website",
|
||||||
|
}
|
||||||
|
|
||||||
|
// From the Amazon docs:
|
||||||
|
//
|
||||||
|
// CanonicalizedResource = [ "/" + Bucket ] +
|
||||||
|
// <HTTP-Request-URI, from the protocol name up to the query string> +
|
||||||
|
// [ sub-resource, if present. For example "?acl", "?location", "?logging", or "?torrent"];
|
||||||
|
func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request) error {
|
||||||
|
// Save request URL.
|
||||||
|
requestURL := req.URL
|
||||||
|
|
||||||
|
// Get encoded URL path.
|
||||||
|
path := encodeURL2Path(requestURL)
|
||||||
|
buf.WriteString(path)
|
||||||
|
|
||||||
|
sort.Strings(resourceList)
|
||||||
|
if requestURL.RawQuery != "" {
|
||||||
|
var n int
|
||||||
|
vals, _ := url.ParseQuery(requestURL.RawQuery)
|
||||||
|
// Verify if any sub resource queries are present, if yes
|
||||||
|
// canonicallize them.
|
||||||
|
for _, resource := range resourceList {
|
||||||
|
if vv, ok := vals[resource]; ok && len(vv) > 0 {
|
||||||
|
n++
|
||||||
|
// First element
|
||||||
|
switch n {
|
||||||
|
case 1:
|
||||||
|
buf.WriteByte('?')
|
||||||
|
// The rest
|
||||||
|
default:
|
||||||
|
buf.WriteByte('&')
|
||||||
|
}
|
||||||
|
buf.WriteString(resource)
|
||||||
|
// Request parameters
|
||||||
|
if len(vv[0]) > 0 {
|
||||||
|
buf.WriteByte('=')
|
||||||
|
buf.WriteString(url.QueryEscape(vv[0]))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
303
vendor/github.com/minio/minio-go/request-signature-v4.go
generated
vendored
Normal file
303
vendor/github.com/minio/minio-go/request-signature-v4.go
generated
vendored
Normal file
@ -0,0 +1,303 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/hex"
|
||||||
|
"net/http"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Signature and API related constants.
|
||||||
|
const (
|
||||||
|
signV4Algorithm = "AWS4-HMAC-SHA256"
|
||||||
|
iso8601DateFormat = "20060102T150405Z"
|
||||||
|
yyyymmdd = "20060102"
|
||||||
|
)
|
||||||
|
|
||||||
|
///
|
||||||
|
/// Excerpts from @lsegal -
|
||||||
|
/// https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258.
|
||||||
|
///
|
||||||
|
/// User-Agent:
|
||||||
|
///
|
||||||
|
/// This is ignored from signing because signing this causes
|
||||||
|
/// problems with generating pre-signed URLs (that are executed
|
||||||
|
/// by other agents) or when customers pass requests through
|
||||||
|
/// proxies, which may modify the user-agent.
|
||||||
|
///
|
||||||
|
/// Content-Length:
|
||||||
|
///
|
||||||
|
/// This is ignored from signing because generating a pre-signed
|
||||||
|
/// URL should not provide a content-length constraint,
|
||||||
|
/// specifically when vending a S3 pre-signed PUT URL. The
|
||||||
|
/// corollary to this is that when sending regular requests
|
||||||
|
/// (non-pre-signed), the signature contains a checksum of the
|
||||||
|
/// body, which implicitly validates the payload length (since
|
||||||
|
/// changing the number of bytes would change the checksum)
|
||||||
|
/// and therefore this header is not valuable in the signature.
|
||||||
|
///
|
||||||
|
/// Content-Type:
|
||||||
|
///
|
||||||
|
/// Signing this header causes quite a number of problems in
|
||||||
|
/// browser environments, where browsers like to modify and
|
||||||
|
/// normalize the content-type header in different ways. There is
|
||||||
|
/// more information on this in https://goo.gl/2E9gyy. Avoiding
|
||||||
|
/// this field simplifies logic and reduces the possibility of
|
||||||
|
/// future bugs.
|
||||||
|
///
|
||||||
|
/// Authorization:
|
||||||
|
///
|
||||||
|
/// Is skipped for obvious reasons
|
||||||
|
///
|
||||||
|
var ignoredHeaders = map[string]bool{
|
||||||
|
"Authorization": true,
|
||||||
|
"Content-Type": true,
|
||||||
|
"Content-Length": true,
|
||||||
|
"User-Agent": true,
|
||||||
|
}
|
||||||
|
|
||||||
|
// getSigningKey hmac seed to calculate final signature.
|
||||||
|
func getSigningKey(secret, loc string, t time.Time) []byte {
|
||||||
|
date := sumHMAC([]byte("AWS4"+secret), []byte(t.Format(yyyymmdd)))
|
||||||
|
location := sumHMAC(date, []byte(loc))
|
||||||
|
service := sumHMAC(location, []byte("s3"))
|
||||||
|
signingKey := sumHMAC(service, []byte("aws4_request"))
|
||||||
|
return signingKey
|
||||||
|
}
|
||||||
|
|
||||||
|
// getSignature final signature in hexadecimal form.
|
||||||
|
func getSignature(signingKey []byte, stringToSign string) string {
|
||||||
|
return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
|
||||||
|
}
|
||||||
|
|
||||||
|
// getScope generate a string of a specific date, an AWS region, and a
|
||||||
|
// service.
|
||||||
|
func getScope(location string, t time.Time) string {
|
||||||
|
scope := strings.Join([]string{
|
||||||
|
t.Format(yyyymmdd),
|
||||||
|
location,
|
||||||
|
"s3",
|
||||||
|
"aws4_request",
|
||||||
|
}, "/")
|
||||||
|
return scope
|
||||||
|
}
|
||||||
|
|
||||||
|
// getCredential generate a credential string.
|
||||||
|
func getCredential(accessKeyID, location string, t time.Time) string {
|
||||||
|
scope := getScope(location, t)
|
||||||
|
return accessKeyID + "/" + scope
|
||||||
|
}
|
||||||
|
|
||||||
|
// getHashedPayload get the hexadecimal value of the SHA256 hash of
|
||||||
|
// the request payload.
|
||||||
|
func getHashedPayload(req http.Request) string {
|
||||||
|
hashedPayload := req.Header.Get("X-Amz-Content-Sha256")
|
||||||
|
if hashedPayload == "" {
|
||||||
|
// Presign does not have a payload, use S3 recommended value.
|
||||||
|
hashedPayload = "UNSIGNED-PAYLOAD"
|
||||||
|
}
|
||||||
|
return hashedPayload
|
||||||
|
}
|
||||||
|
|
||||||
|
// getCanonicalHeaders generate a list of request headers for
|
||||||
|
// signature.
|
||||||
|
func getCanonicalHeaders(req http.Request) string {
|
||||||
|
var headers []string
|
||||||
|
vals := make(map[string][]string)
|
||||||
|
for k, vv := range req.Header {
|
||||||
|
if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok {
|
||||||
|
continue // ignored header
|
||||||
|
}
|
||||||
|
headers = append(headers, strings.ToLower(k))
|
||||||
|
vals[strings.ToLower(k)] = vv
|
||||||
|
}
|
||||||
|
headers = append(headers, "host")
|
||||||
|
sort.Strings(headers)
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
// Save all the headers in canonical form <header>:<value> newline
|
||||||
|
// separated for each header.
|
||||||
|
for _, k := range headers {
|
||||||
|
buf.WriteString(k)
|
||||||
|
buf.WriteByte(':')
|
||||||
|
switch {
|
||||||
|
case k == "host":
|
||||||
|
buf.WriteString(req.URL.Host)
|
||||||
|
fallthrough
|
||||||
|
default:
|
||||||
|
for idx, v := range vals[k] {
|
||||||
|
if idx > 0 {
|
||||||
|
buf.WriteByte(',')
|
||||||
|
}
|
||||||
|
buf.WriteString(v)
|
||||||
|
}
|
||||||
|
buf.WriteByte('\n')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return buf.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// getSignedHeaders generate all signed request headers.
|
||||||
|
// i.e lexically sorted, semicolon-separated list of lowercase
|
||||||
|
// request header names.
|
||||||
|
func getSignedHeaders(req http.Request) string {
|
||||||
|
var headers []string
|
||||||
|
for k := range req.Header {
|
||||||
|
if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok {
|
||||||
|
continue // Ignored header found continue.
|
||||||
|
}
|
||||||
|
headers = append(headers, strings.ToLower(k))
|
||||||
|
}
|
||||||
|
headers = append(headers, "host")
|
||||||
|
sort.Strings(headers)
|
||||||
|
return strings.Join(headers, ";")
|
||||||
|
}
|
||||||
|
|
||||||
|
// getCanonicalRequest generate a canonical request of style.
|
||||||
|
//
|
||||||
|
// canonicalRequest =
|
||||||
|
// <HTTPMethod>\n
|
||||||
|
// <CanonicalURI>\n
|
||||||
|
// <CanonicalQueryString>\n
|
||||||
|
// <CanonicalHeaders>\n
|
||||||
|
// <SignedHeaders>\n
|
||||||
|
// <HashedPayload>
|
||||||
|
func getCanonicalRequest(req http.Request) string {
|
||||||
|
req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1)
|
||||||
|
canonicalRequest := strings.Join([]string{
|
||||||
|
req.Method,
|
||||||
|
urlEncodePath(req.URL.Path),
|
||||||
|
req.URL.RawQuery,
|
||||||
|
getCanonicalHeaders(req),
|
||||||
|
getSignedHeaders(req),
|
||||||
|
getHashedPayload(req),
|
||||||
|
}, "\n")
|
||||||
|
return canonicalRequest
|
||||||
|
}
|
||||||
|
|
||||||
|
// getStringToSign a string based on selected query values.
|
||||||
|
func getStringToSignV4(t time.Time, location, canonicalRequest string) string {
|
||||||
|
stringToSign := signV4Algorithm + "\n" + t.Format(iso8601DateFormat) + "\n"
|
||||||
|
stringToSign = stringToSign + getScope(location, t) + "\n"
|
||||||
|
stringToSign = stringToSign + hex.EncodeToString(sum256([]byte(canonicalRequest)))
|
||||||
|
return stringToSign
|
||||||
|
}
|
||||||
|
|
||||||
|
// preSignV4 presign the request, in accordance with
|
||||||
|
// http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html.
|
||||||
|
func preSignV4(req http.Request, accessKeyID, secretAccessKey, location string, expires int64) *http.Request {
|
||||||
|
// Presign is not needed for anonymous credentials.
|
||||||
|
if accessKeyID == "" || secretAccessKey == "" {
|
||||||
|
return &req
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initial time.
|
||||||
|
t := time.Now().UTC()
|
||||||
|
|
||||||
|
// Get credential string.
|
||||||
|
credential := getCredential(accessKeyID, location, t)
|
||||||
|
|
||||||
|
// Get all signed headers.
|
||||||
|
signedHeaders := getSignedHeaders(req)
|
||||||
|
|
||||||
|
// Set URL query.
|
||||||
|
query := req.URL.Query()
|
||||||
|
query.Set("X-Amz-Algorithm", signV4Algorithm)
|
||||||
|
query.Set("X-Amz-Date", t.Format(iso8601DateFormat))
|
||||||
|
query.Set("X-Amz-Expires", strconv.FormatInt(expires, 10))
|
||||||
|
query.Set("X-Amz-SignedHeaders", signedHeaders)
|
||||||
|
query.Set("X-Amz-Credential", credential)
|
||||||
|
req.URL.RawQuery = query.Encode()
|
||||||
|
|
||||||
|
// Get canonical request.
|
||||||
|
canonicalRequest := getCanonicalRequest(req)
|
||||||
|
|
||||||
|
// Get string to sign from canonical request.
|
||||||
|
stringToSign := getStringToSignV4(t, location, canonicalRequest)
|
||||||
|
|
||||||
|
// Gext hmac signing key.
|
||||||
|
signingKey := getSigningKey(secretAccessKey, location, t)
|
||||||
|
|
||||||
|
// Calculate signature.
|
||||||
|
signature := getSignature(signingKey, stringToSign)
|
||||||
|
|
||||||
|
// Add signature header to RawQuery.
|
||||||
|
req.URL.RawQuery += "&X-Amz-Signature=" + signature
|
||||||
|
|
||||||
|
return &req
|
||||||
|
}
|
||||||
|
|
||||||
|
// postPresignSignatureV4 - presigned signature for PostPolicy
|
||||||
|
// requests.
|
||||||
|
func postPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string {
|
||||||
|
// Get signining key.
|
||||||
|
signingkey := getSigningKey(secretAccessKey, location, t)
|
||||||
|
// Calculate signature.
|
||||||
|
signature := getSignature(signingkey, policyBase64)
|
||||||
|
return signature
|
||||||
|
}
|
||||||
|
|
||||||
|
// signV4 sign the request before Do(), in accordance with
|
||||||
|
// http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html.
|
||||||
|
func signV4(req http.Request, accessKeyID, secretAccessKey, location string) *http.Request {
|
||||||
|
// Signature calculation is not needed for anonymous credentials.
|
||||||
|
if accessKeyID == "" || secretAccessKey == "" {
|
||||||
|
return &req
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initial time.
|
||||||
|
t := time.Now().UTC()
|
||||||
|
|
||||||
|
// Set x-amz-date.
|
||||||
|
req.Header.Set("X-Amz-Date", t.Format(iso8601DateFormat))
|
||||||
|
|
||||||
|
// Get canonical request.
|
||||||
|
canonicalRequest := getCanonicalRequest(req)
|
||||||
|
|
||||||
|
// Get string to sign from canonical request.
|
||||||
|
stringToSign := getStringToSignV4(t, location, canonicalRequest)
|
||||||
|
|
||||||
|
// Get hmac signing key.
|
||||||
|
signingKey := getSigningKey(secretAccessKey, location, t)
|
||||||
|
|
||||||
|
// Get credential string.
|
||||||
|
credential := getCredential(accessKeyID, location, t)
|
||||||
|
|
||||||
|
// Get all signed headers.
|
||||||
|
signedHeaders := getSignedHeaders(req)
|
||||||
|
|
||||||
|
// Calculate signature.
|
||||||
|
signature := getSignature(signingKey, stringToSign)
|
||||||
|
|
||||||
|
// If regular request, construct the final authorization header.
|
||||||
|
parts := []string{
|
||||||
|
signV4Algorithm + " Credential=" + credential,
|
||||||
|
"SignedHeaders=" + signedHeaders,
|
||||||
|
"Signature=" + signature,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set authorization header.
|
||||||
|
auth := strings.Join(parts, ", ")
|
||||||
|
req.Header.Set("Authorization", auth)
|
||||||
|
|
||||||
|
return &req
|
||||||
|
}
|
40
vendor/github.com/minio/minio-go/s3-endpoints.go
generated
vendored
Normal file
40
vendor/github.com/minio/minio-go/s3-endpoints.go
generated
vendored
Normal file
@ -0,0 +1,40 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
// awsS3EndpointMap Amazon S3 endpoint map.
|
||||||
|
var awsS3EndpointMap = map[string]string{
|
||||||
|
"us-east-1": "s3.amazonaws.com",
|
||||||
|
"us-west-2": "s3-us-west-2.amazonaws.com",
|
||||||
|
"us-west-1": "s3-us-west-1.amazonaws.com",
|
||||||
|
"eu-west-1": "s3-eu-west-1.amazonaws.com",
|
||||||
|
"eu-central-1": "s3-eu-central-1.amazonaws.com",
|
||||||
|
"ap-southeast-1": "s3-ap-southeast-1.amazonaws.com",
|
||||||
|
"ap-northeast-1": "s3-ap-northeast-1.amazonaws.com",
|
||||||
|
"ap-northeast-2": "s3-ap-northeast-2.amazonaws.com",
|
||||||
|
"sa-east-1": "s3-sa-east-1.amazonaws.com",
|
||||||
|
}
|
||||||
|
|
||||||
|
// getS3Endpoint get Amazon S3 endpoint based on the bucket location.
|
||||||
|
func getS3Endpoint(bucketLocation string) (s3Endpoint string) {
|
||||||
|
s3Endpoint, ok := awsS3EndpointMap[bucketLocation]
|
||||||
|
if !ok {
|
||||||
|
// Default to 's3.amazonaws.com' endpoint.
|
||||||
|
s3Endpoint = "s3.amazonaws.com"
|
||||||
|
}
|
||||||
|
return s3Endpoint
|
||||||
|
}
|
37
vendor/github.com/minio/minio-go/signature-type.go
generated
vendored
Normal file
37
vendor/github.com/minio/minio-go/signature-type.go
generated
vendored
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
// SignatureType is type of Authorization requested for a given HTTP request.
|
||||||
|
type SignatureType int
|
||||||
|
|
||||||
|
// Different types of supported signatures - default is Latest i.e SignatureV4.
|
||||||
|
const (
|
||||||
|
Latest SignatureType = iota
|
||||||
|
SignatureV4
|
||||||
|
SignatureV2
|
||||||
|
)
|
||||||
|
|
||||||
|
// isV2 - is signature SignatureV2?
|
||||||
|
func (s SignatureType) isV2() bool {
|
||||||
|
return s == SignatureV2
|
||||||
|
}
|
||||||
|
|
||||||
|
// isV4 - is signature SignatureV4?
|
||||||
|
func (s SignatureType) isV4() bool {
|
||||||
|
return s == SignatureV4 || s == Latest
|
||||||
|
}
|
60
vendor/github.com/minio/minio-go/tempfile.go
generated
vendored
Normal file
60
vendor/github.com/minio/minio-go/tempfile.go
generated
vendored
Normal file
@ -0,0 +1,60 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
|
"sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
// tempFile - temporary file container.
|
||||||
|
type tempFile struct {
|
||||||
|
*os.File
|
||||||
|
mutex *sync.Mutex
|
||||||
|
}
|
||||||
|
|
||||||
|
// newTempFile returns a new temporary file, once closed it automatically deletes itself.
|
||||||
|
func newTempFile(prefix string) (*tempFile, error) {
|
||||||
|
// use platform specific temp directory.
|
||||||
|
file, err := ioutil.TempFile(os.TempDir(), prefix)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &tempFile{
|
||||||
|
File: file,
|
||||||
|
mutex: &sync.Mutex{},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close - closer wrapper to close and remove temporary file.
|
||||||
|
func (t *tempFile) Close() error {
|
||||||
|
t.mutex.Lock()
|
||||||
|
defer t.mutex.Unlock()
|
||||||
|
if t.File != nil {
|
||||||
|
// Close the file.
|
||||||
|
if err := t.File.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Remove file.
|
||||||
|
if err := os.Remove(t.File.Name()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
t.File = nil
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
305
vendor/github.com/minio/minio-go/utils.go
generated
vendored
Normal file
305
vendor/github.com/minio/minio-go/utils.go
generated
vendored
Normal file
@ -0,0 +1,305 @@
|
|||||||
|
/*
|
||||||
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package minio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/hmac"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/xml"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
"unicode/utf8"
|
||||||
|
)
|
||||||
|
|
||||||
|
// xmlDecoder provide decoded value in xml.
|
||||||
|
func xmlDecoder(body io.Reader, v interface{}) error {
|
||||||
|
d := xml.NewDecoder(body)
|
||||||
|
return d.Decode(v)
|
||||||
|
}
|
||||||
|
|
||||||
|
// sum256 calculate sha256 sum for an input byte array.
|
||||||
|
func sum256(data []byte) []byte {
|
||||||
|
hash := sha256.New()
|
||||||
|
hash.Write(data)
|
||||||
|
return hash.Sum(nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// sumHMAC calculate hmac between two input byte array.
|
||||||
|
func sumHMAC(key []byte, data []byte) []byte {
|
||||||
|
hash := hmac.New(sha256.New, key)
|
||||||
|
hash.Write(data)
|
||||||
|
return hash.Sum(nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// getEndpointURL - construct a new endpoint.
|
||||||
|
func getEndpointURL(endpoint string, inSecure bool) (*url.URL, error) {
|
||||||
|
if strings.Contains(endpoint, ":") {
|
||||||
|
host, _, err := net.SplitHostPort(endpoint)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if !isValidIP(host) && !isValidDomain(host) {
|
||||||
|
msg := "Endpoint: " + endpoint + " does not follow ip address or domain name standards."
|
||||||
|
return nil, ErrInvalidArgument(msg)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if !isValidIP(endpoint) && !isValidDomain(endpoint) {
|
||||||
|
msg := "Endpoint: " + endpoint + " does not follow ip address or domain name standards."
|
||||||
|
return nil, ErrInvalidArgument(msg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// if inSecure is true, use 'http' scheme.
|
||||||
|
scheme := "https"
|
||||||
|
if inSecure {
|
||||||
|
scheme = "http"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Construct a secured endpoint URL.
|
||||||
|
endpointURLStr := scheme + "://" + endpoint
|
||||||
|
endpointURL, err := url.Parse(endpointURLStr)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate incoming endpoint URL.
|
||||||
|
if err := isValidEndpointURL(endpointURL); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return endpointURL, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// isValidDomain validates if input string is a valid domain name.
|
||||||
|
func isValidDomain(host string) bool {
|
||||||
|
// See RFC 1035, RFC 3696.
|
||||||
|
host = strings.TrimSpace(host)
|
||||||
|
if len(host) == 0 || len(host) > 255 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// host cannot start or end with "-"
|
||||||
|
if host[len(host)-1:] == "-" || host[:1] == "-" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// host cannot start or end with "_"
|
||||||
|
if host[len(host)-1:] == "_" || host[:1] == "_" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// host cannot start or end with a "."
|
||||||
|
if host[len(host)-1:] == "." || host[:1] == "." {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// All non alphanumeric characters are invalid.
|
||||||
|
if strings.ContainsAny(host, "`~!@#$%^&*()+={}[]|\\\"';:><?/") {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// No need to regexp match, since the list is non-exhaustive.
|
||||||
|
// We let it valid and fail later.
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// isValidIP parses input string for ip address validity.
|
||||||
|
func isValidIP(ip string) bool {
|
||||||
|
return net.ParseIP(ip) != nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// closeResponse close non nil response with any response Body.
|
||||||
|
// convenient wrapper to drain any remaining data on response body.
|
||||||
|
//
|
||||||
|
// Subsequently this allows golang http RoundTripper
|
||||||
|
// to re-use the same connection for future requests.
|
||||||
|
func closeResponse(resp *http.Response) {
|
||||||
|
// Callers should close resp.Body when done reading from it.
|
||||||
|
// If resp.Body is not closed, the Client's underlying RoundTripper
|
||||||
|
// (typically Transport) may not be able to re-use a persistent TCP
|
||||||
|
// connection to the server for a subsequent "keep-alive" request.
|
||||||
|
if resp != nil && resp.Body != nil {
|
||||||
|
// Drain any remaining Body and then close the connection.
|
||||||
|
// Without this closing connection would disallow re-using
|
||||||
|
// the same connection for future uses.
|
||||||
|
// - http://stackoverflow.com/a/17961593/4465767
|
||||||
|
io.Copy(ioutil.Discard, resp.Body)
|
||||||
|
resp.Body.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// isVirtualHostSupported - verifies if bucketName can be part of
|
||||||
|
// virtual host. Currently only Amazon S3 and Google Cloud Storage would
|
||||||
|
// support this.
|
||||||
|
func isVirtualHostSupported(endpointURL *url.URL, bucketName string) bool {
|
||||||
|
// bucketName can be valid but '.' in the hostname will fail SSL
|
||||||
|
// certificate validation. So do not use host-style for such buckets.
|
||||||
|
if endpointURL.Scheme == "https" && strings.Contains(bucketName, ".") {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// Return true for all other cases
|
||||||
|
return isAmazonEndpoint(endpointURL) || isGoogleEndpoint(endpointURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Match if it is exactly Amazon S3 endpoint.
|
||||||
|
func isAmazonEndpoint(endpointURL *url.URL) bool {
|
||||||
|
if endpointURL == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if endpointURL.Host == "s3.amazonaws.com" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Match if it is exactly Google cloud storage endpoint.
|
||||||
|
func isGoogleEndpoint(endpointURL *url.URL) bool {
|
||||||
|
if endpointURL == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if endpointURL.Host == "storage.googleapis.com" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify if input endpoint URL is valid.
|
||||||
|
func isValidEndpointURL(endpointURL *url.URL) error {
|
||||||
|
if endpointURL == nil {
|
||||||
|
return ErrInvalidArgument("Endpoint url cannot be empty.")
|
||||||
|
}
|
||||||
|
if endpointURL.Path != "/" && endpointURL.Path != "" {
|
||||||
|
return ErrInvalidArgument("Endpoing url cannot have fully qualified paths.")
|
||||||
|
}
|
||||||
|
if strings.Contains(endpointURL.Host, ".amazonaws.com") {
|
||||||
|
if !isAmazonEndpoint(endpointURL) {
|
||||||
|
return ErrInvalidArgument("Amazon S3 endpoint should be 's3.amazonaws.com'.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(endpointURL.Host, ".googleapis.com") {
|
||||||
|
if !isGoogleEndpoint(endpointURL) {
|
||||||
|
return ErrInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify if input expires value is valid.
|
||||||
|
func isValidExpiry(expires time.Duration) error {
|
||||||
|
expireSeconds := int64(expires / time.Second)
|
||||||
|
if expireSeconds < 1 {
|
||||||
|
return ErrInvalidArgument("Expires cannot be lesser than 1 second.")
|
||||||
|
}
|
||||||
|
if expireSeconds > 604800 {
|
||||||
|
return ErrInvalidArgument("Expires cannot be greater than 7 days.")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// We support '.' with bucket names but we fallback to using path
|
||||||
|
// style requests instead for such buckets.
|
||||||
|
var validBucketName = regexp.MustCompile(`^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$`)
|
||||||
|
|
||||||
|
// isValidBucketName - verify bucket name in accordance with
|
||||||
|
// - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html
|
||||||
|
func isValidBucketName(bucketName string) error {
|
||||||
|
if strings.TrimSpace(bucketName) == "" {
|
||||||
|
return ErrInvalidBucketName("Bucket name cannot be empty.")
|
||||||
|
}
|
||||||
|
if len(bucketName) < 3 {
|
||||||
|
return ErrInvalidBucketName("Bucket name cannot be smaller than 3 characters.")
|
||||||
|
}
|
||||||
|
if len(bucketName) > 63 {
|
||||||
|
return ErrInvalidBucketName("Bucket name cannot be greater than 63 characters.")
|
||||||
|
}
|
||||||
|
if bucketName[0] == '.' || bucketName[len(bucketName)-1] == '.' {
|
||||||
|
return ErrInvalidBucketName("Bucket name cannot start or end with a '.' dot.")
|
||||||
|
}
|
||||||
|
if match, _ := regexp.MatchString("\\.\\.", bucketName); match == true {
|
||||||
|
return ErrInvalidBucketName("Bucket name cannot have successive periods.")
|
||||||
|
}
|
||||||
|
if !validBucketName.MatchString(bucketName) {
|
||||||
|
return ErrInvalidBucketName("Bucket name contains invalid characters.")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// isValidObjectName - verify object name in accordance with
|
||||||
|
// - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
|
||||||
|
func isValidObjectName(objectName string) error {
|
||||||
|
if strings.TrimSpace(objectName) == "" {
|
||||||
|
return ErrInvalidObjectName("Object name cannot be empty.")
|
||||||
|
}
|
||||||
|
if len(objectName) > 1024 {
|
||||||
|
return ErrInvalidObjectName("Object name cannot be greater than 1024 characters.")
|
||||||
|
}
|
||||||
|
if !utf8.ValidString(objectName) {
|
||||||
|
return ErrInvalidBucketName("Object name with non UTF-8 strings are not supported.")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// isValidObjectPrefix - verify if object prefix is valid.
|
||||||
|
func isValidObjectPrefix(objectPrefix string) error {
|
||||||
|
if len(objectPrefix) > 1024 {
|
||||||
|
return ErrInvalidObjectPrefix("Object prefix cannot be greater than 1024 characters.")
|
||||||
|
}
|
||||||
|
if !utf8.ValidString(objectPrefix) {
|
||||||
|
return ErrInvalidObjectPrefix("Object prefix with non UTF-8 strings are not supported.")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// urlEncodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences
|
||||||
|
//
|
||||||
|
// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8
|
||||||
|
// non english characters cannot be parsed due to the nature in which url.Encode() is written
|
||||||
|
//
|
||||||
|
// This function on the other hand is a direct replacement for url.Encode() technique to support
|
||||||
|
// pretty much every UTF-8 character.
|
||||||
|
func urlEncodePath(pathName string) string {
|
||||||
|
// if object matches reserved string, no need to encode them
|
||||||
|
reservedNames := regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$")
|
||||||
|
if reservedNames.MatchString(pathName) {
|
||||||
|
return pathName
|
||||||
|
}
|
||||||
|
var encodedPathname string
|
||||||
|
for _, s := range pathName {
|
||||||
|
if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark)
|
||||||
|
encodedPathname = encodedPathname + string(s)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch s {
|
||||||
|
case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark)
|
||||||
|
encodedPathname = encodedPathname + string(s)
|
||||||
|
continue
|
||||||
|
default:
|
||||||
|
len := utf8.RuneLen(s)
|
||||||
|
if len < 0 {
|
||||||
|
// if utf8 cannot convert return the same string as is
|
||||||
|
return pathName
|
||||||
|
}
|
||||||
|
u := make([]byte, len)
|
||||||
|
utf8.EncodeRune(u, s)
|
||||||
|
for _, r := range u {
|
||||||
|
hex := hex.EncodeToString([]byte{r})
|
||||||
|
encodedPathname = encodedPathname + "%" + strings.ToUpper(hex)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return encodedPathname
|
||||||
|
}
|
15
vendor/vendor.json
vendored
15
vendor/vendor.json
vendored
@ -32,6 +32,16 @@
|
|||||||
"revision": "ee1815431e497d3850809578c93ab6705f1a19f7",
|
"revision": "ee1815431e497d3850809578c93ab6705f1a19f7",
|
||||||
"revisionTime": "2015-08-19T22:15:06-07:00"
|
"revisionTime": "2015-08-19T22:15:06-07:00"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"path": "github.com/gorilla/rpc/v2",
|
||||||
|
"revision": "64e20900b8aa38bb0771dec71ba3bcc2b07fc8ec",
|
||||||
|
"revisionTime": "2015-11-05T07:45:51+08:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "github.com/gorilla/rpc/v2/json",
|
||||||
|
"revision": "64e20900b8aa38bb0771dec71ba3bcc2b07fc8ec",
|
||||||
|
"revisionTime": "2015-11-05T07:45:51+08:00"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"path": "github.com/mattn/go-isatty",
|
"path": "github.com/mattn/go-isatty",
|
||||||
"revision": "7fcbc72f853b92b5720db4a6b8482be612daef24",
|
"revision": "7fcbc72f853b92b5720db4a6b8482be612daef24",
|
||||||
@ -42,6 +52,11 @@
|
|||||||
"revision": "c4a07c7b68db77ccd119183fb1d01dd5972434ab",
|
"revision": "c4a07c7b68db77ccd119183fb1d01dd5972434ab",
|
||||||
"revisionTime": "2015-11-18T20:00:48-08:00"
|
"revisionTime": "2015-11-18T20:00:48-08:00"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"path": "github.com/minio/minio-go",
|
||||||
|
"revision": "412df729f2c19ce60895770403f266cf5eac56f7",
|
||||||
|
"revisionTime": "2016-01-22T16:23:42-08:00"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"path": "github.com/minio/minio-xl/pkg/atomic",
|
"path": "github.com/minio/minio-xl/pkg/atomic",
|
||||||
"revision": "69c47f638917ab1cb9e24649c84ac38e6f1891b8",
|
"revision": "69c47f638917ab1cb9e24649c84ac38e6f1891b8",
|
||||||
|
@ -1,104 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
|
|
||||||
"github.com/dgrijalva/jwt-go"
|
|
||||||
)
|
|
||||||
|
|
||||||
func srvLogin(requestUser *User) (int, []byte) {
|
|
||||||
authBackend := InitJWT()
|
|
||||||
if authBackend.Authenticate(requestUser) {
|
|
||||||
token, err := authBackend.GenerateToken(requestUser.Username)
|
|
||||||
if err != nil {
|
|
||||||
return http.StatusInternalServerError, nil
|
|
||||||
}
|
|
||||||
response, err := json.Marshal(AuthToken{token})
|
|
||||||
if err != nil {
|
|
||||||
return http.StatusInternalServerError, nil
|
|
||||||
}
|
|
||||||
return http.StatusOK, response
|
|
||||||
}
|
|
||||||
return http.StatusUnauthorized, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func srvRefreshToken(req *http.Request) (int, []byte) {
|
|
||||||
authBackend := InitJWT()
|
|
||||||
tokenRequest, err := jwt.ParseFromRequest(req, func(token *jwt.Token) (interface{}, error) {
|
|
||||||
if _, ok := token.Method.(*jwt.SigningMethodRSA); !ok {
|
|
||||||
return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"])
|
|
||||||
}
|
|
||||||
return authBackend.PublicKey, nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return http.StatusInternalServerError, nil
|
|
||||||
}
|
|
||||||
if tokenRequest.Valid {
|
|
||||||
userName, ok := tokenRequest.Claims["sub"].(string)
|
|
||||||
if !ok {
|
|
||||||
return http.StatusUnauthorized, nil
|
|
||||||
}
|
|
||||||
token, err := authBackend.GenerateToken(userName)
|
|
||||||
if err != nil {
|
|
||||||
return http.StatusInternalServerError, nil
|
|
||||||
}
|
|
||||||
response, err := json.Marshal(AuthToken{token})
|
|
||||||
if err != nil {
|
|
||||||
return http.StatusInternalServerError, nil
|
|
||||||
}
|
|
||||||
return http.StatusOK, response
|
|
||||||
}
|
|
||||||
return http.StatusUnauthorized, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func srvLogout(req *http.Request) int {
|
|
||||||
authBackend := InitJWT()
|
|
||||||
tokenRequest, err := jwt.ParseFromRequest(req, func(token *jwt.Token) (interface{}, error) {
|
|
||||||
if _, ok := token.Method.(*jwt.SigningMethodRSA); !ok {
|
|
||||||
return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"])
|
|
||||||
}
|
|
||||||
return authBackend.PublicKey, nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return http.StatusInternalServerError
|
|
||||||
}
|
|
||||||
if tokenRequest.Valid {
|
|
||||||
tokenString := req.Header.Get("Authorization")
|
|
||||||
if err = authBackend.Logout(tokenString, tokenRequest); err != nil {
|
|
||||||
return http.StatusInternalServerError
|
|
||||||
}
|
|
||||||
return http.StatusOK
|
|
||||||
}
|
|
||||||
return http.StatusUnauthorized
|
|
||||||
}
|
|
||||||
|
|
||||||
// LoginHandler - user login handler.
|
|
||||||
func (web WebAPI) LoginHandler(w http.ResponseWriter, r *http.Request) {
|
|
||||||
requestUser := new(User)
|
|
||||||
decoder := json.NewDecoder(r.Body)
|
|
||||||
if err := decoder.Decode(&requestUser); err != nil {
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
responseStatus, token := srvLogin(requestUser)
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
w.WriteHeader(responseStatus)
|
|
||||||
w.Write(token)
|
|
||||||
}
|
|
||||||
|
|
||||||
// RefreshTokenHandler - refresh token handler.
|
|
||||||
func (web WebAPI) RefreshTokenHandler(w http.ResponseWriter, r *http.Request) {
|
|
||||||
responseStatus, token := srvRefreshToken(r)
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
w.WriteHeader(responseStatus)
|
|
||||||
w.Write(token)
|
|
||||||
}
|
|
||||||
|
|
||||||
// LogoutHandler - user logout handler.
|
|
||||||
func (web WebAPI) LogoutHandler(w http.ResponseWriter, r *http.Request) {
|
|
||||||
responseStatus := srvLogout(r)
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
w.WriteHeader(responseStatus)
|
|
||||||
}
|
|
@ -1,12 +1,27 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
// AuthToken - auth token
|
// ListBucketsArgs - list bucket args.
|
||||||
|
type ListBucketsArgs struct{}
|
||||||
|
|
||||||
|
// ListObjectsArgs - list object args.
|
||||||
|
type ListObjectsArgs struct {
|
||||||
|
BucketName string `json:"bucketName"`
|
||||||
|
Prefix string `json:"prefix"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetObjectURLArgs - get object url.
|
||||||
|
type GetObjectURLArgs struct {
|
||||||
|
BucketName string `json:"bucketName"`
|
||||||
|
ObjectName string `json:"objectName"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// AuthToken - auth token reply
|
||||||
type AuthToken struct {
|
type AuthToken struct {
|
||||||
Token string `json:"token" form:"token"`
|
Token string `json:"token" form:"token"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// User users.
|
// LoginArgs - login arguments.
|
||||||
type User struct {
|
type LoginArgs struct {
|
||||||
Username string `json:"username" form:"username"`
|
Username string `json:"username" form:"username"`
|
||||||
Password string `json:"password" form:"password"`
|
Password string `json:"password" form:"password"`
|
||||||
}
|
}
|
||||||
|
122
web-handlers.go
Normal file
122
web-handlers.go
Normal file
@ -0,0 +1,122 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/dgrijalva/jwt-go"
|
||||||
|
"github.com/minio/minio-go"
|
||||||
|
)
|
||||||
|
|
||||||
|
func isAuthenticated(req *http.Request) bool {
|
||||||
|
authBackend := InitJWT()
|
||||||
|
tokenRequest, err := jwt.ParseFromRequest(req, func(token *jwt.Token) (interface{}, error) {
|
||||||
|
if _, ok := token.Method.(*jwt.SigningMethodRSA); !ok {
|
||||||
|
return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"])
|
||||||
|
}
|
||||||
|
return authBackend.PublicKey, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return tokenRequest.Valid
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListBuckets - list buckets api.
|
||||||
|
func (web *WebAPI) ListBuckets(r *http.Request, args *ListBucketsArgs, reply *[]minio.BucketInfo) error {
|
||||||
|
if !isAuthenticated(r) {
|
||||||
|
return errUnAuthorizedRequest
|
||||||
|
}
|
||||||
|
client, err := minio.New("localhost:9000", web.AccessKeyID, web.SecretAccessKey, true)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
buckets, err := client.ListBuckets()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
*reply = buckets
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListObjects - list objects api.
|
||||||
|
func (web *WebAPI) ListObjects(r *http.Request, args *ListObjectsArgs, reply *[]minio.ObjectInfo) error {
|
||||||
|
if !isAuthenticated(r) {
|
||||||
|
return errUnAuthorizedRequest
|
||||||
|
}
|
||||||
|
client, err := minio.New("localhost:9000", web.AccessKeyID, web.SecretAccessKey, true)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
doneCh := make(chan struct{})
|
||||||
|
defer close(doneCh)
|
||||||
|
|
||||||
|
var objects []minio.ObjectInfo
|
||||||
|
for object := range client.ListObjects(args.BucketName, args.Prefix, false, doneCh) {
|
||||||
|
if object.Err != nil {
|
||||||
|
return object.Err
|
||||||
|
}
|
||||||
|
objects = append(objects, object)
|
||||||
|
}
|
||||||
|
*reply = objects
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetObjectURL - get object url.
|
||||||
|
func (web *WebAPI) GetObjectURL(r *http.Request, args *GetObjectURLArgs, reply *string) error {
|
||||||
|
if !isAuthenticated(r) {
|
||||||
|
return errUnAuthorizedRequest
|
||||||
|
}
|
||||||
|
client, err := minio.New("localhost:9000", web.AccessKeyID, web.SecretAccessKey, true)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
urlStr, err := client.PresignedGetObject(args.BucketName, args.ObjectName, time.Duration(60*60)*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
*reply = urlStr
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Login - user login handler.
|
||||||
|
func (web *WebAPI) Login(r *http.Request, args *LoginArgs, reply *AuthToken) error {
|
||||||
|
authBackend := InitJWT()
|
||||||
|
if authBackend.Authenticate(args, web.AccessKeyID, web.SecretAccessKey) {
|
||||||
|
token, err := authBackend.GenerateToken(args.Username)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
reply.Token = token
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return errUnAuthorizedRequest
|
||||||
|
}
|
||||||
|
|
||||||
|
// RefreshToken - refresh token handler.
|
||||||
|
func (web *WebAPI) RefreshToken(r *http.Request, args *LoginArgs, reply *AuthToken) error {
|
||||||
|
if isAuthenticated(r) {
|
||||||
|
authBackend := InitJWT()
|
||||||
|
token, err := authBackend.GenerateToken(args.Username)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
reply.Token = token
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return errUnAuthorizedRequest
|
||||||
|
}
|
||||||
|
|
||||||
|
// Logout - user logout.
|
||||||
|
func (web *WebAPI) Logout(r *http.Request, arg *string, reply *string) error {
|
||||||
|
if isAuthenticated(r) {
|
||||||
|
authBackend := InitJWT()
|
||||||
|
tokenString := r.Header.Get("Authorization")
|
||||||
|
if err := authBackend.Logout(tokenString); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return errUnAuthorizedRequest
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user