web-handlers: Implement API to download files as a zip file. (#3715)

This commit is contained in:
Krishna Srinivas 2017-02-08 23:39:08 -08:00 committed by Harshavardhana
parent e5773e11c6
commit 6800902b43
3 changed files with 175 additions and 0 deletions

View File

@ -17,6 +17,7 @@
package cmd package cmd
import ( import (
"archive/zip"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
@ -530,6 +531,91 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
} }
} }
// DownloadZipArgs - Argument for downloading a bunch of files as a zip file.
// JSON will look like:
// '{"bucketname":"testbucket","prefix":"john/pics/","objects":["hawaii/","maldives/","sanjose.jpg"]}'
type DownloadZipArgs struct {
Objects []string `json:"objects"` // can be files or sub-directories
Prefix string `json:"prefix"` // current directory in the browser-ui
BucketName string `json:"bucketname"` // bucket name.
}
// Takes a list of objects and creates a zip file that sent as the response body.
func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
objectAPI := web.ObjectAPI()
if objectAPI == nil {
writeWebErrorResponse(w, errServerNotInitialized)
return
}
token := r.URL.Query().Get("token")
if !isAuthTokenValid(token) {
writeWebErrorResponse(w, errAuthentication)
return
}
var args DownloadZipArgs
decodeErr := json.NewDecoder(r.Body).Decode(&args)
if decodeErr != nil {
writeWebErrorResponse(w, decodeErr)
return
}
archive := zip.NewWriter(w)
defer archive.Close()
for _, object := range args.Objects {
// Writes compressed object file to the response.
zipit := func(objectName string) error {
info, err := objectAPI.GetObjectInfo(args.BucketName, objectName)
if err != nil {
return err
}
header := &zip.FileHeader{
Name: strings.TrimPrefix(objectName, args.Prefix),
Method: zip.Deflate,
UncompressedSize64: uint64(info.Size),
UncompressedSize: uint32(info.Size),
}
writer, err := archive.CreateHeader(header)
if err != nil {
writeWebErrorResponse(w, errUnexpected)
return err
}
return objectAPI.GetObject(args.BucketName, objectName, 0, info.Size, writer)
}
if !strings.HasSuffix(object, "/") {
// If not a directory, compress the file and write it to response.
err := zipit(pathJoin(args.Prefix, object))
if err != nil {
return
}
continue
}
// For directories, list the contents recursively and write the objects as compressed
// date to the response writer.
marker := ""
for {
lo, err := objectAPI.ListObjects(args.BucketName, pathJoin(args.Prefix, object), marker, "", 1000)
if err != nil {
return
}
marker = lo.NextMarker
for _, obj := range lo.Objects {
err = zipit(obj.Name)
if err != nil {
return
}
}
if !lo.IsTruncated {
break
}
}
}
}
// GetBucketPolicyArgs - get bucket policy args. // GetBucketPolicyArgs - get bucket policy args.
type GetBucketPolicyArgs struct { type GetBucketPolicyArgs struct {
BucketName string `json:"bucketName"` BucketName string `json:"bucketName"`

View File

@ -17,9 +17,14 @@
package cmd package cmd
import ( import (
"archive/zip"
"bytes" "bytes"
"crypto/md5"
"encoding/hex"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"io"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
@ -796,6 +801,89 @@ func testDownloadWebHandler(obj ObjectLayer, instanceType string, t TestErrHandl
} }
} }
// Test web.DownloadZip
func TestWebHandlerDownloadZip(t *testing.T) {
ExecObjectLayerTest(t, testWebHandlerDownloadZip)
}
func testWebHandlerDownloadZip(obj ObjectLayer, instanceType string, t TestErrHandler) {
apiRouter := initTestWebRPCEndPoint(obj)
credentials := serverConfig.GetCredential()
authorization, err := getWebRPCToken(apiRouter, credentials.AccessKey, credentials.SecretKey)
if err != nil {
t.Fatal("Cannot authenticate")
}
bucket := getRandomBucketName()
fileOne := "aaaaaaaaaaaaaa"
fileTwo := "bbbbbbbbbbbbbb"
fileThree := "cccccccccccccc"
// Create bucket.
err = obj.MakeBucket(bucket)
if err != nil {
// failed to create newbucket, abort.
t.Fatalf("%s : %s", instanceType, err)
}
obj.PutObject(bucket, "a/one", int64(len(fileOne)), strings.NewReader(fileOne), nil, "")
obj.PutObject(bucket, "a/b/two", int64(len(fileTwo)), strings.NewReader(fileTwo), nil, "")
obj.PutObject(bucket, "a/c/three", int64(len(fileThree)), strings.NewReader(fileThree), nil, "")
test := func(token string) (int, []byte) {
rec := httptest.NewRecorder()
path := "/minio/zip" + "?token="
if token != "" {
path = path + token
}
args := DownloadZipArgs{
Objects: []string{"one", "b/", "c/"},
Prefix: "a/",
BucketName: bucket,
}
var argsData []byte
argsData, err = json.Marshal(args)
if err != nil {
return 0, nil
}
var req *http.Request
req, err = http.NewRequest("GET", path, bytes.NewBuffer(argsData))
if err != nil {
t.Fatalf("Cannot create upload request, %v", err)
}
apiRouter.ServeHTTP(rec, req)
return rec.Code, rec.Body.Bytes()
}
code, data := test("")
if code != 403 {
t.Fatal("Expected to receive authentication error")
}
code, data = test(authorization)
if code != 200 {
t.Fatal("web.DownloadsZip() failed")
}
reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
if err != nil {
t.Fatal(err)
}
h := md5.New()
for _, file := range reader.File {
fileReader, err := file.Open()
if err != nil {
t.Fatal(err)
}
io.Copy(h, fileReader)
}
// Verify the md5 of the response.
if hex.EncodeToString(h.Sum(nil)) != "ac7196449b14bea42775d29e8bb29f50" {
t.Fatal("Incorrect zip contents")
}
}
// Wrapper for calling PresignedGet handler // Wrapper for calling PresignedGet handler
func TestWebHandlerPresignedGetHandler(t *testing.T) { func TestWebHandlerPresignedGetHandler(t *testing.T) {
ExecObjectLayerTest(t, testWebPresignedGetHandler) ExecObjectLayerTest(t, testWebPresignedGetHandler)

View File

@ -84,6 +84,7 @@ func registerWebRouter(mux *router.Router) error {
webBrowserRouter.Methods("POST").Path("/webrpc").Handler(webRPC) webBrowserRouter.Methods("POST").Path("/webrpc").Handler(webRPC)
webBrowserRouter.Methods("PUT").Path("/upload/{bucket}/{object:.+}").HandlerFunc(web.Upload) webBrowserRouter.Methods("PUT").Path("/upload/{bucket}/{object:.+}").HandlerFunc(web.Upload)
webBrowserRouter.Methods("GET").Path("/download/{bucket}/{object:.+}").Queries("token", "{token:.*}").HandlerFunc(web.Download) webBrowserRouter.Methods("GET").Path("/download/{bucket}/{object:.+}").Queries("token", "{token:.*}").HandlerFunc(web.Download)
webBrowserRouter.Methods("GET").Path("/zip").Queries("token", "{token:.*}").HandlerFunc(web.DownloadZip)
// Add compression for assets. // Add compression for assets.
compressedAssets := handlers.CompressHandler(http.StripPrefix(reservedBucket, http.FileServer(assetFS()))) compressedAssets := handlers.CompressHandler(http.StripPrefix(reservedBucket, http.FileServer(assetFS())))