web-handlers: Implement API to download files as a zip file. (#3715)

cmd/web-handlers.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"archive/zip"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -530,6 +531,91 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// DownloadZipArgs - Argument for downloading a bunch of files as a zip file.
+// JSON will look like:
+// '{"bucketname":"testbucket","prefix":"john/pics/","objects":["hawaii/","maldives/","sanjose.jpg"]}'
+type DownloadZipArgs struct {
+	Objects    []string `json:"objects"`    // can be files or sub-directories
+	Prefix     string   `json:"prefix"`     // current directory in the browser-ui
+	BucketName string   `json:"bucketname"` // bucket name.
+}
+
+// Takes a list of objects and creates a zip file that sent as the response body.
+func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
+	objectAPI := web.ObjectAPI()
+	if objectAPI == nil {
+		writeWebErrorResponse(w, errServerNotInitialized)
+		return
+	}
+
+	token := r.URL.Query().Get("token")
+
+	if !isAuthTokenValid(token) {
+		writeWebErrorResponse(w, errAuthentication)
+		return
+	}
+	var args DownloadZipArgs
+	decodeErr := json.NewDecoder(r.Body).Decode(&args)
+	if decodeErr != nil {
+		writeWebErrorResponse(w, decodeErr)
+		return
+	}
+
+	archive := zip.NewWriter(w)
+	defer archive.Close()
+
+	for _, object := range args.Objects {
+		// Writes compressed object file to the response.
+		zipit := func(objectName string) error {
+			info, err := objectAPI.GetObjectInfo(args.BucketName, objectName)
+			if err != nil {
+				return err
+			}
+			header := &zip.FileHeader{
+				Name:               strings.TrimPrefix(objectName, args.Prefix),
+				Method:             zip.Deflate,
+				UncompressedSize64: uint64(info.Size),
+				UncompressedSize:   uint32(info.Size),
+			}
+			writer, err := archive.CreateHeader(header)
+			if err != nil {
+				writeWebErrorResponse(w, errUnexpected)
+				return err
+			}
+			return objectAPI.GetObject(args.BucketName, objectName, 0, info.Size, writer)
+		}
+
+		if !strings.HasSuffix(object, "/") {
+			// If not a directory, compress the file and write it to response.
+			err := zipit(pathJoin(args.Prefix, object))
+			if err != nil {
+				return
+			}
+			continue
+		}
+
+		// For directories, list the contents recursively and write the objects as compressed
+		// date to the response writer.
+		marker := ""
+		for {
+			lo, err := objectAPI.ListObjects(args.BucketName, pathJoin(args.Prefix, object), marker, "", 1000)
+			if err != nil {
+				return
+			}
+			marker = lo.NextMarker
+			for _, obj := range lo.Objects {
+				err = zipit(obj.Name)
+				if err != nil {
+					return
+				}
+			}
+			if !lo.IsTruncated {
+				break
+			}
+		}
+	}
+}
+
 // GetBucketPolicyArgs - get bucket policy args.
 type GetBucketPolicyArgs struct {
 	BucketName string `json:"bucketName"`

cmd/web-handlers_test.go

@@ -17,9 +17,14 @@
 package cmd
 
 import (
+	"archive/zip"
 	"bytes"
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -796,6 +801,89 @@ func testDownloadWebHandler(obj ObjectLayer, instanceType string, t TestErrHandl
 	}
 }
 
+// Test web.DownloadZip
+func TestWebHandlerDownloadZip(t *testing.T) {
+	ExecObjectLayerTest(t, testWebHandlerDownloadZip)
+}
+
+func testWebHandlerDownloadZip(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	apiRouter := initTestWebRPCEndPoint(obj)
+	credentials := serverConfig.GetCredential()
+
+	authorization, err := getWebRPCToken(apiRouter, credentials.AccessKey, credentials.SecretKey)
+	if err != nil {
+		t.Fatal("Cannot authenticate")
+	}
+
+	bucket := getRandomBucketName()
+	fileOne := "aaaaaaaaaaaaaa"
+	fileTwo := "bbbbbbbbbbbbbb"
+	fileThree := "cccccccccccccc"
+
+	// Create bucket.
+	err = obj.MakeBucket(bucket)
+	if err != nil {
+		// failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err)
+	}
+
+	obj.PutObject(bucket, "a/one", int64(len(fileOne)), strings.NewReader(fileOne), nil, "")
+	obj.PutObject(bucket, "a/b/two", int64(len(fileTwo)), strings.NewReader(fileTwo), nil, "")
+	obj.PutObject(bucket, "a/c/three", int64(len(fileThree)), strings.NewReader(fileThree), nil, "")
+
+	test := func(token string) (int, []byte) {
+		rec := httptest.NewRecorder()
+		path := "/minio/zip" + "?token="
+		if token != "" {
+			path = path + token
+		}
+		args := DownloadZipArgs{
+			Objects:    []string{"one", "b/", "c/"},
+			Prefix:     "a/",
+			BucketName: bucket,
+		}
+
+		var argsData []byte
+		argsData, err = json.Marshal(args)
+		if err != nil {
+			return 0, nil
+		}
+		var req *http.Request
+		req, err = http.NewRequest("GET", path, bytes.NewBuffer(argsData))
+
+		if err != nil {
+			t.Fatalf("Cannot create upload request, %v", err)
+		}
+
+		apiRouter.ServeHTTP(rec, req)
+		return rec.Code, rec.Body.Bytes()
+	}
+	code, data := test("")
+	if code != 403 {
+		t.Fatal("Expected to receive authentication error")
+	}
+	code, data = test(authorization)
+	if code != 200 {
+		t.Fatal("web.DownloadsZip() failed")
+	}
+	reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	h := md5.New()
+	for _, file := range reader.File {
+		fileReader, err := file.Open()
+		if err != nil {
+			t.Fatal(err)
+		}
+		io.Copy(h, fileReader)
+	}
+	// Verify the md5 of the response.
+	if hex.EncodeToString(h.Sum(nil)) != "ac7196449b14bea42775d29e8bb29f50" {
+		t.Fatal("Incorrect zip contents")
+	}
+}
+
 // Wrapper for calling PresignedGet handler
 func TestWebHandlerPresignedGetHandler(t *testing.T) {
 	ExecObjectLayerTest(t, testWebPresignedGetHandler)

cmd/web-router.go

@@ -84,6 +84,7 @@ func registerWebRouter(mux *router.Router) error {
 	webBrowserRouter.Methods("POST").Path("/webrpc").Handler(webRPC)
 	webBrowserRouter.Methods("PUT").Path("/upload/{bucket}/{object:.+}").HandlerFunc(web.Upload)
 	webBrowserRouter.Methods("GET").Path("/download/{bucket}/{object:.+}").Queries("token", "{token:.*}").HandlerFunc(web.Download)
+	webBrowserRouter.Methods("GET").Path("/zip").Queries("token", "{token:.*}").HandlerFunc(web.DownloadZip)
 
 	// Add compression for assets.
 	compressedAssets := handlers.CompressHandler(http.StripPrefix(reservedBucket, http.FileServer(assetFS())))