Moving gateway and storage driver to packages

pkgs/gateway/gateway.go

@@ -0,0 +1,221 @@
+package gateway
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"path"
+
+	"github.com/gorilla/mux"
+	"github.com/minio-io/minio/pkgs/storage"
+	"github.com/tchap/go-patricia/patricia"
+)
+
+// Stores system configuration, populated from CLI or test runner
+type GatewayConfig struct {
+	StorageDriver     StorageDriver
+	BucketDriver      BucketDriver
+	requestBucketChan chan BucketRequest
+	dataDir           string
+}
+
+// Message for requesting a bucket
+type BucketRequest struct {
+	name     string
+	context  Context
+	callback chan Bucket
+}
+
+// Context interface for security and session information
+type Context interface{}
+
+// Bucket definition
+type Bucket interface {
+	GetName(Context) string
+	Get(Context, string) ([]byte, error)
+	Put(Context, string, []byte) error
+}
+
+// Bucket driver function, should read from a channel and respond through callback channels
+type BucketDriver func(config GatewayConfig)
+
+// Storage driver function, should read from a channel and respond through callback channels
+type StorageDriver func(bucket string, input chan ObjectRequest, config GatewayConfig)
+
+// TODO remove when building real context
+type fakeContext struct{}
+
+type GatewayGetHandler struct {
+	config GatewayConfig
+}
+
+// GET requests server
+func (handler GatewayGetHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	vars := mux.Vars(req)
+	bucketName := vars["bucket"]
+	path := vars["path"]
+	context := fakeContext{}
+	callback := make(chan Bucket)
+	handler.config.requestBucketChan <- BucketRequest{
+		name:     bucketName,
+		context:  context,
+		callback: callback,
+	}
+	bucket := <-callback
+	object, err := bucket.Get(context, string(path))
+	if err != nil {
+		http.Error(w, err.Error(), 404)
+	} else if object == nil {
+		http.Error(w, errors.New("Object not found").Error(), 404)
+	} else {
+		fmt.Fprintf(w, string(object))
+	}
+}
+
+type GatewayPutHandler struct {
+	config GatewayConfig
+}
+
+func (handler GatewayPutHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	vars := mux.Vars(req)
+	bucketName := vars["bucket"]
+	path := vars["path"]
+	object, _ := ioutil.ReadAll(req.Body)
+	context := fakeContext{}
+	callback := make(chan Bucket)
+	handler.config.requestBucketChan <- BucketRequest{
+		name:     bucketName,
+		context:  context,
+		callback: callback,
+	}
+	bucket := <-callback
+	bucket.Put(context, path, object)
+}
+
+func RegisterGatewayHandlers(router *mux.Router, config GatewayConfig) {
+	config.requestBucketChan = make(chan BucketRequest)
+	go config.BucketDriver(config)
+	getHandler := GatewayGetHandler{config}
+	putHandler := GatewayPutHandler{config}
+	router.Handle("/{bucket}/{path:.*}", getHandler).Methods("GET")
+	router.Handle("/{bucket}/{path:.*}", putHandler).Methods("PUT")
+}
+
+func SynchronizedBucketDriver(config GatewayConfig) {
+	buckets := make(map[string]*SynchronizedBucket)
+	for request := range config.requestBucketChan {
+		if buckets[request.name] == nil {
+			bucketChannel := make(chan ObjectRequest)
+			go config.StorageDriver(request.name, bucketChannel, config)
+			buckets[request.name] = &SynchronizedBucket{
+				name:    request.name,
+				channel: bucketChannel,
+			}
+		}
+		request.callback <- buckets[request.name]
+	}
+	for key := range buckets {
+		buckets[key].closeChannel()
+	}
+}
+
+type SynchronizedBucket struct {
+	name    string
+	channel chan ObjectRequest
+	objects map[string][]byte
+}
+
+type ObjectRequest struct {
+	requestType string
+	path        string
+	object      []byte
+	callback    chan interface{}
+}
+
+func (bucket SynchronizedBucket) GetName(context Context) string {
+	return bucket.name
+}
+
+func (bucket SynchronizedBucket) Get(context Context, path string) ([]byte, error) {
+	callback := make(chan interface{})
+	bucket.channel <- ObjectRequest{
+		requestType: "GET",
+		path:        path,
+		callback:    callback,
+	}
+	response := <-callback
+
+	switch response.(type) {
+	case error:
+		return nil, response.(error)
+	case nil:
+		return nil, errors.New("Object not found")
+	case interface{}:
+		return response.([]byte), nil
+	default:
+		return nil, errors.New("Unexpected error, service failed")
+	}
+}
+
+func (bucket SynchronizedBucket) Put(context Context, path string, object []byte) error {
+	callback := make(chan interface{})
+	bucket.channel <- ObjectRequest{
+		requestType: "PUT",
+		path:        path,
+		object:      object,
+		callback:    callback,
+	}
+	switch response := <-callback; response.(type) {
+	case error:
+		return response.(error)
+	case nil:
+		return nil
+	default:
+		return errors.New("Unexpected error, service failed")
+	}
+}
+
+func (bucket *SynchronizedBucket) closeChannel() {
+	close(bucket.channel)
+}
+
+func InMemoryStorageDriver(bucket string, input chan ObjectRequest, config GatewayConfig) {
+	objects := patricia.NewTrie()
+	for request := range input {
+		prefix := patricia.Prefix(request.path)
+		switch request.requestType {
+		case "GET":
+			request.callback <- objects.Get(prefix)
+		case "PUT":
+			objects.Insert(prefix, request.object)
+			request.callback <- nil
+		default:
+			request.callback <- errors.New("Unexpected message")
+		}
+	}
+}
+
+func SimpleFileStorageDriver(bucket string, input chan ObjectRequest, config GatewayConfig) {
+	fileStorage := storage.FileStorage{
+		RootDir: config.dataDir,
+	}
+	for request := range input {
+		switch request.requestType {
+		case "GET":
+			objectPath := path.Join(bucket, request.path)
+			object, err := fileStorage.Get(objectPath)
+			if err != nil {
+				request.callback <- nil
+			} else {
+				request.callback <- object
+			}
+		case "PUT":
+			objectPath := path.Join(bucket, request.path)
+			fileStorage.Put(objectPath, request.object)
+			request.callback <- nil
+		default:
+			request.callback <- errors.New("Unexpected message")
+		}
+	}
+}

pkgs/gateway/gateway_test.go

@@ -0,0 +1,152 @@
+package gateway
+
+import (
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+
+	"github.com/gorilla/mux"
+	"github.com/minio-io/minio/pkgs/miniotest"
+	. "gopkg.in/check.v1"
+)
+
+type GatewaySuite struct{}
+
+var _ = Suite(&GatewaySuite{})
+
+func (s *GatewaySuite) TestPrintsGateway(c *C) {
+	// set up router with in memory storage driver
+	router := mux.NewRouter()
+	config := GatewayConfig{
+		StorageDriver: InMemoryStorageDriver,
+		BucketDriver:  SynchronizedBucketDriver,
+	}
+	RegisterGatewayHandlers(router, config)
+	server := httptest.NewServer(router)
+	defer server.Close()
+
+	// GET request, empty
+	getReq1, _ := http.NewRequest("GET", server.URL+"/one/two/three", nil)
+	client := &http.Client{}
+	resp, err := client.Do(getReq1)
+	c.Assert(resp.StatusCode, Equals, 404)
+	c.Assert(err, IsNil)
+
+	// assert object not found response
+	body, _ := ioutil.ReadAll(resp.Body)
+	c.Assert(string(body), Equals, "Object not found\n")
+	c.Assert(err, IsNil)
+
+	// add new object
+	putReq, _ := http.NewRequest("PUT", server.URL+"/one/two/three", strings.NewReader("hello"))
+	resp, err = client.Do(putReq)
+	c.Assert(resp.StatusCode, Equals, 200)
+	c.Assert(err, IsNil)
+
+	// verify object exists
+	getReq2, _ := http.NewRequest("GET", server.URL+"/one/two/three", strings.NewReader("hello"))
+	resp, err = client.Do(getReq2)
+	c.Assert(resp.StatusCode, Equals, 200)
+	c.Assert(err, IsNil)
+
+	// verify object's contents
+	body2, _ := ioutil.ReadAll(resp.Body)
+	c.Assert(string(body2), Equals, "hello")
+	c.Assert(err, IsNil)
+}
+
+type TestContext struct{}
+
+func (s *GatewaySuite) TestBucketCreation(c *C) {
+	config := GatewayConfig{
+		StorageDriver:     InMemoryStorageDriver,
+		requestBucketChan: make(chan BucketRequest),
+	}
+	defer close(config.requestBucketChan)
+	go SynchronizedBucketDriver(config)
+	context := TestContext{}
+
+	// get new bucket A
+	var bucketA1 Bucket
+	callback := make(chan Bucket)
+	config.requestBucketChan <- BucketRequest{
+		name:     "bucketA",
+		context:  context,
+		callback: callback,
+	}
+	bucketA1 = <-callback
+	c.Assert(bucketA1.GetName(context), Equals, "bucketA")
+
+	// get bucket A again
+	var bucketA2 Bucket
+	callback = make(chan Bucket)
+	config.requestBucketChan <- BucketRequest{
+		name:     "bucketA",
+		context:  context,
+		callback: callback,
+	}
+	bucketA2 = <-callback
+	c.Assert(bucketA2.GetName(context), Equals, "bucketA")
+	c.Assert(bucketA1, DeepEquals, bucketA2)
+
+	// get new bucket B
+	var bucketB Bucket
+	callback = make(chan Bucket)
+	config.requestBucketChan <- BucketRequest{
+		name:     "bucketB",
+		context:  context,
+		callback: callback,
+	}
+	bucketB = <-callback
+	c.Assert(bucketB.GetName(context), Equals, "bucketB")
+}
+
+func (s *GatewaySuite) TestInMemoryBucketOperations(c *C) {
+	simpleFileStorageRootDir, err := miniotest.MakeTempTestDir()
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(simpleFileStorageRootDir)
+	configs := []GatewayConfig{
+		GatewayConfig{
+			StorageDriver:     InMemoryStorageDriver,
+			requestBucketChan: make(chan BucketRequest),
+		},
+		GatewayConfig{
+			StorageDriver:     SimpleFileStorageDriver,
+			requestBucketChan: make(chan BucketRequest),
+			dataDir:           simpleFileStorageRootDir,
+		},
+	}
+	for _, config := range configs {
+		defer close(config.requestBucketChan)
+		go SynchronizedBucketDriver(config)
+		context := TestContext{}
+
+		// get bucket
+		callback := make(chan Bucket)
+		config.requestBucketChan <- BucketRequest{
+			name:     "bucket",
+			context:  context,
+			callback: callback,
+		}
+		bucket := <-callback
+		c.Assert(bucket.GetName(context), Equals, "bucket")
+
+		// get missing value
+		nilResult, err := bucket.Get(context, "foo")
+		c.Assert(nilResult, IsNil)
+		c.Assert(err, Not(IsNil))
+		c.Assert(err.Error(), Equals, "Object not found")
+
+		// add new value
+		err = bucket.Put(context, "foo", []byte("bar"))
+		c.Assert(err, IsNil)
+
+		// retrieve value
+		barResult, err := bucket.Get(context, "foo")
+		c.Assert(err, IsNil)
+		c.Assert(string(barResult), Equals, "bar")
+
+	}
+}

pkgs/miniotest/helpers.go

@@ -0,0 +1,7 @@
+package miniotest
+
+import "io/ioutil"
+
+func MakeTempTestDir() (string, error) {
+	return ioutil.TempDir("/tmp", "minio-test-")
+}

pkgs/storage/file_storage.go

@@ -0,0 +1,22 @@
+package storage
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+)
+
+type FileStorage struct {
+	RootDir string
+}
+
+func (storage FileStorage) Get(objectPath string) ([]byte, error) {
+	return ioutil.ReadFile(path.Join(storage.RootDir, objectPath))
+
+}
+
+func (storage FileStorage) Put(objectPath string, object []byte) error {
+	os.MkdirAll(filepath.Dir(path.Join(storage.RootDir, objectPath)), 0700)
+	return ioutil.WriteFile(path.Join(storage.RootDir, objectPath), object, 0600)
+}

pkgs/storage/file_storage_test.go

@@ -0,0 +1,51 @@
+package storage
+
+import (
+	. "gopkg.in/check.v1"
+	"io/ioutil"
+	"os"
+)
+
+type FileStorageSuite struct{}
+
+var _ = Suite(&FileStorageSuite{})
+
+func makeTempTestDir() (string, error) {
+	return ioutil.TempDir("/tmp", "minio-test-")
+}
+
+func (s *FileStorageSuite) TestFileStoragePutAtRootPath(c *C) {
+	rootDir, err := makeTempTestDir()
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(rootDir)
+
+	var storage ObjectStorage
+	storage = FileStorage{
+		RootDir: rootDir,
+	}
+
+	storage.Put("path1", []byte("object1"))
+
+	// assert object1 was created in correct path
+	object1, err := storage.Get("path1")
+	c.Assert(err, IsNil)
+	c.Assert(string(object1), Equals, "object1")
+}
+
+func (s *FileStorageSuite) TestFileStoragePutDirPath(c *C) {
+	rootDir, err := makeTempTestDir()
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(rootDir)
+
+	var storage ObjectStorage
+	storage = FileStorage{
+		RootDir: rootDir,
+	}
+
+	storage.Put("path1/path2/path3", []byte("object"))
+
+	// assert object1 was created in correct path
+	object1, err := storage.Get("path1/path2/path3")
+	c.Assert(err, IsNil)
+	c.Assert(string(object1), Equals, "object")
+}

pkgs/storage/storage.go

@@ -0,0 +1,20 @@
+package storage
+
+import (
+	"fmt"
+	"github.com/gorilla/mux"
+	"net/http"
+)
+
+type ObjectStorage interface {
+	Get(path string) ([]byte, error)
+	Put(path string, object []byte) error
+}
+
+func RegisterStorageHandlers(router *mux.Router) {
+	router.HandleFunc("/storage/rpc", StorageHandler)
+}
+
+func StorageHandler(w http.ResponseWriter, req *http.Request) {
+	fmt.Fprintf(w, "Storage")
+}

pkgs/storage/storage_test.go

@@ -0,0 +1,27 @@
+package storage
+
+import (
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestPrintsStorage(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(StorageHandler))
+	defer server.Close()
+	res, err := http.Get(server.URL)
+	if err != nil {
+		log.Fatal(err)
+	}
+	body, err := ioutil.ReadAll(res.Body)
+	res.Body.Close()
+	if err != nil {
+		log.Fatal(err)
+	}
+	bodyString := string(body)
+	if bodyString != "Storage" {
+		log.Fatal("Expected 'Storage', Received '" + bodyString + "'")
+	}
+}