xl: Moved to minio/minio - fixes #1112

pkg/xl/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

pkg/xl/README.md

@@ -0,0 +1,3 @@
+# XL
+
+xl - XL distributed erasure coded on-disk format released under [Apache license v2](./LICENSE).

pkg/xl/acl.go

@@ -0,0 +1,47 @@
+package xl
+
+// BucketACL - bucket level access control
+type BucketACL string
+
+// different types of ACL's currently supported for buckets
+const (
+	BucketPrivate         = BucketACL("private")
+	BucketPublicRead      = BucketACL("public-read")
+	BucketPublicReadWrite = BucketACL("public-read-write")
+)
+
+func (b BucketACL) String() string {
+	return string(b)
+}
+
+// IsPrivate - is acl Private
+func (b BucketACL) IsPrivate() bool {
+	return b == BucketACL("private")
+}
+
+// IsPublicRead - is acl PublicRead
+func (b BucketACL) IsPublicRead() bool {
+	return b == BucketACL("public-read")
+}
+
+// IsPublicReadWrite - is acl PublicReadWrite
+func (b BucketACL) IsPublicReadWrite() bool {
+	return b == BucketACL("public-read-write")
+}
+
+// IsValidBucketACL - is provided acl string supported
+func IsValidBucketACL(acl string) bool {
+	switch acl {
+	case "private":
+		fallthrough
+	case "public-read":
+		fallthrough
+	case "public-read-write":
+		return true
+	case "":
+		// by default its "private"
+		return true
+	default:
+		return false
+	}
+}

pkg/xl/block/block.go

@@ -0,0 +1,196 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package block
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"sync"
+	"syscall"
+
+	"github.com/minio/minio/pkg/atomic"
+	"github.com/minio/minio/pkg/disk"
+	"github.com/minio/minio/pkg/probe"
+)
+
+// Block container for block disk parameters
+type Block struct {
+	lock   *sync.Mutex
+	path   string
+	fsInfo disk.Info
+}
+
+// ErrInvalidArgument - invalid argument.
+var ErrInvalidArgument = errors.New("Invalid argument")
+
+// New - instantiate new disk
+func New(diskPath string) (Block, *probe.Error) {
+	if diskPath == "" {
+		return Block{}, probe.NewError(ErrInvalidArgument)
+	}
+	st, err := os.Stat(diskPath)
+	if err != nil {
+		return Block{}, probe.NewError(err)
+	}
+
+	if !st.IsDir() {
+		return Block{}, probe.NewError(syscall.ENOTDIR)
+	}
+	info, err := disk.GetInfo(diskPath)
+	if err != nil {
+		return Block{}, probe.NewError(err)
+	}
+	disk := Block{
+		lock:   &sync.Mutex{},
+		path:   diskPath,
+		fsInfo: info,
+	}
+	return disk, nil
+}
+
+// IsUsable - is disk usable, alive
+func (d Block) IsUsable() bool {
+	_, err := os.Stat(d.path)
+	if err != nil {
+		return false
+	}
+	return true
+}
+
+// GetPath - get root disk path
+func (d Block) GetPath() string {
+	return d.path
+}
+
+// GetFSInfo - get disk filesystem and its usage information
+func (d Block) GetFSInfo() disk.Info {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	info, err := disk.GetInfo(d.path)
+	if err != nil {
+		return d.fsInfo
+	}
+	d.fsInfo = info
+	return info
+}
+
+// MakeDir - make a directory inside disk root path
+func (d Block) MakeDir(dirname string) *probe.Error {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+	if err := os.MkdirAll(filepath.Join(d.path, dirname), 0700); err != nil {
+		return probe.NewError(err)
+	}
+	return nil
+}
+
+// ListDir - list a directory inside disk root path, get only directories
+func (d Block) ListDir(dirname string) ([]os.FileInfo, *probe.Error) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	dir, err := os.Open(filepath.Join(d.path, dirname))
+	if err != nil {
+		return nil, probe.NewError(err)
+	}
+	defer dir.Close()
+	contents, err := dir.Readdir(-1)
+	if err != nil {
+		return nil, probe.NewError(err)
+	}
+	var directories []os.FileInfo
+	for _, content := range contents {
+		// Include only directories, ignore everything else
+		if content.IsDir() {
+			directories = append(directories, content)
+		}
+	}
+	return directories, nil
+}
+
+// ListFiles - list a directory inside disk root path, get only files
+func (d Block) ListFiles(dirname string) ([]os.FileInfo, *probe.Error) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	dir, err := os.Open(filepath.Join(d.path, dirname))
+	if err != nil {
+		return nil, probe.NewError(err)
+	}
+	defer dir.Close()
+	contents, err := dir.Readdir(-1)
+	if err != nil {
+		return nil, probe.NewError(err)
+	}
+	var files []os.FileInfo
+	for _, content := range contents {
+		// Include only regular files, ignore everything else
+		if content.Mode().IsRegular() {
+			files = append(files, content)
+		}
+	}
+	return files, nil
+}
+
+// CreateFile - create a file inside disk root path, replies with custome d.File which provides atomic writes
+func (d Block) CreateFile(filename string) (*atomic.File, *probe.Error) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	if filename == "" {
+		return nil, probe.NewError(ErrInvalidArgument)
+	}
+
+	f, err := atomic.FileCreate(filepath.Join(d.path, filename))
+	if err != nil {
+		return nil, probe.NewError(err)
+	}
+
+	return f, nil
+}
+
+// Open - read a file inside disk root path
+func (d Block) Open(filename string) (*os.File, *probe.Error) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	if filename == "" {
+		return nil, probe.NewError(ErrInvalidArgument)
+	}
+	dataFile, err := os.Open(filepath.Join(d.path, filename))
+	if err != nil {
+		return nil, probe.NewError(err)
+	}
+	return dataFile, nil
+}
+
+// OpenFile - Use with caution
+func (d Block) OpenFile(filename string, flags int, perm os.FileMode) (*os.File, *probe.Error) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	if filename == "" {
+		return nil, probe.NewError(ErrInvalidArgument)
+	}
+	dataFile, err := os.OpenFile(filepath.Join(d.path, filename), flags, perm)
+	if err != nil {
+		return nil, probe.NewError(err)
+	}
+	return dataFile, nil
+}

pkg/xl/block/block_test.go

@@ -0,0 +1,83 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or impliedisk.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package block
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+func TestDisk(t *testing.T) { TestingT(t) }
+
+type MyDiskSuite struct {
+	path string
+	d    Block
+}
+
+var _ = Suite(&MyDiskSuite{})
+
+func (s *MyDiskSuite) SetUpSuite(c *C) {
+	path, err := ioutil.TempDir(os.TempDir(), "disk-")
+	c.Assert(err, IsNil)
+	s.path = path
+	d, perr := New(s.path)
+	c.Assert(perr, IsNil)
+	s.d = d
+}
+
+func (s *MyDiskSuite) TearDownSuite(c *C) {
+	os.RemoveAll(s.path)
+}
+
+func (s *MyDiskSuite) TestDiskInfo(c *C) {
+	c.Assert(s.path, Equals, s.d.GetPath())
+	fsInfo := s.d.GetFSInfo()
+	c.Assert(fsInfo.FSType, Not(Equals), "UNKNOWN")
+}
+
+func (s *MyDiskSuite) TestDiskCreateDir(c *C) {
+	c.Assert(s.d.MakeDir("hello"), IsNil)
+}
+
+func (s *MyDiskSuite) TestDiskCreateFile(c *C) {
+	f, err := s.d.CreateFile("hello1")
+	c.Assert(err, IsNil)
+	c.Assert(f.Name(), Not(Equals), filepath.Join(s.path, "hello1"))
+	// close renames the file
+	f.Close()
+
+	// Open should be a success
+	_, err = s.d.Open("hello1")
+	c.Assert(err, IsNil)
+}
+
+func (s *MyDiskSuite) TestDiskOpen(c *C) {
+	f1, err := s.d.CreateFile("hello2")
+	c.Assert(err, IsNil)
+	c.Assert(f1.Name(), Not(Equals), filepath.Join(s.path, "hello2"))
+	// close renames the file
+	f1.Close()
+
+	f2, err := s.d.Open("hello2")
+	c.Assert(err, IsNil)
+	c.Assert(f2.Name(), Equals, filepath.Join(s.path, "hello2"))
+	defer f2.Close()
+}

pkg/xl/bucket.go

@@ -0,0 +1,639 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package xl
+
+import (
+	"bytes"
+	"fmt"
+	"hash"
+	"io"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
+
+	"github.com/minio/minio/pkg/crypto/sha256"
+	"github.com/minio/minio/pkg/crypto/sha512"
+	"github.com/minio/minio/pkg/probe"
+	signV4 "github.com/minio/minio/pkg/signature"
+	"github.com/minio/minio/pkg/xl/block"
+)
+
+const (
+	blockSize = 10 * 1024 * 1024
+)
+
+// internal struct carrying bucket specific information
+type bucket struct {
+	name   string
+	acl    string
+	time   time.Time
+	xlName string
+	nodes  map[string]node
+	lock   *sync.Mutex
+}
+
+// newBucket - instantiate a new bucket
+func newBucket(bucketName, aclType, xlName string, nodes map[string]node) (bucket, BucketMetadata, *probe.Error) {
+	if strings.TrimSpace(bucketName) == "" || strings.TrimSpace(xlName) == "" {
+		return bucket{}, BucketMetadata{}, probe.NewError(InvalidArgument{})
+	}
+
+	b := bucket{}
+	t := time.Now().UTC()
+	b.name = bucketName
+	b.acl = aclType
+	b.time = t
+	b.xlName = xlName
+	b.nodes = nodes
+	b.lock = new(sync.Mutex)
+
+	metadata := BucketMetadata{}
+	metadata.Version = bucketMetadataVersion
+	metadata.Name = bucketName
+	metadata.ACL = BucketACL(aclType)
+	metadata.Created = t
+	metadata.Metadata = make(map[string]string)
+	metadata.BucketObjects = make(map[string]struct{})
+
+	return b, metadata, nil
+}
+
+// getBucketName -
+func (b bucket) getBucketName() string {
+	return b.name
+}
+
+// getBucketMetadataReaders -
+func (b bucket) getBucketMetadataReaders() (map[int]io.ReadCloser, *probe.Error) {
+	readers := make(map[int]io.ReadCloser)
+	var disks map[int]block.Block
+	var err *probe.Error
+	for _, node := range b.nodes {
+		disks, err = node.ListDisks()
+		if err != nil {
+			return nil, err.Trace()
+		}
+	}
+	var bucketMetaDataReader io.ReadCloser
+	for order, disk := range disks {
+		bucketMetaDataReader, err = disk.Open(filepath.Join(b.xlName, bucketMetadataConfig))
+		if err != nil {
+			continue
+		}
+		readers[order] = bucketMetaDataReader
+	}
+	if err != nil {
+		return nil, err.Trace()
+	}
+	return readers, nil
+}
+
+// getBucketMetadata -
+func (b bucket) getBucketMetadata() (*AllBuckets, *probe.Error) {
+	metadata := new(AllBuckets)
+	var readers map[int]io.ReadCloser
+	{
+		var err *probe.Error
+		readers, err = b.getBucketMetadataReaders()
+		if err != nil {
+			return nil, err.Trace()
+		}
+	}
+	for _, reader := range readers {
+		defer reader.Close()
+	}
+	var err error
+	for _, reader := range readers {
+		jenc := json.NewDecoder(reader)
+		if err = jenc.Decode(metadata); err == nil {
+			return metadata, nil
+		}
+	}
+	return nil, probe.NewError(err)
+}
+
+// GetObjectMetadata - get metadata for an object
+func (b bucket) GetObjectMetadata(objectName string) (ObjectMetadata, *probe.Error) {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+	return b.readObjectMetadata(normalizeObjectName(objectName))
+}
+
+// ListObjects - list all objects
+func (b bucket) ListObjects(prefix, marker, delimiter string, maxkeys int) (ListObjectsResults, *probe.Error) {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+	if maxkeys <= 0 {
+		maxkeys = 1000
+	}
+	var isTruncated bool
+	var objects []string
+	bucketMetadata, err := b.getBucketMetadata()
+	if err != nil {
+		return ListObjectsResults{}, err.Trace()
+	}
+	for objectName := range bucketMetadata.Buckets[b.getBucketName()].Multiparts {
+		if strings.HasPrefix(objectName, strings.TrimSpace(prefix)) {
+			if objectName > marker {
+				objects = append(objects, objectName)
+			}
+		}
+	}
+	for objectName := range bucketMetadata.Buckets[b.getBucketName()].BucketObjects {
+		if strings.HasPrefix(objectName, strings.TrimSpace(prefix)) {
+			if objectName > marker {
+				objects = append(objects, objectName)
+			}
+		}
+	}
+	if strings.TrimSpace(prefix) != "" {
+		objects = TrimPrefix(objects, prefix)
+	}
+	var prefixes []string
+	var filteredObjects []string
+	filteredObjects = objects
+	if strings.TrimSpace(delimiter) != "" {
+		filteredObjects = HasNoDelimiter(objects, delimiter)
+		prefixes = HasDelimiter(objects, delimiter)
+		prefixes = SplitDelimiter(prefixes, delimiter)
+		prefixes = SortUnique(prefixes)
+	}
+	var results []string
+	var commonPrefixes []string
+
+	for _, commonPrefix := range prefixes {
+		commonPrefixes = append(commonPrefixes, prefix+commonPrefix)
+	}
+	filteredObjects = RemoveDuplicates(filteredObjects)
+	sort.Strings(filteredObjects)
+	for _, objectName := range filteredObjects {
+		if len(results) >= maxkeys {
+			isTruncated = true
+			break
+		}
+		results = append(results, prefix+objectName)
+	}
+	results = RemoveDuplicates(results)
+	commonPrefixes = RemoveDuplicates(commonPrefixes)
+	sort.Strings(commonPrefixes)
+
+	listObjects := ListObjectsResults{}
+	listObjects.Objects = make(map[string]ObjectMetadata)
+	listObjects.CommonPrefixes = commonPrefixes
+	listObjects.IsTruncated = isTruncated
+
+	for _, objectName := range results {
+		objMetadata, err := b.readObjectMetadata(normalizeObjectName(objectName))
+		if err != nil {
+			return ListObjectsResults{}, err.Trace()
+		}
+		listObjects.Objects[objectName] = objMetadata
+	}
+	return listObjects, nil
+}
+
+// ReadObject - open an object to read
+func (b bucket) ReadObject(objectName string) (reader io.ReadCloser, size int64, err *probe.Error) {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+	reader, writer := io.Pipe()
+	// get list of objects
+	bucketMetadata, err := b.getBucketMetadata()
+	if err != nil {
+		return nil, 0, err.Trace()
+	}
+	// check if object exists
+	if _, ok := bucketMetadata.Buckets[b.getBucketName()].BucketObjects[objectName]; !ok {
+		return nil, 0, probe.NewError(ObjectNotFound{Object: objectName})
+	}
+	objMetadata, err := b.readObjectMetadata(normalizeObjectName(objectName))
+	if err != nil {
+		return nil, 0, err.Trace()
+	}
+	// read and reply back to GetObject() request in a go-routine
+	go b.readObjectData(normalizeObjectName(objectName), writer, objMetadata)
+	return reader, objMetadata.Size, nil
+}
+
+// WriteObject - write a new object into bucket
+func (b bucket) WriteObject(objectName string, objectData io.Reader, size int64, expectedMD5Sum string, metadata map[string]string, signature *signV4.Signature) (ObjectMetadata, *probe.Error) {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+	if objectName == "" || objectData == nil {
+		return ObjectMetadata{}, probe.NewError(InvalidArgument{})
+	}
+	writers, err := b.getObjectWriters(normalizeObjectName(objectName), "data")
+	if err != nil {
+		return ObjectMetadata{}, err.Trace()
+	}
+	sumMD5 := md5.New()
+	sum512 := sha512.New()
+	var sum256 hash.Hash
+	var mwriter io.Writer
+
+	if signature != nil {
+		sum256 = sha256.New()
+		mwriter = io.MultiWriter(sumMD5, sum256, sum512)
+	} else {
+		mwriter = io.MultiWriter(sumMD5, sum512)
+	}
+	objMetadata := ObjectMetadata{}
+	objMetadata.Version = objectMetadataVersion
+	objMetadata.Created = time.Now().UTC()
+	// if total writers are only '1' do not compute erasure
+	switch len(writers) == 1 {
+	case true:
+		mw := io.MultiWriter(writers[0], mwriter)
+		totalLength, err := io.Copy(mw, objectData)
+		if err != nil {
+			CleanupWritersOnError(writers)
+			return ObjectMetadata{}, probe.NewError(err)
+		}
+		objMetadata.Size = totalLength
+	case false:
+		// calculate data and parity dictated by total number of writers
+		k, m, err := b.getDataAndParity(len(writers))
+		if err != nil {
+			CleanupWritersOnError(writers)
+			return ObjectMetadata{}, err.Trace()
+		}
+		// write encoded data with k, m and writers
+		chunkCount, totalLength, err := b.writeObjectData(k, m, writers, objectData, size, mwriter)
+		if err != nil {
+			CleanupWritersOnError(writers)
+			return ObjectMetadata{}, err.Trace()
+		}
+		/// xlMetadata section
+		objMetadata.BlockSize = blockSize
+		objMetadata.ChunkCount = chunkCount
+		objMetadata.DataDisks = k
+		objMetadata.ParityDisks = m
+		objMetadata.Size = int64(totalLength)
+	}
+	objMetadata.Bucket = b.getBucketName()
+	objMetadata.Object = objectName
+	dataMD5sum := sumMD5.Sum(nil)
+	dataSHA512sum := sum512.Sum(nil)
+	if signature != nil {
+		ok, err := signature.DoesSignatureMatch(hex.EncodeToString(sum256.Sum(nil)))
+		if err != nil {
+			// error occurred while doing signature calculation, we return and also cleanup any temporary writers.
+			CleanupWritersOnError(writers)
+			return ObjectMetadata{}, err.Trace()
+		}
+		if !ok {
+			// purge all writers, when control flow reaches here
+			//
+			// Signature mismatch occurred all temp files to be removed and all data purged.
+			CleanupWritersOnError(writers)
+			return ObjectMetadata{}, probe.NewError(signV4.SigDoesNotMatch{})
+		}
+	}
+	objMetadata.MD5Sum = hex.EncodeToString(dataMD5sum)
+	objMetadata.SHA512Sum = hex.EncodeToString(dataSHA512sum)
+
+	// Verify if the written object is equal to what is expected, only if it is requested as such
+	if strings.TrimSpace(expectedMD5Sum) != "" {
+		if err := b.isMD5SumEqual(strings.TrimSpace(expectedMD5Sum), objMetadata.MD5Sum); err != nil {
+			return ObjectMetadata{}, err.Trace()
+		}
+	}
+	objMetadata.Metadata = metadata
+	// write object specific metadata
+	if err := b.writeObjectMetadata(normalizeObjectName(objectName), objMetadata); err != nil {
+		// purge all writers, when control flow reaches here
+		CleanupWritersOnError(writers)
+		return ObjectMetadata{}, err.Trace()
+	}
+	// close all writers, when control flow reaches here
+	for _, writer := range writers {
+		writer.Close()
+	}
+	return objMetadata, nil
+}
+
+// isMD5SumEqual - returns error if md5sum mismatches, other its `nil`
+func (b bucket) isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) *probe.Error {
+	if strings.TrimSpace(expectedMD5Sum) != "" && strings.TrimSpace(actualMD5Sum) != "" {
+		expectedMD5SumBytes, err := hex.DecodeString(expectedMD5Sum)
+		if err != nil {
+			return probe.NewError(err)
+		}
+		actualMD5SumBytes, err := hex.DecodeString(actualMD5Sum)
+		if err != nil {
+			return probe.NewError(err)
+		}
+		if !bytes.Equal(expectedMD5SumBytes, actualMD5SumBytes) {
+			return probe.NewError(BadDigest{})
+		}
+		return nil
+	}
+	return probe.NewError(InvalidArgument{})
+}
+
+// writeObjectMetadata - write additional object metadata
+func (b bucket) writeObjectMetadata(objectName string, objMetadata ObjectMetadata) *probe.Error {
+	if objMetadata.Object == "" {
+		return probe.NewError(InvalidArgument{})
+	}
+	objMetadataWriters, err := b.getObjectWriters(objectName, objectMetadataConfig)
+	if err != nil {
+		return err.Trace()
+	}
+	for _, objMetadataWriter := range objMetadataWriters {
+		jenc := json.NewEncoder(objMetadataWriter)
+		if err := jenc.Encode(&objMetadata); err != nil {
+			// Close writers and purge all temporary entries
+			CleanupWritersOnError(objMetadataWriters)
+			return probe.NewError(err)
+		}
+	}
+	for _, objMetadataWriter := range objMetadataWriters {
+		objMetadataWriter.Close()
+	}
+	return nil
+}
+
+// readObjectMetadata - read object metadata
+func (b bucket) readObjectMetadata(objectName string) (ObjectMetadata, *probe.Error) {
+	if objectName == "" {
+		return ObjectMetadata{}, probe.NewError(InvalidArgument{})
+	}
+	objMetadata := ObjectMetadata{}
+	objMetadataReaders, err := b.getObjectReaders(objectName, objectMetadataConfig)
+	if err != nil {
+		return ObjectMetadata{}, err.Trace()
+	}
+	for _, objMetadataReader := range objMetadataReaders {
+		defer objMetadataReader.Close()
+	}
+	{
+		var err error
+		for _, objMetadataReader := range objMetadataReaders {
+			jdec := json.NewDecoder(objMetadataReader)
+			if err = jdec.Decode(&objMetadata); err == nil {
+				return objMetadata, nil
+			}
+		}
+		return ObjectMetadata{}, probe.NewError(err)
+	}
+}
+
+// TODO - This a temporary normalization of objectNames, need to find a better way
+//
+// normalizedObjectName - all objectNames with "/" get normalized to a simple objectName
+//
+// example:
+// user provided value - "this/is/my/deep/directory/structure"
+// xl normalized value - "this-is-my-deep-directory-structure"
+//
+func normalizeObjectName(objectName string) string {
+	// replace every '/' with '-'
+	return strings.Replace(objectName, "/", "-", -1)
+}
+
+// getDataAndParity - calculate k, m (data and parity) values from number of disks
+func (b bucket) getDataAndParity(totalWriters int) (k uint8, m uint8, err *probe.Error) {
+	if totalWriters <= 1 {
+		return 0, 0, probe.NewError(InvalidArgument{})
+	}
+	quotient := totalWriters / 2 // not using float or abs to let integer round off to lower value
+	// quotient cannot be bigger than (255 / 2) = 127
+	if quotient > 127 {
+		return 0, 0, probe.NewError(ParityOverflow{})
+	}
+	remainder := totalWriters % 2 // will be 1 for odd and 0 for even numbers
+	k = uint8(quotient + remainder)
+	m = uint8(quotient)
+	return k, m, nil
+}
+
+// writeObjectData -
+func (b bucket) writeObjectData(k, m uint8, writers []io.WriteCloser, objectData io.Reader, size int64, hashWriter io.Writer) (int, int, *probe.Error) {
+	encoder, err := newEncoder(k, m)
+	if err != nil {
+		return 0, 0, err.Trace()
+	}
+	chunkSize := int64(10 * 1024 * 1024)
+	chunkCount := 0
+	totalLength := 0
+
+	var e error
+	for e == nil {
+		var length int
+		inputData := make([]byte, chunkSize)
+		length, e = objectData.Read(inputData)
+		if length != 0 {
+			encodedBlocks, err := encoder.Encode(inputData[0:length])
+			if err != nil {
+				return 0, 0, err.Trace()
+			}
+			if _, err := hashWriter.Write(inputData[0:length]); err != nil {
+				return 0, 0, probe.NewError(err)
+			}
+			for blockIndex, block := range encodedBlocks {
+				errCh := make(chan error, 1)
+				go func(writer io.Writer, reader io.Reader, errCh chan<- error) {
+					defer close(errCh)
+					_, err := io.Copy(writer, reader)
+					errCh <- err
+				}(writers[blockIndex], bytes.NewReader(block), errCh)
+				if err := <-errCh; err != nil {
+					// Returning error is fine here CleanupErrors() would cleanup writers
+					return 0, 0, probe.NewError(err)
+				}
+			}
+			totalLength += length
+			chunkCount = chunkCount + 1
+		}
+	}
+	if e != io.EOF {
+		return 0, 0, probe.NewError(e)
+	}
+	return chunkCount, totalLength, nil
+}
+
+// readObjectData -
+func (b bucket) readObjectData(objectName string, writer *io.PipeWriter, objMetadata ObjectMetadata) {
+	readers, err := b.getObjectReaders(objectName, "data")
+	if err != nil {
+		writer.CloseWithError(probe.WrapError(err))
+		return
+	}
+	for _, reader := range readers {
+		defer reader.Close()
+	}
+	var expected512Sum, expectedMd5sum []byte
+	{
+		var err error
+		expectedMd5sum, err = hex.DecodeString(objMetadata.MD5Sum)
+		if err != nil {
+			writer.CloseWithError(probe.WrapError(probe.NewError(err)))
+			return
+		}
+		expected512Sum, err = hex.DecodeString(objMetadata.SHA512Sum)
+		if err != nil {
+			writer.CloseWithError(probe.WrapError(probe.NewError(err)))
+			return
+		}
+	}
+	hasher := md5.New()
+	sum512hasher := sha256.New()
+	mwriter := io.MultiWriter(writer, hasher, sum512hasher)
+	switch len(readers) > 1 {
+	case true:
+		encoder, err := newEncoder(objMetadata.DataDisks, objMetadata.ParityDisks)
+		if err != nil {
+			writer.CloseWithError(probe.WrapError(err))
+			return
+		}
+		totalLeft := objMetadata.Size
+		for i := 0; i < objMetadata.ChunkCount; i++ {
+			decodedData, err := b.decodeEncodedData(totalLeft, int64(objMetadata.BlockSize), readers, encoder, writer)
+			if err != nil {
+				writer.CloseWithError(probe.WrapError(err))
+				return
+			}
+			if _, err := io.Copy(mwriter, bytes.NewReader(decodedData)); err != nil {
+				writer.CloseWithError(probe.WrapError(probe.NewError(err)))
+				return
+			}
+			totalLeft = totalLeft - int64(objMetadata.BlockSize)
+		}
+	case false:
+		_, err := io.Copy(writer, readers[0])
+		if err != nil {
+			writer.CloseWithError(probe.WrapError(probe.NewError(err)))
+			return
+		}
+	}
+	// check if decodedData md5sum matches
+	if !bytes.Equal(expectedMd5sum, hasher.Sum(nil)) {
+		writer.CloseWithError(probe.WrapError(probe.NewError(ChecksumMismatch{})))
+		return
+	}
+	if !bytes.Equal(expected512Sum, sum512hasher.Sum(nil)) {
+		writer.CloseWithError(probe.WrapError(probe.NewError(ChecksumMismatch{})))
+		return
+	}
+	writer.Close()
+	return
+}
+
+// decodeEncodedData -
+func (b bucket) decodeEncodedData(totalLeft, blockSize int64, readers map[int]io.ReadCloser, encoder encoder, writer *io.PipeWriter) ([]byte, *probe.Error) {
+	var curBlockSize int64
+	if blockSize < totalLeft {
+		curBlockSize = blockSize
+	} else {
+		curBlockSize = totalLeft
+	}
+	curChunkSize, err := encoder.GetEncodedBlockLen(int(curBlockSize))
+	if err != nil {
+		return nil, err.Trace()
+	}
+	encodedBytes := make([][]byte, encoder.k+encoder.m)
+	errCh := make(chan error, len(readers))
+	var errRet error
+	var readCnt int
+
+	for i, reader := range readers {
+		go func(reader io.Reader, i int) {
+			encodedBytes[i] = make([]byte, curChunkSize)
+			_, err := io.ReadFull(reader, encodedBytes[i])
+			if err != nil {
+				encodedBytes[i] = nil
+				errCh <- err
+				return
+			}
+			errCh <- nil
+		}(reader, i)
+		// read through errCh for any errors
+		err := <-errCh
+		if err != nil {
+			errRet = err
+		} else {
+			readCnt++
+		}
+	}
+	if readCnt < int(encoder.k) {
+		return nil, probe.NewError(errRet)
+	}
+	decodedData, err := encoder.Decode(encodedBytes, int(curBlockSize))
+	if err != nil {
+		return nil, err.Trace()
+	}
+	return decodedData, nil
+}
+
+// getObjectReaders -
+func (b bucket) getObjectReaders(objectName, objectMeta string) (map[int]io.ReadCloser, *probe.Error) {
+	readers := make(map[int]io.ReadCloser)
+	var disks map[int]block.Block
+	var err *probe.Error
+	nodeSlice := 0
+	for _, node := range b.nodes {
+		disks, err = node.ListDisks()
+		if err != nil {
+			return nil, err.Trace()
+		}
+		for order, disk := range disks {
+			var objectSlice io.ReadCloser
+			bucketSlice := fmt.Sprintf("%s$%d$%d", b.name, nodeSlice, order)
+			objectPath := filepath.Join(b.xlName, bucketSlice, objectName, objectMeta)
+			objectSlice, err = disk.Open(objectPath)
+			if err == nil {
+				readers[order] = objectSlice
+			}
+		}
+		nodeSlice = nodeSlice + 1
+	}
+	if err != nil {
+		return nil, err.Trace()
+	}
+	return readers, nil
+}
+
+// getObjectWriters -
+func (b bucket) getObjectWriters(objectName, objectMeta string) ([]io.WriteCloser, *probe.Error) {
+	var writers []io.WriteCloser
+	nodeSlice := 0
+	for _, node := range b.nodes {
+		disks, err := node.ListDisks()
+		if err != nil {
+			return nil, err.Trace()
+		}
+		writers = make([]io.WriteCloser, len(disks))
+		for order, disk := range disks {
+			bucketSlice := fmt.Sprintf("%s$%d$%d", b.name, nodeSlice, order)
+			objectPath := filepath.Join(b.xlName, bucketSlice, objectName, objectMeta)
+			objectSlice, err := disk.CreateFile(objectPath)
+			if err != nil {
+				return nil, err.Trace()
+			}
+			writers[order] = objectSlice
+		}
+		nodeSlice = nodeSlice + 1
+	}
+	return writers, nil
+}

pkg/xl/cache/data/data.go

@@ -0,0 +1,204 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package data implements in memory caching methods for data
+package data
+
+import (
+	"container/list"
+	"sync"
+	"time"
+)
+
+var noExpiration = time.Duration(0)
+
+// Cache holds the required variables to compose an in memory cache system
+// which also provides expiring key mechanism and also maxSize
+type Cache struct {
+	// Mutex is used for handling the concurrent
+	// read/write requests for cache
+	sync.Mutex
+
+	// items hold the cached objects
+	items *list.List
+
+	// reverseItems holds the time that related item's updated at
+	reverseItems map[interface{}]*list.Element
+
+	// maxSize is a total size for overall cache
+	maxSize uint64
+
+	// currentSize is a current size in memory
+	currentSize uint64
+
+	// OnEvicted - callback function for eviction
+	OnEvicted func(a ...interface{})
+
+	// totalEvicted counter to keep track of total expirations
+	totalEvicted int
+}
+
+// Stats current cache statistics
+type Stats struct {
+	Bytes   uint64
+	Items   int
+	Evicted int
+}
+
+type element struct {
+	key   interface{}
+	value []byte
+}
+
+// NewCache creates an inmemory cache
+//
+// maxSize is used for expiring objects before we run out of memory
+// expiration is used for expiration of a key from cache
+func NewCache(maxSize uint64) *Cache {
+	return &Cache{
+		items:        list.New(),
+		reverseItems: make(map[interface{}]*list.Element),
+		maxSize:      maxSize,
+	}
+}
+
+// SetMaxSize set a new max size
+func (r *Cache) SetMaxSize(maxSize uint64) {
+	r.Lock()
+	defer r.Unlock()
+	r.maxSize = maxSize
+	return
+}
+
+// Stats get current cache statistics
+func (r *Cache) Stats() Stats {
+	return Stats{
+		Bytes:   r.currentSize,
+		Items:   r.items.Len(),
+		Evicted: r.totalEvicted,
+	}
+}
+
+// Get returns a value of a given key if it exists
+func (r *Cache) Get(key interface{}) ([]byte, bool) {
+	r.Lock()
+	defer r.Unlock()
+	ele, hit := r.reverseItems[key]
+	if !hit {
+		return nil, false
+	}
+	r.items.MoveToFront(ele)
+	return ele.Value.(*element).value, true
+}
+
+// Len returns length of the value of a given key, returns zero if key doesn't exist
+func (r *Cache) Len(key interface{}) int {
+	r.Lock()
+	defer r.Unlock()
+	_, ok := r.reverseItems[key]
+	if !ok {
+		return 0
+	}
+	return len(r.reverseItems[key].Value.(*element).value)
+}
+
+// Append will append new data to an existing key,
+// if key doesn't exist it behaves like Set()
+func (r *Cache) Append(key interface{}, value []byte) bool {
+	r.Lock()
+	defer r.Unlock()
+	valueLen := uint64(len(value))
+	if r.maxSize > 0 {
+		// check if the size of the object is not bigger than the
+		// capacity of the cache
+		if valueLen > r.maxSize {
+			return false
+		}
+		// remove random key if only we reach the maxSize threshold
+		for (r.currentSize + valueLen) > r.maxSize {
+			r.doDeleteOldest()
+			break
+		}
+	}
+	ele, hit := r.reverseItems[key]
+	if !hit {
+		ele := r.items.PushFront(&element{key, value})
+		r.currentSize += valueLen
+		r.reverseItems[key] = ele
+		return true
+	}
+	r.items.MoveToFront(ele)
+	r.currentSize += valueLen
+	ele.Value.(*element).value = append(ele.Value.(*element).value, value...)
+	return true
+}
+
+// Set will persist a value to the cache
+func (r *Cache) Set(key interface{}, value []byte) bool {
+	r.Lock()
+	defer r.Unlock()
+	valueLen := uint64(len(value))
+	if r.maxSize > 0 {
+		// check if the size of the object is not bigger than the
+		// capacity of the cache
+		if valueLen > r.maxSize {
+			return false
+		}
+		// remove random key if only we reach the maxSize threshold
+		for (r.currentSize + valueLen) > r.maxSize {
+			r.doDeleteOldest()
+		}
+	}
+	if _, hit := r.reverseItems[key]; hit {
+		return false
+	}
+	ele := r.items.PushFront(&element{key, value})
+	r.currentSize += valueLen
+	r.reverseItems[key] = ele
+	return true
+}
+
+// Delete deletes a given key if exists
+func (r *Cache) Delete(key interface{}) {
+	r.Lock()
+	defer r.Unlock()
+	ele, ok := r.reverseItems[key]
+	if !ok {
+		return
+	}
+	if ele != nil {
+		r.currentSize -= uint64(len(r.reverseItems[key].Value.(*element).value))
+		r.items.Remove(ele)
+		delete(r.reverseItems, key)
+		r.totalEvicted++
+		if r.OnEvicted != nil {
+			r.OnEvicted(key)
+		}
+	}
+}
+
+func (r *Cache) doDeleteOldest() {
+	ele := r.items.Back()
+	if ele != nil {
+		r.currentSize -= uint64(len(r.reverseItems[ele.Value.(*element).key].Value.(*element).value))
+		delete(r.reverseItems, ele.Value.(*element).key)
+		r.items.Remove(ele)
+		r.totalEvicted++
+		if r.OnEvicted != nil {
+			r.OnEvicted(ele.Value.(*element).key)
+		}
+	}
+}

pkg/xl/cache/data/data_test.go

@@ -0,0 +1,45 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package data
+
+import (
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type MySuite struct{}
+
+var _ = Suite(&MySuite{})
+
+func (s *MySuite) TestCache(c *C) {
+	cache := NewCache(1000)
+	data := []byte("Hello, world!")
+	ok := cache.Set("filename", data)
+
+	c.Assert(ok, Equals, true)
+	storedata, ok := cache.Get("filename")
+
+	c.Assert(ok, Equals, true)
+	c.Assert(data, DeepEquals, storedata)
+
+	cache.Delete("filename")
+	_, ok = cache.Get("filename")
+	c.Assert(ok, Equals, false)
+}

pkg/xl/cache/metadata/metadata.go

@@ -0,0 +1,110 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package metadata implements in memory caching methods for metadata information
+package metadata
+
+import (
+	"sync"
+	"time"
+)
+
+var noExpiration = time.Duration(0)
+
+// Cache holds the required variables to compose an in memory cache system
+// which also provides expiring key mechanism and also maxSize
+type Cache struct {
+	// Mutex is used for handling the concurrent
+	// read/write requests for cache
+	sync.Mutex
+
+	// items hold the cached objects
+	items map[string]interface{}
+
+	// updatedAt holds the time that related item's updated at
+	updatedAt map[string]time.Time
+}
+
+// Stats current cache statistics
+type Stats struct {
+	Items int
+}
+
+// NewCache creates an inmemory cache
+//
+func NewCache() *Cache {
+	return &Cache{
+		items:     make(map[string]interface{}),
+		updatedAt: map[string]time.Time{},
+	}
+}
+
+// Stats get current cache statistics
+func (r *Cache) Stats() Stats {
+	return Stats{
+		Items: len(r.items),
+	}
+}
+
+// GetAll returs all the items
+func (r *Cache) GetAll() map[string]interface{} {
+	r.Lock()
+	defer r.Unlock()
+	// copy
+	items := r.items
+	return items
+}
+
+// Get returns a value of a given key if it exists
+func (r *Cache) Get(key string) interface{} {
+	r.Lock()
+	defer r.Unlock()
+	value, ok := r.items[key]
+	if !ok {
+		return nil
+	}
+	return value
+}
+
+// Exists returns true if key exists
+func (r *Cache) Exists(key string) bool {
+	r.Lock()
+	defer r.Unlock()
+	_, ok := r.items[key]
+	return ok
+}
+
+// Set will persist a value to the cache
+func (r *Cache) Set(key string, value interface{}) bool {
+	r.Lock()
+	defer r.Unlock()
+	r.items[key] = value
+	return true
+}
+
+// Delete deletes a given key if exists
+func (r *Cache) Delete(key string) {
+	r.Lock()
+	defer r.Unlock()
+	r.doDelete(key)
+}
+
+func (r *Cache) doDelete(key string) {
+	if _, ok := r.items[key]; ok {
+		delete(r.items, key)
+		delete(r.updatedAt, key)
+	}
+}

pkg/xl/cache/metadata/metadata_test.go

@@ -0,0 +1,46 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package metadata
+
+import (
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type MySuite struct{}
+
+var _ = Suite(&MySuite{})
+
+func (s *MySuite) TestCache(c *C) {
+	cache := NewCache()
+	data := []byte("Hello, world!")
+	ok := cache.Set("filename", data)
+
+	c.Assert(ok, Equals, true)
+	storedata := cache.Get("filename")
+
+	c.Assert(ok, Equals, true)
+	c.Assert(data, DeepEquals, storedata)
+
+	cache.Delete("filename")
+
+	ok = cache.Exists("filename")
+	c.Assert(ok, Equals, false)
+}

pkg/xl/common.go

@@ -0,0 +1,190 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package xl
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"regexp"
+	"sort"
+	"strings"
+	"unicode/utf8"
+
+	"github.com/minio/minio/pkg/atomic"
+)
+
+// IsValidXL - verify xl name is correct
+func IsValidXL(xlName string) bool {
+	if len(xlName) < 3 || len(xlName) > 63 {
+		return false
+	}
+	if xlName[0] == '.' || xlName[len(xlName)-1] == '.' {
+		return false
+	}
+	if match, _ := regexp.MatchString("\\.\\.", xlName); match == true {
+		return false
+	}
+	// We don't support xlNames with '.' in them
+	match, _ := regexp.MatchString("^[a-zA-Z][a-zA-Z0-9\\-]+[a-zA-Z0-9]$", xlName)
+	return match
+}
+
+// IsValidBucket - verify bucket name in accordance with
+//  - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html
+func IsValidBucket(bucket string) bool {
+	if len(bucket) < 3 || len(bucket) > 63 {
+		return false
+	}
+	if bucket[0] == '.' || bucket[len(bucket)-1] == '.' {
+		return false
+	}
+	if match, _ := regexp.MatchString("\\.\\.", bucket); match == true {
+		return false
+	}
+	// We don't support buckets with '.' in them
+	match, _ := regexp.MatchString("^[a-zA-Z][a-zA-Z0-9\\-]+[a-zA-Z0-9]$", bucket)
+	return match
+}
+
+// IsValidObjectName - verify object name in accordance with
+//   - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
+func IsValidObjectName(object string) bool {
+	if strings.TrimSpace(object) == "" {
+		return false
+	}
+	if len(object) > 1024 || len(object) == 0 {
+		return false
+	}
+	if !utf8.ValidString(object) {
+		return false
+	}
+	return true
+}
+
+// IsValidPrefix - verify prefix name is correct, an empty prefix is valid
+func IsValidPrefix(prefix string) bool {
+	if strings.TrimSpace(prefix) == "" {
+		return true
+	}
+	return IsValidObjectName(prefix)
+}
+
+// ProxyWriter implements io.Writer to trap written bytes
+type ProxyWriter struct {
+	writer       io.Writer
+	writtenBytes []byte
+}
+
+func (r *ProxyWriter) Write(p []byte) (n int, err error) {
+	n, err = r.writer.Write(p)
+	if err != nil {
+		return
+	}
+	r.writtenBytes = append(r.writtenBytes, p[0:n]...)
+	return
+}
+
+// NewProxyWriter - wrap around a given writer with ProxyWriter
+func NewProxyWriter(w io.Writer) *ProxyWriter {
+	return &ProxyWriter{writer: w, writtenBytes: nil}
+}
+
+// Delimiter delims the string at delimiter
+func Delimiter(object, delimiter string) string {
+	readBuffer := bytes.NewBufferString(object)
+	reader := bufio.NewReader(readBuffer)
+	stringReader := strings.NewReader(delimiter)
+	delimited, _ := stringReader.ReadByte()
+	delimitedStr, _ := reader.ReadString(delimited)
+	return delimitedStr
+}
+
+// RemoveDuplicates removes duplicate elements from a slice
+func RemoveDuplicates(slice []string) []string {
+	newSlice := []string{}
+	seen := make(map[string]struct{})
+	for _, val := range slice {
+		if _, ok := seen[val]; !ok {
+			newSlice = append(newSlice, val)
+			seen[val] = struct{}{} // avoiding byte allocation
+		}
+	}
+	return newSlice
+}
+
+// TrimPrefix trims off a prefix string from all the elements in a given slice
+func TrimPrefix(objects []string, prefix string) []string {
+	var results []string
+	for _, object := range objects {
+		results = append(results, strings.TrimPrefix(object, prefix))
+	}
+	return results
+}
+
+// HasNoDelimiter provides a new slice from an input slice which has elements without delimiter
+func HasNoDelimiter(objects []string, delim string) []string {
+	var results []string
+	for _, object := range objects {
+		if !strings.Contains(object, delim) {
+			results = append(results, object)
+		}
+	}
+	return results
+}
+
+// HasDelimiter provides a new slice from an input slice which has elements with a delimiter
+func HasDelimiter(objects []string, delim string) []string {
+	var results []string
+	for _, object := range objects {
+		if strings.Contains(object, delim) {
+			results = append(results, object)
+		}
+	}
+	return results
+}
+
+// SplitDelimiter provides a new slice from an input slice by splitting a delimiter
+func SplitDelimiter(objects []string, delim string) []string {
+	var results []string
+	for _, object := range objects {
+		parts := strings.Split(object, delim)
+		results = append(results, parts[0]+delim)
+	}
+	return results
+}
+
+// SortUnique sort a slice in lexical order, removing duplicate elements
+func SortUnique(objects []string) []string {
+	objectMap := make(map[string]string)
+	for _, v := range objects {
+		objectMap[v] = v
+	}
+	var results []string
+	for k := range objectMap {
+		results = append(results, k)
+	}
+	sort.Strings(results)
+	return results
+}
+
+// CleanupWritersOnError purge writers on error
+func CleanupWritersOnError(writers []io.WriteCloser) {
+	for _, writer := range writers {
+		writer.(*atomic.File).CloseAndPurge()
+	}
+}

pkg/xl/config.go

@@ -0,0 +1,80 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package xl
+
+import (
+	"os/user"
+	"path/filepath"
+
+	"github.com/minio/minio/pkg/probe"
+	"github.com/minio/minio/pkg/quick"
+)
+
+// getXLConfigPath get xl config file path
+func getXLConfigPath() (string, *probe.Error) {
+	if customConfigPath != "" {
+		return customConfigPath, nil
+	}
+	u, err := user.Current()
+	if err != nil {
+		return "", probe.NewError(err)
+	}
+	xlConfigPath := filepath.Join(u.HomeDir, ".minio", "xl.json")
+	return xlConfigPath, nil
+}
+
+// internal variable only accessed via get/set methods
+var customConfigPath string
+
+// SetXLConfigPath - set custom xl config path
+func SetXLConfigPath(configPath string) {
+	customConfigPath = configPath
+}
+
+// SaveConfig save xl config
+func SaveConfig(a *Config) *probe.Error {
+	xlConfigPath, err := getXLConfigPath()
+	if err != nil {
+		return err.Trace()
+	}
+	qc, err := quick.New(a)
+	if err != nil {
+		return err.Trace()
+	}
+	if err := qc.Save(xlConfigPath); err != nil {
+		return err.Trace()
+	}
+	return nil
+}
+
+// LoadConfig load xl config
+func LoadConfig() (*Config, *probe.Error) {
+	xlConfigPath, err := getXLConfigPath()
+	if err != nil {
+		return nil, err.Trace()
+	}
+	a := &Config{}
+	a.Version = "0.0.1"
+	qc, err := quick.New(a)
+	if err != nil {
+		return nil, err.Trace()
+	}
+	if err := qc.Load(xlConfigPath); err != nil {
+		return nil, err.Trace()
+	}
+	return qc.Data().(*Config), nil
+}

pkg/xl/definitions.go

@@ -0,0 +1,157 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package xl
+
+import "time"
+
+// ObjectMetadata container for object on xl system
+type ObjectMetadata struct {
+	// version
+	Version string `json:"version"`
+
+	// object metadata
+	Created time.Time `json:"created"`
+	Bucket  string    `json:"bucket"`
+	Object  string    `json:"object"`
+	Size    int64     `json:"size"`
+
+	// erasure
+	DataDisks   uint8 `json:"sys.erasureK"`
+	ParityDisks uint8 `json:"sys.erasureM"`
+	BlockSize   int   `json:"sys.blockSize"`
+	ChunkCount  int   `json:"sys.chunkCount"`
+
+	// checksums
+	MD5Sum    string `json:"sys.md5sum"`
+	SHA512Sum string `json:"sys.sha512sum"`
+
+	// metadata
+	Metadata map[string]string `json:"metadata"`
+}
+
+// Metadata container for xl metadata
+type Metadata struct {
+	Version string `json:"version"`
+}
+
+// AllBuckets container for all buckets
+type AllBuckets struct {
+	Version string                    `json:"version"`
+	Buckets map[string]BucketMetadata `json:"buckets"`
+}
+
+// BucketMetadata container for bucket level metadata
+type BucketMetadata struct {
+	Version       string                      `json:"version"`
+	Name          string                      `json:"name"`
+	ACL           BucketACL                   `json:"acl"`
+	Created       time.Time                   `json:"created"`
+	Multiparts    map[string]MultiPartSession `json:"multiparts"`
+	Metadata      map[string]string           `json:"metadata"`
+	BucketObjects map[string]struct{}         `json:"objects"`
+}
+
+// ListObjectsResults container for list objects response
+type ListObjectsResults struct {
+	Objects        map[string]ObjectMetadata `json:"objects"`
+	CommonPrefixes []string                  `json:"commonPrefixes"`
+	IsTruncated    bool                      `json:"isTruncated"`
+}
+
+// MultiPartSession multipart session
+type MultiPartSession struct {
+	UploadID   string                  `json:"uploadId"`
+	Initiated  time.Time               `json:"initiated"`
+	Parts      map[string]PartMetadata `json:"parts"`
+	TotalParts int                     `json:"total-parts"`
+}
+
+// PartMetadata - various types of individual part resources
+type PartMetadata struct {
+	PartNumber   int
+	LastModified time.Time
+	ETag         string
+	Size         int64
+}
+
+// CompletePart - completed part container
+type CompletePart struct {
+	PartNumber int
+	ETag       string
+}
+
+// completedParts is a sortable interface for Part slice
+type completedParts []CompletePart
+
+func (a completedParts) Len() int           { return len(a) }
+func (a completedParts) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
+
+// CompleteMultipartUpload container for completing multipart upload
+type CompleteMultipartUpload struct {
+	Part []CompletePart
+}
+
+// ObjectResourcesMetadata - various types of object resources
+type ObjectResourcesMetadata struct {
+	Bucket               string
+	EncodingType         string
+	Key                  string
+	UploadID             string
+	StorageClass         string
+	PartNumberMarker     int
+	NextPartNumberMarker int
+	MaxParts             int
+	IsTruncated          bool
+
+	Part []*PartMetadata
+}
+
+// UploadMetadata container capturing metadata on in progress multipart upload in a given bucket
+type UploadMetadata struct {
+	Key          string
+	UploadID     string
+	StorageClass string
+	Initiated    time.Time
+}
+
+// BucketMultipartResourcesMetadata - various types of bucket resources for inprogress multipart uploads
+type BucketMultipartResourcesMetadata struct {
+	KeyMarker          string
+	UploadIDMarker     string
+	NextKeyMarker      string
+	NextUploadIDMarker string
+	EncodingType       string
+	MaxUploads         int
+	IsTruncated        bool
+	Upload             []*UploadMetadata
+	Prefix             string
+	Delimiter          string
+	CommonPrefixes     []string
+}
+
+// BucketResourcesMetadata - various types of bucket resources
+type BucketResourcesMetadata struct {
+	Prefix         string
+	Marker         string
+	NextMarker     string
+	Maxkeys        int
+	EncodingType   string
+	Delimiter      string
+	IsTruncated    bool
+	CommonPrefixes []string
+}

pkg/xl/encoder.go

@@ -0,0 +1,71 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package xl
+
+import (
+	encoding "github.com/minio/minio/pkg/erasure"
+	"github.com/minio/minio/pkg/probe"
+)
+
+// encoder internal struct
+type encoder struct {
+	encoder *encoding.Erasure
+	k, m    uint8
+}
+
+// newEncoder - instantiate a new encoder
+func newEncoder(k, m uint8) (encoder, *probe.Error) {
+	e := encoder{}
+	params, err := encoding.ValidateParams(k, m)
+	if err != nil {
+		return encoder{}, probe.NewError(err)
+	}
+	e.encoder = encoding.NewErasure(params)
+	e.k = k
+	e.m = m
+	return e, nil
+}
+
+// TODO - think again if this is needed
+// GetEncodedBlockLen - wrapper around erasure function with the same name
+func (e encoder) GetEncodedBlockLen(dataLength int) (int, *probe.Error) {
+	if dataLength <= 0 {
+		return 0, probe.NewError(InvalidArgument{})
+	}
+	return encoding.GetEncodedBlockLen(dataLength, e.k), nil
+}
+
+// Encode - erasure code input bytes
+func (e encoder) Encode(data []byte) ([][]byte, *probe.Error) {
+	if data == nil {
+		return nil, probe.NewError(InvalidArgument{})
+	}
+	encodedData, err := e.encoder.Encode(data)
+	if err != nil {
+		return nil, probe.NewError(err)
+	}
+	return encodedData, nil
+}
+
+// Decode - erasure decode input encoded bytes
+func (e encoder) Decode(encodedData [][]byte, dataLength int) ([]byte, *probe.Error) {
+	decodedData, err := e.encoder.Decode(encodedData, dataLength)
+	if err != nil {
+		return nil, probe.NewError(err)
+	}
+	return decodedData, nil
+}

pkg/xl/errors.go

@@ -0,0 +1,333 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package xl
+
+import "fmt"
+
+// InvalidArgument invalid argument
+type InvalidArgument struct{}
+
+func (e InvalidArgument) Error() string {
+	return "Invalid argument"
+}
+
+// UnsupportedFilesystem unsupported filesystem type
+type UnsupportedFilesystem struct {
+	Type string
+}
+
+func (e UnsupportedFilesystem) Error() string {
+	return "Unsupported filesystem: " + e.Type
+}
+
+// BucketNotFound bucket does not exist
+type BucketNotFound struct {
+	Bucket string
+}
+
+func (e BucketNotFound) Error() string {
+	return "Bucket not found: " + e.Bucket
+}
+
+// ObjectExists object exists
+type ObjectExists struct {
+	Object string
+}
+
+func (e ObjectExists) Error() string {
+	return "Object exists: " + e.Object
+}
+
+// ObjectNotFound object does not exist
+type ObjectNotFound struct {
+	Object string
+}
+
+func (e ObjectNotFound) Error() string {
+	return "Object not found: " + e.Object
+}
+
+// ObjectCorrupted object found to be corrupted
+type ObjectCorrupted struct {
+	Object string
+}
+
+func (e ObjectCorrupted) Error() string {
+	return "Object found corrupted: " + e.Object
+}
+
+// BucketExists bucket exists
+type BucketExists struct {
+	Bucket string
+}
+
+func (e BucketExists) Error() string {
+	return "Bucket exists: " + e.Bucket
+}
+
+// CorruptedBackend backend found to be corrupted
+type CorruptedBackend struct {
+	Backend string
+}
+
+func (e CorruptedBackend) Error() string {
+	return "Corrupted backend: " + e.Backend
+}
+
+// NotImplemented function not implemented
+type NotImplemented struct {
+	Function string
+}
+
+func (e NotImplemented) Error() string {
+	return "Not implemented: " + e.Function
+}
+
+// InvalidDisksArgument invalid number of disks per node
+type InvalidDisksArgument struct{}
+
+func (e InvalidDisksArgument) Error() string {
+	return "Invalid number of disks per node"
+}
+
+// BadDigest bad md5sum
+type BadDigest struct{}
+
+func (e BadDigest) Error() string {
+	return "Bad digest"
+}
+
+// ParityOverflow parity over flow
+type ParityOverflow struct{}
+
+func (e ParityOverflow) Error() string {
+	return "Parity overflow"
+}
+
+// ChecksumMismatch checksum mismatch
+type ChecksumMismatch struct{}
+
+func (e ChecksumMismatch) Error() string {
+	return "Checksum mismatch"
+}
+
+// MissingPOSTPolicy missing post policy
+type MissingPOSTPolicy struct{}
+
+func (e MissingPOSTPolicy) Error() string {
+	return "Missing POST policy in multipart form"
+}
+
+// InternalError - generic internal error
+type InternalError struct {
+}
+
+// BackendError - generic disk backend error
+type BackendError struct {
+	Path string
+}
+
+// BackendCorrupted - path has corrupted data
+type BackendCorrupted BackendError
+
+// APINotImplemented - generic API not implemented error
+type APINotImplemented struct {
+	API string
+}
+
+// GenericBucketError - generic bucket error
+type GenericBucketError struct {
+	Bucket string
+}
+
+// GenericObjectError - generic object error
+type GenericObjectError struct {
+	Bucket string
+	Object string
+}
+
+// ImplementationError - generic implementation error
+type ImplementationError struct {
+	Bucket string
+	Object string
+	Err    error
+}
+
+// DigestError - Generic Md5 error
+type DigestError struct {
+	Bucket string
+	Key    string
+	Md5    string
+}
+
+/// ACL related errors
+
+// InvalidACL - acl invalid
+type InvalidACL struct {
+	ACL string
+}
+
+func (e InvalidACL) Error() string {
+	return "Requested ACL is " + e.ACL + " invalid"
+}
+
+/// Bucket related errors
+
+// BucketNameInvalid - bucketname provided is invalid
+type BucketNameInvalid GenericBucketError
+
+// TooManyBuckets - total buckets exceeded
+type TooManyBuckets GenericBucketError
+
+/// Object related errors
+
+// EntityTooLarge - object size exceeds maximum limit
+type EntityTooLarge struct {
+	GenericObjectError
+	Size    string
+	MaxSize string
+}
+
+// ObjectNameInvalid - object name provided is invalid
+type ObjectNameInvalid GenericObjectError
+
+// InvalidDigest - md5 in request header invalid
+type InvalidDigest DigestError
+
+// Return string an error formatted as the given text
+func (e ImplementationError) Error() string {
+	error := ""
+	if e.Bucket != "" {
+		error = error + "Bucket: " + e.Bucket + " "
+	}
+	if e.Object != "" {
+		error = error + "Object: " + e.Object + " "
+	}
+	error = error + "Error: " + e.Err.Error()
+	return error
+}
+
+// EmbedError - wrapper function for error object
+func EmbedError(bucket, object string, err error) ImplementationError {
+	return ImplementationError{
+		Bucket: bucket,
+		Object: object,
+		Err:    err,
+	}
+}
+
+// Return string an error formatted as the given text
+func (e InternalError) Error() string {
+	return "Internal error occured"
+}
+
+// Return string an error formatted as the given text
+func (e APINotImplemented) Error() string {
+	return "Api not implemented: " + e.API
+}
+
+// Return string an error formatted as the given text
+func (e BucketNameInvalid) Error() string {
+	return "Bucket name invalid: " + e.Bucket
+}
+
+// Return string an error formatted as the given text
+func (e TooManyBuckets) Error() string {
+	return "Bucket limit exceeded beyond 100, cannot create bucket: " + e.Bucket
+}
+
+// Return string an error formatted as the given text
+func (e ObjectNameInvalid) Error() string {
+	return "Object name invalid: " + e.Bucket + "#" + e.Object
+}
+
+// Return string an error formatted as the given text
+func (e EntityTooLarge) Error() string {
+	return e.Bucket + "#" + e.Object + "with " + e.Size + "reached maximum allowed size limit " + e.MaxSize
+}
+
+// IncompleteBody You did not provide the number of bytes specified by the Content-Length HTTP header
+type IncompleteBody GenericObjectError
+
+// Return string an error formatted as the given text
+func (e IncompleteBody) Error() string {
+	return e.Bucket + "#" + e.Object + "has incomplete body"
+}
+
+// Return string an error formatted as the given text
+func (e BackendCorrupted) Error() string {
+	return "Backend corrupted: " + e.Path
+}
+
+// Return string an error formatted as the given text
+func (e InvalidDigest) Error() string {
+	return "Md5 provided " + e.Md5 + " is invalid"
+}
+
+// OperationNotPermitted - operation not permitted
+type OperationNotPermitted struct {
+	Op     string
+	Reason string
+}
+
+func (e OperationNotPermitted) Error() string {
+	return "Operation " + e.Op + " not permitted for reason: " + e.Reason
+}
+
+// InvalidRange - invalid range
+type InvalidRange struct {
+	Start  int64
+	Length int64
+}
+
+func (e InvalidRange) Error() string {
+	return fmt.Sprintf("Invalid range start:%d length:%d", e.Start, e.Length)
+}
+
+/// Multipart related errors
+
+// InvalidUploadID invalid upload id
+type InvalidUploadID struct {
+	UploadID string
+}
+
+func (e InvalidUploadID) Error() string {
+	return "Invalid upload id " + e.UploadID
+}
+
+// InvalidPart One or more of the specified parts could not be found
+type InvalidPart struct{}
+
+func (e InvalidPart) Error() string {
+	return "One or more of the specified parts could not be found"
+}
+
+// InvalidPartOrder parts are not ordered as Requested
+type InvalidPartOrder struct {
+	UploadID string
+}
+
+func (e InvalidPartOrder) Error() string {
+	return "Invalid part order sent for " + e.UploadID
+}
+
+// MalformedXML invalid xml format
+type MalformedXML struct{}
+
+func (e MalformedXML) Error() string {
+	return "Malformed XML"
+}

pkg/xl/heal.go

@@ -0,0 +1,69 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package xl
+
+import (
+	"encoding/json"
+	"fmt"
+	"path/filepath"
+
+	"github.com/minio/minio/pkg/probe"
+	"github.com/minio/minio/pkg/xl/block"
+)
+
+// healBuckets heal bucket slices
+func (xl API) healBuckets() *probe.Error {
+	if err := xl.listXLBuckets(); err != nil {
+		return err.Trace()
+	}
+	bucketMetadata, err := xl.getXLBucketMetadata()
+	if err != nil {
+		return err.Trace()
+	}
+	disks := make(map[int]block.Block)
+	for _, node := range xl.nodes {
+		nDisks, err := node.ListDisks()
+		if err != nil {
+			return err.Trace()
+		}
+		for k, v := range nDisks {
+			disks[k] = v
+		}
+	}
+	for order, disk := range disks {
+		if disk.IsUsable() {
+			disk.MakeDir(xl.config.XLName)
+			bucketMetadataWriter, err := disk.CreateFile(filepath.Join(xl.config.XLName, bucketMetadataConfig))
+			if err != nil {
+				return err.Trace()
+			}
+			defer bucketMetadataWriter.Close()
+			jenc := json.NewEncoder(bucketMetadataWriter)
+			if err := jenc.Encode(bucketMetadata); err != nil {
+				return probe.NewError(err)
+			}
+			for bucket := range bucketMetadata.Buckets {
+				bucketSlice := fmt.Sprintf("%s$0$%d", bucket, order) // TODO handle node slices
+				err := disk.MakeDir(filepath.Join(xl.config.XLName, bucketSlice))
+				if err != nil {
+					return err.Trace()
+				}
+			}
+		}
+	}
+	return nil
+}

pkg/xl/interfaces.go

@@ -0,0 +1,72 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package xl
+
+import (
+	"io"
+
+	"github.com/minio/minio/pkg/probe"
+	signV4 "github.com/minio/minio/pkg/signature"
+)
+
+// Collection of XL specification interfaces
+
+// Interface is a collection of cloud storage and management interface
+type Interface interface {
+	CloudStorage
+	Management
+}
+
+// CloudStorage is a xl cloud storage interface
+type CloudStorage interface {
+	// Storage service operations
+	GetBucketMetadata(bucket string) (BucketMetadata, *probe.Error)
+	SetBucketMetadata(bucket string, metadata map[string]string) *probe.Error
+	ListBuckets() ([]BucketMetadata, *probe.Error)
+	MakeBucket(bucket string, ACL string, location io.Reader, signature *signV4.Signature) *probe.Error
+
+	// Bucket operations
+	ListObjects(string, BucketResourcesMetadata) ([]ObjectMetadata, BucketResourcesMetadata, *probe.Error)
+
+	// Object operations
+	GetObject(w io.Writer, bucket, object string, start, length int64) (int64, *probe.Error)
+	GetObjectMetadata(bucket, object string) (ObjectMetadata, *probe.Error)
+	// bucket, object, expectedMD5Sum, size, reader, metadata, signature
+	CreateObject(string, string, string, int64, io.Reader, map[string]string, *signV4.Signature) (ObjectMetadata, *probe.Error)
+
+	Multipart
+}
+
+// Multipart API
+type Multipart interface {
+	NewMultipartUpload(bucket, key, contentType string) (string, *probe.Error)
+	AbortMultipartUpload(bucket, key, uploadID string) *probe.Error
+	CreateObjectPart(string, string, string, int, string, string, int64, io.Reader, *signV4.Signature) (string, *probe.Error)
+	CompleteMultipartUpload(bucket, key, uploadID string, data io.Reader, signature *signV4.Signature) (ObjectMetadata, *probe.Error)
+	ListMultipartUploads(string, BucketMultipartResourcesMetadata) (BucketMultipartResourcesMetadata, *probe.Error)
+	ListObjectParts(string, string, ObjectResourcesMetadata) (ObjectResourcesMetadata, *probe.Error)
+}
+
+// Management is a xl management system interface
+type Management interface {
+	Heal() *probe.Error
+	Rebalance() *probe.Error
+	Info() (map[string][]string, *probe.Error)
+
+	AttachNode(hostname string, disks []string) *probe.Error
+	DetachNode(hostname string) *probe.Error
+}

pkg/xl/management.go

@@ -0,0 +1,81 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package xl
+
+import (
+	"github.com/minio/minio/pkg/probe"
+	"github.com/minio/minio/pkg/xl/block"
+)
+
+// Info - return info about xl configuration
+func (xl API) Info() (nodeDiskMap map[string][]string, err *probe.Error) {
+	nodeDiskMap = make(map[string][]string)
+	for nodeName, n := range xl.nodes {
+		disks, err := n.ListDisks()
+		if err != nil {
+			return nil, err.Trace()
+		}
+		diskList := make([]string, len(disks))
+		for diskOrder, disk := range disks {
+			diskList[diskOrder] = disk.GetPath()
+		}
+		nodeDiskMap[nodeName] = diskList
+	}
+	return nodeDiskMap, nil
+}
+
+// AttachNode - attach node
+func (xl API) AttachNode(hostname string, disks []string) *probe.Error {
+	if hostname == "" || len(disks) == 0 {
+		return probe.NewError(InvalidArgument{})
+	}
+	n, err := newNode(hostname)
+	if err != nil {
+		return err.Trace()
+	}
+	xl.nodes[hostname] = n
+	for i, d := range disks {
+		newDisk, err := block.New(d)
+		if err != nil {
+			continue
+		}
+		if err := newDisk.MakeDir(xl.config.XLName); err != nil {
+			return err.Trace()
+		}
+		if err := n.AttachDisk(newDisk, i); err != nil {
+			return err.Trace()
+		}
+	}
+	return nil
+}
+
+// DetachNode - detach node
+func (xl API) DetachNode(hostname string) *probe.Error {
+	delete(xl.nodes, hostname)
+	return nil
+}
+
+// Rebalance - rebalance an existing xl with new disks and nodes
+func (xl API) Rebalance() *probe.Error {
+	return probe.NewError(APINotImplemented{API: "management.Rebalance"})
+}
+
+// Heal - heal your xls
+func (xl API) Heal() *probe.Error {
+	// TODO handle data heal
+	return xl.healBuckets()
+}

pkg/xl/multipart.go

@@ -0,0 +1,514 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package xl
+
+import (
+	"bytes"
+	"crypto/md5"
+	"crypto/sha512"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/xml"
+
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"runtime/debug"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/minio/minio/pkg/crypto/sha256"
+	"github.com/minio/minio/pkg/probe"
+	signV4 "github.com/minio/minio/pkg/signature"
+	"github.com/minio/minio/pkg/xl/cache/data"
+)
+
+/// V2 API functions
+
+// NewMultipartUpload - initiate a new multipart session
+func (xl API) NewMultipartUpload(bucket, key, contentType string) (string, *probe.Error) {
+	xl.lock.Lock()
+	defer xl.lock.Unlock()
+
+	if !IsValidBucket(bucket) {
+		return "", probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+	if !IsValidObjectName(key) {
+		return "", probe.NewError(ObjectNameInvalid{Object: key})
+	}
+	//	if len(xl.config.NodeDiskMap) > 0 {
+	//		return xl.newMultipartUpload(bucket, key, contentType)
+	//	}
+	if !xl.storedBuckets.Exists(bucket) {
+		return "", probe.NewError(BucketNotFound{Bucket: bucket})
+	}
+	storedBucket := xl.storedBuckets.Get(bucket).(storedBucket)
+	objectKey := bucket + "/" + key
+	if _, ok := storedBucket.objectMetadata[objectKey]; ok == true {
+		return "", probe.NewError(ObjectExists{Object: key})
+	}
+	id := []byte(strconv.Itoa(rand.Int()) + bucket + key + time.Now().UTC().String())
+	uploadIDSum := sha512.Sum512(id)
+	uploadID := base64.URLEncoding.EncodeToString(uploadIDSum[:])[:47]
+
+	storedBucket.multiPartSession[key] = MultiPartSession{
+		UploadID:   uploadID,
+		Initiated:  time.Now().UTC(),
+		TotalParts: 0,
+	}
+	storedBucket.partMetadata[key] = make(map[int]PartMetadata)
+	multiPartCache := data.NewCache(0)
+	multiPartCache.OnEvicted = xl.evictedPart
+	xl.multiPartObjects[uploadID] = multiPartCache
+	xl.storedBuckets.Set(bucket, storedBucket)
+	return uploadID, nil
+}
+
+// AbortMultipartUpload - abort an incomplete multipart session
+func (xl API) AbortMultipartUpload(bucket, key, uploadID string) *probe.Error {
+	xl.lock.Lock()
+	defer xl.lock.Unlock()
+
+	if !IsValidBucket(bucket) {
+		return probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+	if !IsValidObjectName(key) {
+		return probe.NewError(ObjectNameInvalid{Object: key})
+	}
+	// TODO: multipart support for xl is broken, since we haven't finalized the format in which
+	//       it can be stored, disabling this for now until we get the underlying layout stable.
+	//
+	//	if len(xl.config.NodeDiskMap) > 0 {
+	//		return xl.abortMultipartUpload(bucket, key, uploadID)
+	//	}
+	if !xl.storedBuckets.Exists(bucket) {
+		return probe.NewError(BucketNotFound{Bucket: bucket})
+	}
+	storedBucket := xl.storedBuckets.Get(bucket).(storedBucket)
+	if storedBucket.multiPartSession[key].UploadID != uploadID {
+		return probe.NewError(InvalidUploadID{UploadID: uploadID})
+	}
+	xl.cleanupMultipartSession(bucket, key, uploadID)
+	return nil
+}
+
+// CreateObjectPart - create a part in a multipart session
+func (xl API) CreateObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader, signature *signV4.Signature) (string, *probe.Error) {
+	xl.lock.Lock()
+	etag, err := xl.createObjectPart(bucket, key, uploadID, partID, "", expectedMD5Sum, size, data, signature)
+	xl.lock.Unlock()
+	// possible free
+	debug.FreeOSMemory()
+
+	return etag, err.Trace()
+}
+
+// createObject - internal wrapper function called by CreateObjectPart
+func (xl API) createObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader, signature *signV4.Signature) (string, *probe.Error) {
+	if !IsValidBucket(bucket) {
+		return "", probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+	if !IsValidObjectName(key) {
+		return "", probe.NewError(ObjectNameInvalid{Object: key})
+	}
+	// TODO: multipart support for xl is broken, since we haven't finalized the format in which
+	//       it can be stored, disabling this for now until we get the underlying layout stable.
+	//
+	/*
+		if len(xl.config.NodeDiskMap) > 0 {
+			metadata := make(map[string]string)
+			if contentType == "" {
+				contentType = "application/octet-stream"
+			}
+			contentType = strings.TrimSpace(contentType)
+			metadata["contentType"] = contentType
+			if strings.TrimSpace(expectedMD5Sum) != "" {
+				expectedMD5SumBytes, err := base64.StdEncoding.DecodeString(strings.TrimSpace(expectedMD5Sum))
+				if err != nil {
+					// pro-actively close the connection
+					return "", probe.NewError(InvalidDigest{Md5: expectedMD5Sum})
+				}
+				expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes)
+			}
+			partMetadata, err := xl.putObjectPart(bucket, key, expectedMD5Sum, uploadID, partID, data, size, metadata, signature)
+			if err != nil {
+				return "", err.Trace()
+			}
+			return partMetadata.ETag, nil
+		}
+	*/
+	if !xl.storedBuckets.Exists(bucket) {
+		return "", probe.NewError(BucketNotFound{Bucket: bucket})
+	}
+	strBucket := xl.storedBuckets.Get(bucket).(storedBucket)
+	// Verify upload id
+	if strBucket.multiPartSession[key].UploadID != uploadID {
+		return "", probe.NewError(InvalidUploadID{UploadID: uploadID})
+	}
+
+	// get object key
+	parts := strBucket.partMetadata[key]
+	if _, ok := parts[partID]; ok {
+		return parts[partID].ETag, nil
+	}
+
+	if contentType == "" {
+		contentType = "application/octet-stream"
+	}
+	contentType = strings.TrimSpace(contentType)
+	if strings.TrimSpace(expectedMD5Sum) != "" {
+		expectedMD5SumBytes, err := base64.StdEncoding.DecodeString(strings.TrimSpace(expectedMD5Sum))
+		if err != nil {
+			// pro-actively close the connection
+			return "", probe.NewError(InvalidDigest{Md5: expectedMD5Sum})
+		}
+		expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes)
+	}
+
+	// calculate md5
+	hash := md5.New()
+	sha256hash := sha256.New()
+
+	var totalLength int64
+	var err error
+	for err == nil {
+		var length int
+		byteBuffer := make([]byte, 1024*1024)
+		length, err = data.Read(byteBuffer) // do not read error return error here, we will handle this error later
+		if length != 0 {
+			hash.Write(byteBuffer[0:length])
+			sha256hash.Write(byteBuffer[0:length])
+			ok := xl.multiPartObjects[uploadID].Append(partID, byteBuffer[0:length])
+			if !ok {
+				return "", probe.NewError(InternalError{})
+			}
+			totalLength += int64(length)
+			go debug.FreeOSMemory()
+		}
+	}
+	if totalLength != size {
+		xl.multiPartObjects[uploadID].Delete(partID)
+		return "", probe.NewError(IncompleteBody{Bucket: bucket, Object: key})
+	}
+	if err != io.EOF {
+		return "", probe.NewError(err)
+	}
+
+	md5SumBytes := hash.Sum(nil)
+	md5Sum := hex.EncodeToString(md5SumBytes)
+	// Verify if the written object is equal to what is expected, only if it is requested as such
+	if strings.TrimSpace(expectedMD5Sum) != "" {
+		if err := isMD5SumEqual(strings.TrimSpace(expectedMD5Sum), md5Sum); err != nil {
+			return "", err.Trace()
+		}
+	}
+
+	if signature != nil {
+		{
+			ok, err := signature.DoesSignatureMatch(hex.EncodeToString(sha256hash.Sum(nil)))
+			if err != nil {
+				return "", err.Trace()
+			}
+			if !ok {
+				return "", probe.NewError(signV4.SigDoesNotMatch{})
+			}
+		}
+	}
+
+	newPart := PartMetadata{
+		PartNumber:   partID,
+		LastModified: time.Now().UTC(),
+		ETag:         md5Sum,
+		Size:         totalLength,
+	}
+
+	parts[partID] = newPart
+	strBucket.partMetadata[key] = parts
+	multiPartSession := strBucket.multiPartSession[key]
+	multiPartSession.TotalParts++
+	strBucket.multiPartSession[key] = multiPartSession
+	xl.storedBuckets.Set(bucket, strBucket)
+	return md5Sum, nil
+}
+
+// cleanupMultipartSession invoked during an abort or complete multipart session to cleanup session from memory
+func (xl API) cleanupMultipartSession(bucket, key, uploadID string) {
+	storedBucket := xl.storedBuckets.Get(bucket).(storedBucket)
+	for i := 1; i <= storedBucket.multiPartSession[key].TotalParts; i++ {
+		xl.multiPartObjects[uploadID].Delete(i)
+	}
+	delete(storedBucket.multiPartSession, key)
+	delete(storedBucket.partMetadata, key)
+	xl.storedBuckets.Set(bucket, storedBucket)
+}
+
+func (xl API) mergeMultipart(parts *CompleteMultipartUpload, uploadID string, fullObjectWriter *io.PipeWriter) {
+	for _, part := range parts.Part {
+		recvMD5 := part.ETag
+		object, ok := xl.multiPartObjects[uploadID].Get(part.PartNumber)
+		if ok == false {
+			fullObjectWriter.CloseWithError(probe.WrapError(probe.NewError(InvalidPart{})))
+			return
+		}
+		calcMD5Bytes := md5.Sum(object)
+		// complete multi part request header md5sum per part is hex encoded
+		recvMD5Bytes, err := hex.DecodeString(strings.Trim(recvMD5, "\""))
+		if err != nil {
+			fullObjectWriter.CloseWithError(probe.WrapError(probe.NewError(InvalidDigest{Md5: recvMD5})))
+			return
+		}
+		if !bytes.Equal(recvMD5Bytes, calcMD5Bytes[:]) {
+			fullObjectWriter.CloseWithError(probe.WrapError(probe.NewError(BadDigest{})))
+			return
+		}
+
+		if _, err := io.Copy(fullObjectWriter, bytes.NewReader(object)); err != nil {
+			fullObjectWriter.CloseWithError(probe.WrapError(probe.NewError(err)))
+			return
+		}
+		object = nil
+	}
+	fullObjectWriter.Close()
+	return
+}
+
+// CompleteMultipartUpload - complete a multipart upload and persist the data
+func (xl API) CompleteMultipartUpload(bucket, key, uploadID string, data io.Reader, signature *signV4.Signature) (ObjectMetadata, *probe.Error) {
+	xl.lock.Lock()
+	defer xl.lock.Unlock()
+	size := int64(xl.multiPartObjects[uploadID].Stats().Bytes)
+	fullObjectReader, err := xl.completeMultipartUploadV2(bucket, key, uploadID, data, signature)
+	if err != nil {
+		return ObjectMetadata{}, err.Trace()
+	}
+	objectMetadata, err := xl.createObject(bucket, key, "", "", size, fullObjectReader, nil)
+	if err != nil {
+		// No need to call internal cleanup functions here, caller should call AbortMultipartUpload()
+		// which would in-turn cleanup properly in accordance with S3 Spec
+		return ObjectMetadata{}, err.Trace()
+	}
+	xl.cleanupMultipartSession(bucket, key, uploadID)
+	return objectMetadata, nil
+}
+
+func (xl API) completeMultipartUploadV2(bucket, key, uploadID string, data io.Reader, signature *signV4.Signature) (io.Reader, *probe.Error) {
+	if !IsValidBucket(bucket) {
+		return nil, probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+	if !IsValidObjectName(key) {
+		return nil, probe.NewError(ObjectNameInvalid{Object: key})
+	}
+
+	// TODO: multipart support for xl is broken, since we haven't finalized the format in which
+	//       it can be stored, disabling this for now until we get the underlying layout stable.
+	//
+	//	if len(xl.config.NodeDiskMap) > 0 {
+	//		xl.lock.Unlock()
+	//		return xl.completeMultipartUpload(bucket, key, uploadID, data, signature)
+	//	}
+
+	if !xl.storedBuckets.Exists(bucket) {
+		return nil, probe.NewError(BucketNotFound{Bucket: bucket})
+	}
+	storedBucket := xl.storedBuckets.Get(bucket).(storedBucket)
+	// Verify upload id
+	if storedBucket.multiPartSession[key].UploadID != uploadID {
+		return nil, probe.NewError(InvalidUploadID{UploadID: uploadID})
+	}
+	partBytes, err := ioutil.ReadAll(data)
+	if err != nil {
+		return nil, probe.NewError(err)
+	}
+	if signature != nil {
+		partHashBytes := sha256.Sum256(partBytes)
+		ok, err := signature.DoesSignatureMatch(hex.EncodeToString(partHashBytes[:]))
+		if err != nil {
+			return nil, err.Trace()
+		}
+		if !ok {
+			return nil, probe.NewError(signV4.SigDoesNotMatch{})
+		}
+	}
+	parts := &CompleteMultipartUpload{}
+	if err := xml.Unmarshal(partBytes, parts); err != nil {
+		return nil, probe.NewError(MalformedXML{})
+	}
+	if !sort.IsSorted(completedParts(parts.Part)) {
+		return nil, probe.NewError(InvalidPartOrder{})
+	}
+
+	fullObjectReader, fullObjectWriter := io.Pipe()
+	go xl.mergeMultipart(parts, uploadID, fullObjectWriter)
+
+	return fullObjectReader, nil
+}
+
+// byKey is a sortable interface for UploadMetadata slice
+type byKey []*UploadMetadata
+
+func (a byKey) Len() int           { return len(a) }
+func (a byKey) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a byKey) Less(i, j int) bool { return a[i].Key < a[j].Key }
+
+// ListMultipartUploads - list incomplete multipart sessions for a given bucket
+func (xl API) ListMultipartUploads(bucket string, resources BucketMultipartResourcesMetadata) (BucketMultipartResourcesMetadata, *probe.Error) {
+	// TODO handle delimiter, low priority
+	xl.lock.Lock()
+	defer xl.lock.Unlock()
+
+	if !IsValidBucket(bucket) {
+		return BucketMultipartResourcesMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+
+	// TODO: multipart support for xl is broken, since we haven't finalized the format in which
+	//       it can be stored, disabling this for now until we get the underlying layout stable.
+	//
+	//	if len(xl.config.NodeDiskMap) > 0 {
+	//		return xl.listMultipartUploads(bucket, resources)
+	//	}
+
+	if !xl.storedBuckets.Exists(bucket) {
+		return BucketMultipartResourcesMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
+	}
+
+	storedBucket := xl.storedBuckets.Get(bucket).(storedBucket)
+	var uploads []*UploadMetadata
+
+	for key, session := range storedBucket.multiPartSession {
+		if strings.HasPrefix(key, resources.Prefix) {
+			if len(uploads) > resources.MaxUploads {
+				sort.Sort(byKey(uploads))
+				resources.Upload = uploads
+				resources.NextKeyMarker = key
+				resources.NextUploadIDMarker = session.UploadID
+				resources.IsTruncated = true
+				return resources, nil
+			}
+			// uploadIDMarker is ignored if KeyMarker is empty
+			switch {
+			case resources.KeyMarker != "" && resources.UploadIDMarker == "":
+				if key > resources.KeyMarker {
+					upload := new(UploadMetadata)
+					upload.Key = key
+					upload.UploadID = session.UploadID
+					upload.Initiated = session.Initiated
+					uploads = append(uploads, upload)
+				}
+			case resources.KeyMarker != "" && resources.UploadIDMarker != "":
+				if session.UploadID > resources.UploadIDMarker {
+					if key >= resources.KeyMarker {
+						upload := new(UploadMetadata)
+						upload.Key = key
+						upload.UploadID = session.UploadID
+						upload.Initiated = session.Initiated
+						uploads = append(uploads, upload)
+					}
+				}
+			default:
+				upload := new(UploadMetadata)
+				upload.Key = key
+				upload.UploadID = session.UploadID
+				upload.Initiated = session.Initiated
+				uploads = append(uploads, upload)
+			}
+		}
+	}
+	sort.Sort(byKey(uploads))
+	resources.Upload = uploads
+	return resources, nil
+}
+
+// partNumber is a sortable interface for Part slice
+type partNumber []*PartMetadata
+
+func (a partNumber) Len() int           { return len(a) }
+func (a partNumber) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a partNumber) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
+
+// ListObjectParts - list parts from incomplete multipart session for a given object
+func (xl API) ListObjectParts(bucket, key string, resources ObjectResourcesMetadata) (ObjectResourcesMetadata, *probe.Error) {
+	// Verify upload id
+	xl.lock.Lock()
+	defer xl.lock.Unlock()
+
+	if !IsValidBucket(bucket) {
+		return ObjectResourcesMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+	if !IsValidObjectName(key) {
+		return ObjectResourcesMetadata{}, probe.NewError(ObjectNameInvalid{Object: key})
+	}
+
+	// TODO: multipart support for xl is broken, since we haven't finalized the format in which
+	//       it can be stored, disabling this for now until we get the underlying layout stable.
+	//
+	//	if len(xl.config.NodeDiskMap) > 0 {
+	//		return xl.listObjectParts(bucket, key, resources)
+	//	}
+
+	if !xl.storedBuckets.Exists(bucket) {
+		return ObjectResourcesMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
+	}
+	storedBucket := xl.storedBuckets.Get(bucket).(storedBucket)
+	if _, ok := storedBucket.multiPartSession[key]; ok == false {
+		return ObjectResourcesMetadata{}, probe.NewError(ObjectNotFound{Object: key})
+	}
+	if storedBucket.multiPartSession[key].UploadID != resources.UploadID {
+		return ObjectResourcesMetadata{}, probe.NewError(InvalidUploadID{UploadID: resources.UploadID})
+	}
+	storedParts := storedBucket.partMetadata[key]
+	objectResourcesMetadata := resources
+	objectResourcesMetadata.Bucket = bucket
+	objectResourcesMetadata.Key = key
+	var parts []*PartMetadata
+	var startPartNumber int
+	switch {
+	case objectResourcesMetadata.PartNumberMarker == 0:
+		startPartNumber = 1
+	default:
+		startPartNumber = objectResourcesMetadata.PartNumberMarker
+	}
+	for i := startPartNumber; i <= storedBucket.multiPartSession[key].TotalParts; i++ {
+		if len(parts) > objectResourcesMetadata.MaxParts {
+			sort.Sort(partNumber(parts))
+			objectResourcesMetadata.IsTruncated = true
+			objectResourcesMetadata.Part = parts
+			objectResourcesMetadata.NextPartNumberMarker = i
+			return objectResourcesMetadata, nil
+		}
+		part, ok := storedParts[i]
+		if !ok {
+			return ObjectResourcesMetadata{}, probe.NewError(InvalidPart{})
+		}
+		parts = append(parts, &part)
+	}
+	sort.Sort(partNumber(parts))
+	objectResourcesMetadata.Part = parts
+	return objectResourcesMetadata, nil
+}
+
+// evictedPart - call back function called by caching module during individual cache evictions
+func (xl API) evictedPart(a ...interface{}) {
+	// loop through all buckets
+	buckets := xl.storedBuckets.GetAll()
+	for bucketName, bucket := range buckets {
+		b := bucket.(storedBucket)
+		xl.storedBuckets.Set(bucketName, b)
+	}
+	debug.FreeOSMemory()
+}

pkg/xl/node.go

@@ -0,0 +1,76 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package xl
+
+import (
+	"github.com/minio/minio/pkg/probe"
+	"github.com/minio/minio/pkg/xl/block"
+)
+
+// node struct internal
+type node struct {
+	hostname string
+	disks    map[int]block.Block
+}
+
+// newNode - instantiates a new node
+func newNode(hostname string) (node, *probe.Error) {
+	if hostname == "" {
+		return node{}, probe.NewError(InvalidArgument{})
+	}
+	disks := make(map[int]block.Block)
+	n := node{
+		hostname: hostname,
+		disks:    disks,
+	}
+	return n, nil
+}
+
+// GetHostname - return hostname
+func (n node) GetHostname() string {
+	return n.hostname
+}
+
+// ListDisks - return number of disks
+func (n node) ListDisks() (map[int]block.Block, *probe.Error) {
+	return n.disks, nil
+}
+
+// AttachDisk - attach a disk
+func (n node) AttachDisk(disk block.Block, diskOrder int) *probe.Error {
+	if diskOrder < 0 {
+		return probe.NewError(InvalidArgument{})
+	}
+	n.disks[diskOrder] = disk
+	return nil
+}
+
+// DetachDisk - detach a disk
+func (n node) DetachDisk(diskOrder int) *probe.Error {
+	delete(n.disks, diskOrder)
+	return nil
+}
+
+// SaveConfig - save node configuration
+func (n node) SaveConfig() *probe.Error {
+	return probe.NewError(NotImplemented{Function: "SaveConfig"})
+}
+
+// LoadConfig - load node configuration from saved configs
+func (n node) LoadConfig() *probe.Error {
+	return probe.NewError(NotImplemented{Function: "LoadConfig"})
+}

pkg/xl/xl-metadata.md

@@ -0,0 +1,55 @@
+##### Users Collection
+
+```js
+
+"minio": {
+    "version": 1,
+    "users": [{
+    	"secretAccessKey": String,
+    	"accessKeyId": String,
+    	"status": String // enum: ok, disabled, deleted
+    }],
+    "hosts": [{
+    	"address": String,
+    	"uuid": String,
+    	"status": String, // enum: ok, disabled, deleted, busy, offline.
+    	"disks": [{
+        		"disk": String,
+        		"uuid": String,
+        		"status": String // ok, offline, disabled, busy.
+        }]
+    }]
+}
+```
+
+##### Bucket Collection
+
+```js
+"buckets": {
+    "bucket": String, // index
+    "deleted": Boolean,
+    "permissions": String
+}
+```
+
+##### Object Collection
+
+```js
+"objects": {
+    "key": String, // index
+    "createdAt": Date,
+    "hosts[16]": [{
+    	"host": String,
+    	"disk": String,
+    }],
+    "deleted": Boolean
+}
+```
+
+```js
+"meta": {
+    "key": String, // index
+    "type": String // content-type
+    // type speific meta
+}
+```

pkg/xl/xl-v1.go

@@ -0,0 +1,681 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package xl
+
+import (
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/minio/minio/pkg/crypto/sha256"
+	"github.com/minio/minio/pkg/crypto/sha512"
+	"github.com/minio/minio/pkg/probe"
+	signV4 "github.com/minio/minio/pkg/signature"
+	"github.com/minio/minio/pkg/xl/block"
+)
+
+// config files used inside XL
+const (
+	// bucket, object metadata
+	bucketMetadataConfig = "bucketMetadata.json"
+	objectMetadataConfig = "objectMetadata.json"
+
+	// versions
+	objectMetadataVersion = "1.0.0"
+	bucketMetadataVersion = "1.0.0"
+)
+
+/// v1 API functions
+
+// makeBucket - make a new bucket
+func (xl API) makeBucket(bucket string, acl BucketACL) *probe.Error {
+	if bucket == "" || strings.TrimSpace(bucket) == "" {
+		return probe.NewError(InvalidArgument{})
+	}
+	return xl.makeXLBucket(bucket, acl.String())
+}
+
+// getBucketMetadata - get bucket metadata
+func (xl API) getBucketMetadata(bucketName string) (BucketMetadata, *probe.Error) {
+	if err := xl.listXLBuckets(); err != nil {
+		return BucketMetadata{}, err.Trace()
+	}
+	if _, ok := xl.buckets[bucketName]; !ok {
+		return BucketMetadata{}, probe.NewError(BucketNotFound{Bucket: bucketName})
+	}
+	metadata, err := xl.getXLBucketMetadata()
+	if err != nil {
+		return BucketMetadata{}, err.Trace()
+	}
+	return metadata.Buckets[bucketName], nil
+}
+
+// setBucketMetadata - set bucket metadata
+func (xl API) setBucketMetadata(bucketName string, bucketMetadata map[string]string) *probe.Error {
+	if err := xl.listXLBuckets(); err != nil {
+		return err.Trace()
+	}
+	metadata, err := xl.getXLBucketMetadata()
+	if err != nil {
+		return err.Trace()
+	}
+	oldBucketMetadata := metadata.Buckets[bucketName]
+	acl, ok := bucketMetadata["acl"]
+	if !ok {
+		return probe.NewError(InvalidArgument{})
+	}
+	oldBucketMetadata.ACL = BucketACL(acl)
+	metadata.Buckets[bucketName] = oldBucketMetadata
+	return xl.setXLBucketMetadata(metadata)
+}
+
+// listBuckets - return list of buckets
+func (xl API) listBuckets() (map[string]BucketMetadata, *probe.Error) {
+	if err := xl.listXLBuckets(); err != nil {
+		return nil, err.Trace()
+	}
+	metadata, err := xl.getXLBucketMetadata()
+	if err != nil {
+		// intentionally left out the error when XL is empty
+		// but we need to revisit this area in future - since we need
+		// to figure out between acceptable and unacceptable errors
+		return make(map[string]BucketMetadata), nil
+	}
+	if metadata == nil {
+		return make(map[string]BucketMetadata), nil
+	}
+	return metadata.Buckets, nil
+}
+
+// listObjects - return list of objects
+func (xl API) listObjects(bucket, prefix, marker, delimiter string, maxkeys int) (ListObjectsResults, *probe.Error) {
+	if err := xl.listXLBuckets(); err != nil {
+		return ListObjectsResults{}, err.Trace()
+	}
+	if _, ok := xl.buckets[bucket]; !ok {
+		return ListObjectsResults{}, probe.NewError(BucketNotFound{Bucket: bucket})
+	}
+	listObjects, err := xl.buckets[bucket].ListObjects(prefix, marker, delimiter, maxkeys)
+	if err != nil {
+		return ListObjectsResults{}, err.Trace()
+	}
+	return listObjects, nil
+}
+
+// putObject - put object
+func (xl API) putObject(bucket, object, expectedMD5Sum string, reader io.Reader, size int64, metadata map[string]string, signature *signV4.Signature) (ObjectMetadata, *probe.Error) {
+	if bucket == "" || strings.TrimSpace(bucket) == "" {
+		return ObjectMetadata{}, probe.NewError(InvalidArgument{})
+	}
+	if object == "" || strings.TrimSpace(object) == "" {
+		return ObjectMetadata{}, probe.NewError(InvalidArgument{})
+	}
+	if err := xl.listXLBuckets(); err != nil {
+		return ObjectMetadata{}, err.Trace()
+	}
+	if _, ok := xl.buckets[bucket]; !ok {
+		return ObjectMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
+	}
+	bucketMeta, err := xl.getXLBucketMetadata()
+	if err != nil {
+		return ObjectMetadata{}, err.Trace()
+	}
+	if _, ok := bucketMeta.Buckets[bucket].BucketObjects[object]; ok {
+		return ObjectMetadata{}, probe.NewError(ObjectExists{Object: object})
+	}
+	objMetadata, err := xl.buckets[bucket].WriteObject(object, reader, size, expectedMD5Sum, metadata, signature)
+	if err != nil {
+		return ObjectMetadata{}, err.Trace()
+	}
+	bucketMeta.Buckets[bucket].BucketObjects[object] = struct{}{}
+	if err := xl.setXLBucketMetadata(bucketMeta); err != nil {
+		return ObjectMetadata{}, err.Trace()
+	}
+	return objMetadata, nil
+}
+
+// putObject - put object
+func (xl API) putObjectPart(bucket, object, expectedMD5Sum, uploadID string, partID int, reader io.Reader, size int64, metadata map[string]string, signature *signV4.Signature) (PartMetadata, *probe.Error) {
+	if bucket == "" || strings.TrimSpace(bucket) == "" {
+		return PartMetadata{}, probe.NewError(InvalidArgument{})
+	}
+	if object == "" || strings.TrimSpace(object) == "" {
+		return PartMetadata{}, probe.NewError(InvalidArgument{})
+	}
+	if err := xl.listXLBuckets(); err != nil {
+		return PartMetadata{}, err.Trace()
+	}
+	if _, ok := xl.buckets[bucket]; !ok {
+		return PartMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
+	}
+	bucketMeta, err := xl.getXLBucketMetadata()
+	if err != nil {
+		return PartMetadata{}, err.Trace()
+	}
+	if _, ok := bucketMeta.Buckets[bucket].Multiparts[object]; !ok {
+		return PartMetadata{}, probe.NewError(InvalidUploadID{UploadID: uploadID})
+	}
+	if _, ok := bucketMeta.Buckets[bucket].BucketObjects[object]; ok {
+		return PartMetadata{}, probe.NewError(ObjectExists{Object: object})
+	}
+	objectPart := object + "/" + "multipart" + "/" + strconv.Itoa(partID)
+	objmetadata, err := xl.buckets[bucket].WriteObject(objectPart, reader, size, expectedMD5Sum, metadata, signature)
+	if err != nil {
+		return PartMetadata{}, err.Trace()
+	}
+	partMetadata := PartMetadata{
+		PartNumber:   partID,
+		LastModified: objmetadata.Created,
+		ETag:         objmetadata.MD5Sum,
+		Size:         objmetadata.Size,
+	}
+	multipartSession := bucketMeta.Buckets[bucket].Multiparts[object]
+	multipartSession.Parts[strconv.Itoa(partID)] = partMetadata
+	bucketMeta.Buckets[bucket].Multiparts[object] = multipartSession
+	if err := xl.setXLBucketMetadata(bucketMeta); err != nil {
+		return PartMetadata{}, err.Trace()
+	}
+	return partMetadata, nil
+}
+
+// getObject - get object
+func (xl API) getObject(bucket, object string) (reader io.ReadCloser, size int64, err *probe.Error) {
+	if bucket == "" || strings.TrimSpace(bucket) == "" {
+		return nil, 0, probe.NewError(InvalidArgument{})
+	}
+	if object == "" || strings.TrimSpace(object) == "" {
+		return nil, 0, probe.NewError(InvalidArgument{})
+	}
+	if err := xl.listXLBuckets(); err != nil {
+		return nil, 0, err.Trace()
+	}
+	if _, ok := xl.buckets[bucket]; !ok {
+		return nil, 0, probe.NewError(BucketNotFound{Bucket: bucket})
+	}
+	return xl.buckets[bucket].ReadObject(object)
+}
+
+// getObjectMetadata - get object metadata
+func (xl API) getObjectMetadata(bucket, object string) (ObjectMetadata, *probe.Error) {
+	if err := xl.listXLBuckets(); err != nil {
+		return ObjectMetadata{}, err.Trace()
+	}
+	if _, ok := xl.buckets[bucket]; !ok {
+		return ObjectMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
+	}
+	bucketMeta, err := xl.getXLBucketMetadata()
+	if err != nil {
+		return ObjectMetadata{}, err.Trace()
+	}
+	if _, ok := bucketMeta.Buckets[bucket].BucketObjects[object]; !ok {
+		return ObjectMetadata{}, probe.NewError(ObjectNotFound{Object: object})
+	}
+	objectMetadata, err := xl.buckets[bucket].GetObjectMetadata(object)
+	if err != nil {
+		return ObjectMetadata{}, err.Trace()
+	}
+	return objectMetadata, nil
+}
+
+// newMultipartUpload - new multipart upload request
+func (xl API) newMultipartUpload(bucket, object, contentType string) (string, *probe.Error) {
+	if err := xl.listXLBuckets(); err != nil {
+		return "", err.Trace()
+	}
+	if _, ok := xl.buckets[bucket]; !ok {
+		return "", probe.NewError(BucketNotFound{Bucket: bucket})
+	}
+	allbuckets, err := xl.getXLBucketMetadata()
+	if err != nil {
+		return "", err.Trace()
+	}
+	bucketMetadata := allbuckets.Buckets[bucket]
+	multiparts := make(map[string]MultiPartSession)
+	if len(bucketMetadata.Multiparts) > 0 {
+		multiparts = bucketMetadata.Multiparts
+	}
+
+	id := []byte(strconv.Itoa(rand.Int()) + bucket + object + time.Now().String())
+	uploadIDSum := sha512.Sum512(id)
+	uploadID := base64.URLEncoding.EncodeToString(uploadIDSum[:])[:47]
+
+	multipartSession := MultiPartSession{
+		UploadID:   uploadID,
+		Initiated:  time.Now().UTC(),
+		Parts:      make(map[string]PartMetadata),
+		TotalParts: 0,
+	}
+	multiparts[object] = multipartSession
+	bucketMetadata.Multiparts = multiparts
+	allbuckets.Buckets[bucket] = bucketMetadata
+
+	if err := xl.setXLBucketMetadata(allbuckets); err != nil {
+		return "", err.Trace()
+	}
+
+	return uploadID, nil
+}
+
+// listObjectParts list all object parts
+func (xl API) listObjectParts(bucket, object string, resources ObjectResourcesMetadata) (ObjectResourcesMetadata, *probe.Error) {
+	if bucket == "" || strings.TrimSpace(bucket) == "" {
+		return ObjectResourcesMetadata{}, probe.NewError(InvalidArgument{})
+	}
+	if object == "" || strings.TrimSpace(object) == "" {
+		return ObjectResourcesMetadata{}, probe.NewError(InvalidArgument{})
+	}
+	if err := xl.listXLBuckets(); err != nil {
+		return ObjectResourcesMetadata{}, err.Trace()
+	}
+	if _, ok := xl.buckets[bucket]; !ok {
+		return ObjectResourcesMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
+	}
+	allBuckets, err := xl.getXLBucketMetadata()
+	if err != nil {
+		return ObjectResourcesMetadata{}, err.Trace()
+	}
+	bucketMetadata := allBuckets.Buckets[bucket]
+	if _, ok := bucketMetadata.Multiparts[object]; !ok {
+		return ObjectResourcesMetadata{}, probe.NewError(InvalidUploadID{UploadID: resources.UploadID})
+	}
+	if bucketMetadata.Multiparts[object].UploadID != resources.UploadID {
+		return ObjectResourcesMetadata{}, probe.NewError(InvalidUploadID{UploadID: resources.UploadID})
+	}
+	objectResourcesMetadata := resources
+	objectResourcesMetadata.Bucket = bucket
+	objectResourcesMetadata.Key = object
+	var parts []*PartMetadata
+	var startPartNumber int
+	switch {
+	case objectResourcesMetadata.PartNumberMarker == 0:
+		startPartNumber = 1
+	default:
+		startPartNumber = objectResourcesMetadata.PartNumberMarker
+	}
+	for i := startPartNumber; i <= bucketMetadata.Multiparts[object].TotalParts; i++ {
+		if len(parts) > objectResourcesMetadata.MaxParts {
+			sort.Sort(partNumber(parts))
+			objectResourcesMetadata.IsTruncated = true
+			objectResourcesMetadata.Part = parts
+			objectResourcesMetadata.NextPartNumberMarker = i
+			return objectResourcesMetadata, nil
+		}
+		part, ok := bucketMetadata.Multiparts[object].Parts[strconv.Itoa(i)]
+		if !ok {
+			return ObjectResourcesMetadata{}, probe.NewError(InvalidPart{})
+		}
+		parts = append(parts, &part)
+	}
+	sort.Sort(partNumber(parts))
+	objectResourcesMetadata.Part = parts
+	return objectResourcesMetadata, nil
+}
+
+// completeMultipartUpload complete an incomplete multipart upload
+func (xl API) completeMultipartUpload(bucket, object, uploadID string, data io.Reader, signature *signV4.Signature) (ObjectMetadata, *probe.Error) {
+	if bucket == "" || strings.TrimSpace(bucket) == "" {
+		return ObjectMetadata{}, probe.NewError(InvalidArgument{})
+	}
+	if object == "" || strings.TrimSpace(object) == "" {
+		return ObjectMetadata{}, probe.NewError(InvalidArgument{})
+	}
+	if err := xl.listXLBuckets(); err != nil {
+		return ObjectMetadata{}, err.Trace()
+	}
+	if _, ok := xl.buckets[bucket]; !ok {
+		return ObjectMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
+	}
+	allBuckets, err := xl.getXLBucketMetadata()
+	if err != nil {
+		return ObjectMetadata{}, err.Trace()
+	}
+	bucketMetadata := allBuckets.Buckets[bucket]
+	if _, ok := bucketMetadata.Multiparts[object]; !ok {
+		return ObjectMetadata{}, probe.NewError(InvalidUploadID{UploadID: uploadID})
+	}
+	if bucketMetadata.Multiparts[object].UploadID != uploadID {
+		return ObjectMetadata{}, probe.NewError(InvalidUploadID{UploadID: uploadID})
+	}
+	var partBytes []byte
+	{
+		var err error
+		partBytes, err = ioutil.ReadAll(data)
+		if err != nil {
+			return ObjectMetadata{}, probe.NewError(err)
+		}
+	}
+	if signature != nil {
+		partHashBytes := sha256.Sum256(partBytes)
+		ok, err := signature.DoesSignatureMatch(hex.EncodeToString(partHashBytes[:]))
+		if err != nil {
+			return ObjectMetadata{}, err.Trace()
+		}
+		if !ok {
+			return ObjectMetadata{}, probe.NewError(signV4.SigDoesNotMatch{})
+		}
+	}
+	parts := &CompleteMultipartUpload{}
+	if err := xml.Unmarshal(partBytes, parts); err != nil {
+		return ObjectMetadata{}, probe.NewError(MalformedXML{})
+	}
+	if !sort.IsSorted(completedParts(parts.Part)) {
+		return ObjectMetadata{}, probe.NewError(InvalidPartOrder{})
+	}
+	for _, part := range parts.Part {
+		if strings.Trim(part.ETag, "\"") != bucketMetadata.Multiparts[object].Parts[strconv.Itoa(part.PartNumber)].ETag {
+			return ObjectMetadata{}, probe.NewError(InvalidPart{})
+		}
+	}
+	var finalETagBytes []byte
+	var finalSize int64
+	totalParts := strconv.Itoa(bucketMetadata.Multiparts[object].TotalParts)
+	for _, part := range bucketMetadata.Multiparts[object].Parts {
+		partETagBytes, err := hex.DecodeString(part.ETag)
+		if err != nil {
+			return ObjectMetadata{}, probe.NewError(err)
+		}
+		finalETagBytes = append(finalETagBytes, partETagBytes...)
+		finalSize += part.Size
+	}
+	finalETag := hex.EncodeToString(finalETagBytes)
+	objMetadata := ObjectMetadata{}
+	objMetadata.MD5Sum = finalETag + "-" + totalParts
+	objMetadata.Object = object
+	objMetadata.Bucket = bucket
+	objMetadata.Size = finalSize
+	objMetadata.Created = bucketMetadata.Multiparts[object].Parts[totalParts].LastModified
+	return objMetadata, nil
+}
+
+// listMultipartUploads list all multipart uploads
+func (xl API) listMultipartUploads(bucket string, resources BucketMultipartResourcesMetadata) (BucketMultipartResourcesMetadata, *probe.Error) {
+	if err := xl.listXLBuckets(); err != nil {
+		return BucketMultipartResourcesMetadata{}, err.Trace()
+	}
+	if _, ok := xl.buckets[bucket]; !ok {
+		return BucketMultipartResourcesMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
+	}
+	allbuckets, err := xl.getXLBucketMetadata()
+	if err != nil {
+		return BucketMultipartResourcesMetadata{}, err.Trace()
+	}
+	bucketMetadata := allbuckets.Buckets[bucket]
+	var uploads []*UploadMetadata
+	for key, session := range bucketMetadata.Multiparts {
+		if strings.HasPrefix(key, resources.Prefix) {
+			if len(uploads) > resources.MaxUploads {
+				sort.Sort(byKey(uploads))
+				resources.Upload = uploads
+				resources.NextKeyMarker = key
+				resources.NextUploadIDMarker = session.UploadID
+				resources.IsTruncated = true
+				return resources, nil
+			}
+			// uploadIDMarker is ignored if KeyMarker is empty
+			switch {
+			case resources.KeyMarker != "" && resources.UploadIDMarker == "":
+				if key > resources.KeyMarker {
+					upload := new(UploadMetadata)
+					upload.Key = key
+					upload.UploadID = session.UploadID
+					upload.Initiated = session.Initiated
+					uploads = append(uploads, upload)
+				}
+			case resources.KeyMarker != "" && resources.UploadIDMarker != "":
+				if session.UploadID > resources.UploadIDMarker {
+					if key >= resources.KeyMarker {
+						upload := new(UploadMetadata)
+						upload.Key = key
+						upload.UploadID = session.UploadID
+						upload.Initiated = session.Initiated
+						uploads = append(uploads, upload)
+					}
+				}
+			default:
+				upload := new(UploadMetadata)
+				upload.Key = key
+				upload.UploadID = session.UploadID
+				upload.Initiated = session.Initiated
+				uploads = append(uploads, upload)
+			}
+		}
+	}
+	sort.Sort(byKey(uploads))
+	resources.Upload = uploads
+	return resources, nil
+}
+
+// abortMultipartUpload - abort a incomplete multipart upload
+func (xl API) abortMultipartUpload(bucket, object, uploadID string) *probe.Error {
+	if err := xl.listXLBuckets(); err != nil {
+		return err.Trace()
+	}
+	if _, ok := xl.buckets[bucket]; !ok {
+		return probe.NewError(BucketNotFound{Bucket: bucket})
+	}
+	allbuckets, err := xl.getXLBucketMetadata()
+	if err != nil {
+		return err.Trace()
+	}
+	bucketMetadata := allbuckets.Buckets[bucket]
+	if _, ok := bucketMetadata.Multiparts[object]; !ok {
+		return probe.NewError(InvalidUploadID{UploadID: uploadID})
+	}
+	if bucketMetadata.Multiparts[object].UploadID != uploadID {
+		return probe.NewError(InvalidUploadID{UploadID: uploadID})
+	}
+	delete(bucketMetadata.Multiparts, object)
+
+	allbuckets.Buckets[bucket] = bucketMetadata
+	if err := xl.setXLBucketMetadata(allbuckets); err != nil {
+		return err.Trace()
+	}
+
+	return nil
+}
+
+//// internal functions
+
+// getBucketMetadataWriters -
+func (xl API) getBucketMetadataWriters() ([]io.WriteCloser, *probe.Error) {
+	var writers []io.WriteCloser
+	for _, node := range xl.nodes {
+		disks, err := node.ListDisks()
+		if err != nil {
+			return nil, err.Trace()
+		}
+		writers = make([]io.WriteCloser, len(disks))
+		for order, disk := range disks {
+			bucketMetaDataWriter, err := disk.CreateFile(filepath.Join(xl.config.XLName, bucketMetadataConfig))
+			if err != nil {
+				return nil, err.Trace()
+			}
+			writers[order] = bucketMetaDataWriter
+		}
+	}
+	return writers, nil
+}
+
+// getBucketMetadataReaders - readers are returned in map rather than slice
+func (xl API) getBucketMetadataReaders() (map[int]io.ReadCloser, *probe.Error) {
+	readers := make(map[int]io.ReadCloser)
+	disks := make(map[int]block.Block)
+	var err *probe.Error
+	for _, node := range xl.nodes {
+		nDisks := make(map[int]block.Block)
+		nDisks, err = node.ListDisks()
+		if err != nil {
+			return nil, err.Trace()
+		}
+		for k, v := range nDisks {
+			disks[k] = v
+		}
+	}
+	var bucketMetaDataReader io.ReadCloser
+	for order, disk := range disks {
+		bucketMetaDataReader, err = disk.Open(filepath.Join(xl.config.XLName, bucketMetadataConfig))
+		if err != nil {
+			continue
+		}
+		readers[order] = bucketMetaDataReader
+	}
+	if err != nil {
+		return nil, err.Trace()
+	}
+	return readers, nil
+}
+
+// setXLBucketMetadata -
+func (xl API) setXLBucketMetadata(metadata *AllBuckets) *probe.Error {
+	writers, err := xl.getBucketMetadataWriters()
+	if err != nil {
+		return err.Trace()
+	}
+	for _, writer := range writers {
+		jenc := json.NewEncoder(writer)
+		if err := jenc.Encode(metadata); err != nil {
+			CleanupWritersOnError(writers)
+			return probe.NewError(err)
+		}
+	}
+	for _, writer := range writers {
+		writer.Close()
+	}
+	return nil
+}
+
+// getXLBucketMetadata -
+func (xl API) getXLBucketMetadata() (*AllBuckets, *probe.Error) {
+	metadata := &AllBuckets{}
+	readers, err := xl.getBucketMetadataReaders()
+	if err != nil {
+		return nil, err.Trace()
+	}
+	for _, reader := range readers {
+		defer reader.Close()
+	}
+	{
+		var err error
+		for _, reader := range readers {
+			jenc := json.NewDecoder(reader)
+			if err = jenc.Decode(metadata); err == nil {
+				return metadata, nil
+			}
+		}
+		return nil, probe.NewError(err)
+	}
+}
+
+// makeXLBucket -
+func (xl API) makeXLBucket(bucketName, acl string) *probe.Error {
+	if err := xl.listXLBuckets(); err != nil {
+		return err.Trace()
+	}
+	if _, ok := xl.buckets[bucketName]; ok {
+		return probe.NewError(BucketExists{Bucket: bucketName})
+	}
+	bkt, bucketMetadata, err := newBucket(bucketName, acl, xl.config.XLName, xl.nodes)
+	if err != nil {
+		return err.Trace()
+	}
+	nodeNumber := 0
+	xl.buckets[bucketName] = bkt
+	for _, node := range xl.nodes {
+		disks := make(map[int]block.Block)
+		disks, err = node.ListDisks()
+		if err != nil {
+			return err.Trace()
+		}
+		for order, disk := range disks {
+			bucketSlice := fmt.Sprintf("%s$%d$%d", bucketName, nodeNumber, order)
+			err := disk.MakeDir(filepath.Join(xl.config.XLName, bucketSlice))
+			if err != nil {
+				return err.Trace()
+			}
+		}
+		nodeNumber = nodeNumber + 1
+	}
+	var metadata *AllBuckets
+	metadata, err = xl.getXLBucketMetadata()
+	if err != nil {
+		if os.IsNotExist(err.ToGoError()) {
+			metadata = new(AllBuckets)
+			metadata.Buckets = make(map[string]BucketMetadata)
+			metadata.Buckets[bucketName] = bucketMetadata
+			err = xl.setXLBucketMetadata(metadata)
+			if err != nil {
+				return err.Trace()
+			}
+			return nil
+		}
+		return err.Trace()
+	}
+	metadata.Buckets[bucketName] = bucketMetadata
+	err = xl.setXLBucketMetadata(metadata)
+	if err != nil {
+		return err.Trace()
+	}
+	return nil
+}
+
+// listXLBuckets -
+func (xl API) listXLBuckets() *probe.Error {
+	var disks map[int]block.Block
+	var err *probe.Error
+	for _, node := range xl.nodes {
+		disks, err = node.ListDisks()
+		if err != nil {
+			return err.Trace()
+		}
+	}
+	var dirs []os.FileInfo
+	for _, disk := range disks {
+		dirs, err = disk.ListDir(xl.config.XLName)
+		if err == nil {
+			break
+		}
+	}
+	// if all disks are missing then return error
+	if err != nil {
+		return err.Trace()
+	}
+	for _, dir := range dirs {
+		splitDir := strings.Split(dir.Name(), "$")
+		if len(splitDir) < 3 {
+			return probe.NewError(CorruptedBackend{Backend: dir.Name()})
+		}
+		bucketName := splitDir[0]
+		// we dont need this once we cache from makeXLBucket()
+		bkt, _, err := newBucket(bucketName, "private", xl.config.XLName, xl.nodes)
+		if err != nil {
+			return err.Trace()
+		}
+		xl.buckets[bucketName] = bkt
+	}
+	return nil
+}

pkg/xl/xl-v1_test.go

@@ -0,0 +1,290 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or impliedd.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package xl
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/base64"
+	"encoding/hex"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+func TestXL(t *testing.T) { TestingT(t) }
+
+type MyXLSuite struct {
+	root string
+}
+
+var _ = Suite(&MyXLSuite{})
+
+// create a dummy TestNodeDiskMap
+func createTestNodeDiskMap(p string) map[string][]string {
+	nodes := make(map[string][]string)
+	nodes["localhost"] = make([]string, 16)
+	for i := 0; i < len(nodes["localhost"]); i++ {
+		diskPath := filepath.Join(p, strconv.Itoa(i))
+		if _, err := os.Stat(diskPath); err != nil {
+			if os.IsNotExist(err) {
+				os.MkdirAll(diskPath, 0700)
+			}
+		}
+		nodes["localhost"][i] = diskPath
+	}
+	return nodes
+}
+
+var dd Interface
+
+func (s *MyXLSuite) SetUpSuite(c *C) {
+	root, err := ioutil.TempDir(os.TempDir(), "xl-")
+	c.Assert(err, IsNil)
+	s.root = root
+
+	conf := new(Config)
+	conf.Version = "0.0.1"
+	conf.XLName = "test"
+	conf.NodeDiskMap = createTestNodeDiskMap(root)
+	conf.MaxSize = 100000
+	SetXLConfigPath(filepath.Join(root, "xl.json"))
+	perr := SaveConfig(conf)
+	c.Assert(perr, IsNil)
+
+	dd, perr = New()
+	c.Assert(perr, IsNil)
+
+	// testing empty xl
+	buckets, perr := dd.ListBuckets()
+	c.Assert(perr, IsNil)
+	c.Assert(len(buckets), Equals, 0)
+}
+
+func (s *MyXLSuite) TearDownSuite(c *C) {
+	os.RemoveAll(s.root)
+}
+
+// test make bucket without name
+func (s *MyXLSuite) TestBucketWithoutNameFails(c *C) {
+	// fail to create new bucket without a name
+	err := dd.MakeBucket("", "private", nil, nil)
+	c.Assert(err, Not(IsNil))
+
+	err = dd.MakeBucket(" ", "private", nil, nil)
+	c.Assert(err, Not(IsNil))
+}
+
+// test empty bucket
+func (s *MyXLSuite) TestEmptyBucket(c *C) {
+	c.Assert(dd.MakeBucket("foo1", "private", nil, nil), IsNil)
+	// check if bucket is empty
+	var resources BucketResourcesMetadata
+	resources.Maxkeys = 1
+	objectsMetadata, resources, err := dd.ListObjects("foo1", resources)
+	c.Assert(err, IsNil)
+	c.Assert(len(objectsMetadata), Equals, 0)
+	c.Assert(resources.CommonPrefixes, DeepEquals, []string{})
+	c.Assert(resources.IsTruncated, Equals, false)
+}
+
+// test bucket list
+func (s *MyXLSuite) TestMakeBucketAndList(c *C) {
+	// create bucket
+	err := dd.MakeBucket("foo2", "private", nil, nil)
+	c.Assert(err, IsNil)
+
+	// check bucket exists
+	buckets, err := dd.ListBuckets()
+	c.Assert(err, IsNil)
+	c.Assert(len(buckets), Equals, 5)
+	c.Assert(buckets[0].ACL, Equals, BucketACL("private"))
+}
+
+// test re-create bucket
+func (s *MyXLSuite) TestMakeBucketWithSameNameFails(c *C) {
+	err := dd.MakeBucket("foo3", "private", nil, nil)
+	c.Assert(err, IsNil)
+
+	err = dd.MakeBucket("foo3", "private", nil, nil)
+	c.Assert(err, Not(IsNil))
+}
+
+// test make multiple buckets
+func (s *MyXLSuite) TestCreateMultipleBucketsAndList(c *C) {
+	// add a second bucket
+	err := dd.MakeBucket("foo4", "private", nil, nil)
+	c.Assert(err, IsNil)
+
+	err = dd.MakeBucket("bar1", "private", nil, nil)
+	c.Assert(err, IsNil)
+
+	buckets, err := dd.ListBuckets()
+	c.Assert(err, IsNil)
+
+	c.Assert(len(buckets), Equals, 2)
+	c.Assert(buckets[0].Name, Equals, "bar1")
+	c.Assert(buckets[1].Name, Equals, "foo4")
+
+	err = dd.MakeBucket("foobar1", "private", nil, nil)
+	c.Assert(err, IsNil)
+
+	buckets, err = dd.ListBuckets()
+	c.Assert(err, IsNil)
+
+	c.Assert(len(buckets), Equals, 3)
+	c.Assert(buckets[2].Name, Equals, "foobar1")
+}
+
+// test object create without bucket
+func (s *MyXLSuite) TestNewObjectFailsWithoutBucket(c *C) {
+	_, err := dd.CreateObject("unknown", "obj", "", 0, nil, nil, nil)
+	c.Assert(err, Not(IsNil))
+}
+
+// test create object metadata
+func (s *MyXLSuite) TestNewObjectMetadata(c *C) {
+	data := "Hello World"
+	hasher := md5.New()
+	hasher.Write([]byte(data))
+	expectedMd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
+	reader := ioutil.NopCloser(bytes.NewReader([]byte(data)))
+
+	err := dd.MakeBucket("foo6", "private", nil, nil)
+	c.Assert(err, IsNil)
+
+	objectMetadata, err := dd.CreateObject("foo6", "obj", expectedMd5Sum, int64(len(data)), reader, map[string]string{"contentType": "application/json"}, nil)
+	c.Assert(err, IsNil)
+	c.Assert(objectMetadata.MD5Sum, Equals, hex.EncodeToString(hasher.Sum(nil)))
+	c.Assert(objectMetadata.Metadata["contentType"], Equals, "application/json")
+}
+
+// test create object fails without name
+func (s *MyXLSuite) TestNewObjectFailsWithEmptyName(c *C) {
+	_, err := dd.CreateObject("foo", "", "", 0, nil, nil, nil)
+	c.Assert(err, Not(IsNil))
+}
+
+// test create object
+func (s *MyXLSuite) TestNewObjectCanBeWritten(c *C) {
+	err := dd.MakeBucket("foo", "private", nil, nil)
+	c.Assert(err, IsNil)
+
+	data := "Hello World"
+
+	hasher := md5.New()
+	hasher.Write([]byte(data))
+	expectedMd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
+	reader := ioutil.NopCloser(bytes.NewReader([]byte(data)))
+
+	actualMetadata, err := dd.CreateObject("foo", "obj", expectedMd5Sum, int64(len(data)), reader, map[string]string{"contentType": "application/octet-stream"}, nil)
+	c.Assert(err, IsNil)
+	c.Assert(actualMetadata.MD5Sum, Equals, hex.EncodeToString(hasher.Sum(nil)))
+
+	var buffer bytes.Buffer
+	size, err := dd.GetObject(&buffer, "foo", "obj", 0, 0)
+	c.Assert(err, IsNil)
+	c.Assert(size, Equals, int64(len(data)))
+	c.Assert(buffer.Bytes(), DeepEquals, []byte(data))
+
+	actualMetadata, err = dd.GetObjectMetadata("foo", "obj")
+	c.Assert(err, IsNil)
+	c.Assert(hex.EncodeToString(hasher.Sum(nil)), Equals, actualMetadata.MD5Sum)
+	c.Assert(int64(len(data)), Equals, actualMetadata.Size)
+}
+
+// test list objects
+func (s *MyXLSuite) TestMultipleNewObjects(c *C) {
+	c.Assert(dd.MakeBucket("foo5", "private", nil, nil), IsNil)
+
+	one := ioutil.NopCloser(bytes.NewReader([]byte("one")))
+
+	_, err := dd.CreateObject("foo5", "obj1", "", int64(len("one")), one, nil, nil)
+	c.Assert(err, IsNil)
+
+	two := ioutil.NopCloser(bytes.NewReader([]byte("two")))
+	_, err = dd.CreateObject("foo5", "obj2", "", int64(len("two")), two, nil, nil)
+	c.Assert(err, IsNil)
+
+	var buffer1 bytes.Buffer
+	size, err := dd.GetObject(&buffer1, "foo5", "obj1", 0, 0)
+	c.Assert(err, IsNil)
+	c.Assert(size, Equals, int64(len([]byte("one"))))
+	c.Assert(buffer1.Bytes(), DeepEquals, []byte("one"))
+
+	var buffer2 bytes.Buffer
+	size, err = dd.GetObject(&buffer2, "foo5", "obj2", 0, 0)
+	c.Assert(err, IsNil)
+	c.Assert(size, Equals, int64(len([]byte("two"))))
+
+	c.Assert(buffer2.Bytes(), DeepEquals, []byte("two"))
+
+	/// test list of objects
+
+	// test list objects with prefix and delimiter
+	var resources BucketResourcesMetadata
+	resources.Prefix = "o"
+	resources.Delimiter = "1"
+	resources.Maxkeys = 10
+	objectsMetadata, resources, err := dd.ListObjects("foo5", resources)
+	c.Assert(err, IsNil)
+	c.Assert(resources.IsTruncated, Equals, false)
+	c.Assert(resources.CommonPrefixes[0], Equals, "obj1")
+
+	// test list objects with only delimiter
+	resources.Prefix = ""
+	resources.Delimiter = "1"
+	resources.Maxkeys = 10
+	objectsMetadata, resources, err = dd.ListObjects("foo5", resources)
+	c.Assert(err, IsNil)
+	c.Assert(objectsMetadata[0].Object, Equals, "obj2")
+	c.Assert(resources.IsTruncated, Equals, false)
+	c.Assert(resources.CommonPrefixes[0], Equals, "obj1")
+
+	// test list objects with only prefix
+	resources.Prefix = "o"
+	resources.Delimiter = ""
+	resources.Maxkeys = 10
+	objectsMetadata, resources, err = dd.ListObjects("foo5", resources)
+	c.Assert(err, IsNil)
+	c.Assert(resources.IsTruncated, Equals, false)
+	c.Assert(objectsMetadata[0].Object, Equals, "obj1")
+	c.Assert(objectsMetadata[1].Object, Equals, "obj2")
+
+	three := ioutil.NopCloser(bytes.NewReader([]byte("three")))
+	_, err = dd.CreateObject("foo5", "obj3", "", int64(len("three")), three, nil, nil)
+	c.Assert(err, IsNil)
+
+	var buffer bytes.Buffer
+	size, err = dd.GetObject(&buffer, "foo5", "obj3", 0, 0)
+	c.Assert(err, IsNil)
+	c.Assert(size, Equals, int64(len([]byte("three"))))
+	c.Assert(buffer.Bytes(), DeepEquals, []byte("three"))
+
+	// test list objects with maxkeys
+	resources.Prefix = "o"
+	resources.Delimiter = ""
+	resources.Maxkeys = 2
+	objectsMetadata, resources, err = dd.ListObjects("foo5", resources)
+	c.Assert(err, IsNil)
+	c.Assert(resources.IsTruncated, Equals, true)
+	c.Assert(len(objectsMetadata), Equals, 2)
+}

pkg/xl/xl-v2.go

@@ -0,0 +1,637 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package xl
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/base64"
+	"encoding/hex"
+	"io"
+	"io/ioutil"
+	"log"
+	"runtime/debug"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/minio/minio/pkg/crypto/sha256"
+	"github.com/minio/minio/pkg/probe"
+	"github.com/minio/minio/pkg/quick"
+	signV4 "github.com/minio/minio/pkg/signature"
+	"github.com/minio/minio/pkg/xl/cache/data"
+	"github.com/minio/minio/pkg/xl/cache/metadata"
+)
+
+// total Number of buckets allowed
+const (
+	totalBuckets = 100
+)
+
+// Config xl config
+type Config struct {
+	Version     string              `json:"version"`
+	MaxSize     uint64              `json:"max-size"`
+	XLName      string              `json:"xl-name"`
+	NodeDiskMap map[string][]string `json:"node-disk-map"`
+}
+
+// API - local variables
+type API struct {
+	config           *Config
+	lock             *sync.Mutex
+	objects          *data.Cache
+	multiPartObjects map[string]*data.Cache
+	storedBuckets    *metadata.Cache
+	nodes            map[string]node
+	buckets          map[string]bucket
+}
+
+// storedBucket saved bucket
+type storedBucket struct {
+	bucketMetadata   BucketMetadata
+	objectMetadata   map[string]ObjectMetadata
+	partMetadata     map[string]map[int]PartMetadata
+	multiPartSession map[string]MultiPartSession
+}
+
+// New instantiate a new xl
+func New() (Interface, *probe.Error) {
+	var conf *Config
+	var err *probe.Error
+	conf, err = LoadConfig()
+	if err != nil {
+		conf = &Config{
+			Version:     "0.0.1",
+			MaxSize:     512000000,
+			NodeDiskMap: nil,
+			XLName:      "",
+		}
+		if err := quick.CheckData(conf); err != nil {
+			return nil, err.Trace()
+		}
+	}
+	a := API{config: conf}
+	a.storedBuckets = metadata.NewCache()
+	a.nodes = make(map[string]node)
+	a.buckets = make(map[string]bucket)
+	a.objects = data.NewCache(a.config.MaxSize)
+	a.multiPartObjects = make(map[string]*data.Cache)
+	a.objects.OnEvicted = a.evictedObject
+	a.lock = new(sync.Mutex)
+
+	if len(a.config.NodeDiskMap) > 0 {
+		for k, v := range a.config.NodeDiskMap {
+			if len(v) == 0 {
+				return nil, probe.NewError(InvalidDisksArgument{})
+			}
+			err := a.AttachNode(k, v)
+			if err != nil {
+				return nil, err.Trace()
+			}
+		}
+		/// Initialization, populate all buckets into memory
+		buckets, err := a.listBuckets()
+		if err != nil {
+			return nil, err.Trace()
+		}
+		for k, v := range buckets {
+			var newBucket = storedBucket{}
+			newBucket.bucketMetadata = v
+			newBucket.objectMetadata = make(map[string]ObjectMetadata)
+			newBucket.multiPartSession = make(map[string]MultiPartSession)
+			newBucket.partMetadata = make(map[string]map[int]PartMetadata)
+			a.storedBuckets.Set(k, newBucket)
+		}
+		a.Heal()
+	}
+	return a, nil
+}
+
+/// V2 API functions
+
+// GetObject - GET object from cache buffer
+func (xl API) GetObject(w io.Writer, bucket string, object string, start, length int64) (int64, *probe.Error) {
+	xl.lock.Lock()
+	defer xl.lock.Unlock()
+
+	if !IsValidBucket(bucket) {
+		return 0, probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+	if !IsValidObjectName(object) {
+		return 0, probe.NewError(ObjectNameInvalid{Object: object})
+	}
+	if start < 0 {
+		return 0, probe.NewError(InvalidRange{
+			Start:  start,
+			Length: length,
+		})
+	}
+	if !xl.storedBuckets.Exists(bucket) {
+		return 0, probe.NewError(BucketNotFound{Bucket: bucket})
+	}
+	objectKey := bucket + "/" + object
+	data, ok := xl.objects.Get(objectKey)
+	var written int64
+	if !ok {
+		if len(xl.config.NodeDiskMap) > 0 {
+			reader, size, err := xl.getObject(bucket, object)
+			if err != nil {
+				return 0, err.Trace()
+			}
+			if start > 0 {
+				if _, err := io.CopyN(ioutil.Discard, reader, start); err != nil {
+					return 0, probe.NewError(err)
+				}
+			}
+			// new proxy writer to capture data read from disk
+			pw := NewProxyWriter(w)
+			{
+				var err error
+				if length > 0 {
+					written, err = io.CopyN(pw, reader, length)
+					if err != nil {
+						return 0, probe.NewError(err)
+					}
+				} else {
+					written, err = io.CopyN(pw, reader, size)
+					if err != nil {
+						return 0, probe.NewError(err)
+					}
+				}
+			}
+			/// cache object read from disk
+			ok := xl.objects.Append(objectKey, pw.writtenBytes)
+			pw.writtenBytes = nil
+			go debug.FreeOSMemory()
+			if !ok {
+				return 0, probe.NewError(InternalError{})
+			}
+			return written, nil
+		}
+		return 0, probe.NewError(ObjectNotFound{Object: object})
+	}
+	var err error
+	if start == 0 && length == 0 {
+		written, err = io.CopyN(w, bytes.NewBuffer(data), int64(xl.objects.Len(objectKey)))
+		if err != nil {
+			return 0, probe.NewError(err)
+		}
+		return written, nil
+	}
+	written, err = io.CopyN(w, bytes.NewBuffer(data[start:]), length)
+	if err != nil {
+		return 0, probe.NewError(err)
+	}
+	return written, nil
+}
+
+// GetBucketMetadata -
+func (xl API) GetBucketMetadata(bucket string) (BucketMetadata, *probe.Error) {
+	xl.lock.Lock()
+	defer xl.lock.Unlock()
+
+	if !IsValidBucket(bucket) {
+		return BucketMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+	if !xl.storedBuckets.Exists(bucket) {
+		if len(xl.config.NodeDiskMap) > 0 {
+			bucketMetadata, err := xl.getBucketMetadata(bucket)
+			if err != nil {
+				return BucketMetadata{}, err.Trace()
+			}
+			storedBucket := xl.storedBuckets.Get(bucket).(storedBucket)
+			storedBucket.bucketMetadata = bucketMetadata
+			xl.storedBuckets.Set(bucket, storedBucket)
+		}
+		return BucketMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
+	}
+	return xl.storedBuckets.Get(bucket).(storedBucket).bucketMetadata, nil
+}
+
+// SetBucketMetadata -
+func (xl API) SetBucketMetadata(bucket string, metadata map[string]string) *probe.Error {
+	xl.lock.Lock()
+	defer xl.lock.Unlock()
+
+	if !IsValidBucket(bucket) {
+		return probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+	if !xl.storedBuckets.Exists(bucket) {
+		return probe.NewError(BucketNotFound{Bucket: bucket})
+	}
+	if len(xl.config.NodeDiskMap) > 0 {
+		if err := xl.setBucketMetadata(bucket, metadata); err != nil {
+			return err.Trace()
+		}
+	}
+	storedBucket := xl.storedBuckets.Get(bucket).(storedBucket)
+	storedBucket.bucketMetadata.ACL = BucketACL(metadata["acl"])
+	xl.storedBuckets.Set(bucket, storedBucket)
+	return nil
+}
+
+// isMD5SumEqual - returns error if md5sum mismatches, success its `nil`
+func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) *probe.Error {
+	if strings.TrimSpace(expectedMD5Sum) != "" && strings.TrimSpace(actualMD5Sum) != "" {
+		expectedMD5SumBytes, err := hex.DecodeString(expectedMD5Sum)
+		if err != nil {
+			return probe.NewError(err)
+		}
+		actualMD5SumBytes, err := hex.DecodeString(actualMD5Sum)
+		if err != nil {
+			return probe.NewError(err)
+		}
+		if !bytes.Equal(expectedMD5SumBytes, actualMD5SumBytes) {
+			return probe.NewError(BadDigest{})
+		}
+		return nil
+	}
+	return probe.NewError(InvalidArgument{})
+}
+
+// CreateObject - create an object
+func (xl API) CreateObject(bucket, key, expectedMD5Sum string, size int64, data io.Reader, metadata map[string]string, signature *signV4.Signature) (ObjectMetadata, *probe.Error) {
+	xl.lock.Lock()
+	defer xl.lock.Unlock()
+
+	contentType := metadata["contentType"]
+	objectMetadata, err := xl.createObject(bucket, key, contentType, expectedMD5Sum, size, data, signature)
+	// free
+	debug.FreeOSMemory()
+
+	return objectMetadata, err.Trace()
+}
+
+// createObject - PUT object to cache buffer
+func (xl API) createObject(bucket, key, contentType, expectedMD5Sum string, size int64, data io.Reader, signature *signV4.Signature) (ObjectMetadata, *probe.Error) {
+	if len(xl.config.NodeDiskMap) == 0 {
+		if size > int64(xl.config.MaxSize) {
+			generic := GenericObjectError{Bucket: bucket, Object: key}
+			return ObjectMetadata{}, probe.NewError(EntityTooLarge{
+				GenericObjectError: generic,
+				Size:               strconv.FormatInt(size, 10),
+				MaxSize:            strconv.FormatUint(xl.config.MaxSize, 10),
+			})
+		}
+	}
+	if !IsValidBucket(bucket) {
+		return ObjectMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+	if !IsValidObjectName(key) {
+		return ObjectMetadata{}, probe.NewError(ObjectNameInvalid{Object: key})
+	}
+	if !xl.storedBuckets.Exists(bucket) {
+		return ObjectMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
+	}
+	storedBucket := xl.storedBuckets.Get(bucket).(storedBucket)
+	// get object key
+	objectKey := bucket + "/" + key
+	if _, ok := storedBucket.objectMetadata[objectKey]; ok == true {
+		return ObjectMetadata{}, probe.NewError(ObjectExists{Object: key})
+	}
+
+	if contentType == "" {
+		contentType = "application/octet-stream"
+	}
+	contentType = strings.TrimSpace(contentType)
+	if strings.TrimSpace(expectedMD5Sum) != "" {
+		expectedMD5SumBytes, err := base64.StdEncoding.DecodeString(strings.TrimSpace(expectedMD5Sum))
+		if err != nil {
+			// pro-actively close the connection
+			return ObjectMetadata{}, probe.NewError(InvalidDigest{Md5: expectedMD5Sum})
+		}
+		expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes)
+	}
+
+	if len(xl.config.NodeDiskMap) > 0 {
+		objMetadata, err := xl.putObject(
+			bucket,
+			key,
+			expectedMD5Sum,
+			data,
+			size,
+			map[string]string{
+				"contentType":   contentType,
+				"contentLength": strconv.FormatInt(size, 10),
+			},
+			signature,
+		)
+		if err != nil {
+			return ObjectMetadata{}, err.Trace()
+		}
+		storedBucket.objectMetadata[objectKey] = objMetadata
+		xl.storedBuckets.Set(bucket, storedBucket)
+		return objMetadata, nil
+	}
+
+	// calculate md5
+	hash := md5.New()
+	sha256hash := sha256.New()
+
+	var err error
+	var totalLength int64
+	for err == nil {
+		var length int
+		byteBuffer := make([]byte, 1024*1024)
+		length, err = data.Read(byteBuffer)
+		if length != 0 {
+			hash.Write(byteBuffer[0:length])
+			sha256hash.Write(byteBuffer[0:length])
+			ok := xl.objects.Append(objectKey, byteBuffer[0:length])
+			if !ok {
+				return ObjectMetadata{}, probe.NewError(InternalError{})
+			}
+			totalLength += int64(length)
+			go debug.FreeOSMemory()
+		}
+	}
+	if size != 0 {
+		if totalLength != size {
+			// Delete perhaps the object is already saved, due to the nature of append()
+			xl.objects.Delete(objectKey)
+			return ObjectMetadata{}, probe.NewError(IncompleteBody{Bucket: bucket, Object: key})
+		}
+	}
+	if err != io.EOF {
+		return ObjectMetadata{}, probe.NewError(err)
+	}
+	md5SumBytes := hash.Sum(nil)
+	md5Sum := hex.EncodeToString(md5SumBytes)
+	// Verify if the written object is equal to what is expected, only if it is requested as such
+	if strings.TrimSpace(expectedMD5Sum) != "" {
+		if err := isMD5SumEqual(strings.TrimSpace(expectedMD5Sum), md5Sum); err != nil {
+			// Delete perhaps the object is already saved, due to the nature of append()
+			xl.objects.Delete(objectKey)
+			return ObjectMetadata{}, probe.NewError(BadDigest{})
+		}
+	}
+	if signature != nil {
+		ok, err := signature.DoesSignatureMatch(hex.EncodeToString(sha256hash.Sum(nil)))
+		if err != nil {
+			// Delete perhaps the object is already saved, due to the nature of append()
+			xl.objects.Delete(objectKey)
+			return ObjectMetadata{}, err.Trace()
+		}
+		if !ok {
+			// Delete perhaps the object is already saved, due to the nature of append()
+			xl.objects.Delete(objectKey)
+			return ObjectMetadata{}, probe.NewError(signV4.SigDoesNotMatch{})
+		}
+	}
+
+	m := make(map[string]string)
+	m["contentType"] = contentType
+	newObject := ObjectMetadata{
+		Bucket: bucket,
+		Object: key,
+
+		Metadata: m,
+		Created:  time.Now().UTC(),
+		MD5Sum:   md5Sum,
+		Size:     int64(totalLength),
+	}
+
+	storedBucket.objectMetadata[objectKey] = newObject
+	xl.storedBuckets.Set(bucket, storedBucket)
+	return newObject, nil
+}
+
+// MakeBucket - create bucket in cache
+func (xl API) MakeBucket(bucketName, acl string, location io.Reader, signature *signV4.Signature) *probe.Error {
+	xl.lock.Lock()
+	defer xl.lock.Unlock()
+
+	// do not have to parse location constraint, using this just for signature verification
+	locationSum := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	if location != nil {
+		locationConstraintBytes, err := ioutil.ReadAll(location)
+		if err != nil {
+			return probe.NewError(InternalError{})
+		}
+		locationConstraintHashBytes := sha256.Sum256(locationConstraintBytes)
+		locationSum = hex.EncodeToString(locationConstraintHashBytes[:])
+	}
+
+	if signature != nil {
+		ok, err := signature.DoesSignatureMatch(locationSum)
+		if err != nil {
+			return err.Trace()
+		}
+		if !ok {
+			return probe.NewError(signV4.SigDoesNotMatch{})
+		}
+	}
+
+	if xl.storedBuckets.Stats().Items == totalBuckets {
+		return probe.NewError(TooManyBuckets{Bucket: bucketName})
+	}
+	if !IsValidBucket(bucketName) {
+		return probe.NewError(BucketNameInvalid{Bucket: bucketName})
+	}
+	if !IsValidBucketACL(acl) {
+		return probe.NewError(InvalidACL{ACL: acl})
+	}
+	if xl.storedBuckets.Exists(bucketName) {
+		return probe.NewError(BucketExists{Bucket: bucketName})
+	}
+
+	if strings.TrimSpace(acl) == "" {
+		// default is private
+		acl = "private"
+	}
+	if len(xl.config.NodeDiskMap) > 0 {
+		if err := xl.makeBucket(bucketName, BucketACL(acl)); err != nil {
+			return err.Trace()
+		}
+	}
+	var newBucket = storedBucket{}
+	newBucket.objectMetadata = make(map[string]ObjectMetadata)
+	newBucket.multiPartSession = make(map[string]MultiPartSession)
+	newBucket.partMetadata = make(map[string]map[int]PartMetadata)
+	newBucket.bucketMetadata = BucketMetadata{}
+	newBucket.bucketMetadata.Name = bucketName
+	newBucket.bucketMetadata.Created = time.Now().UTC()
+	newBucket.bucketMetadata.ACL = BucketACL(acl)
+	xl.storedBuckets.Set(bucketName, newBucket)
+	return nil
+}
+
+// ListObjects - list objects from cache
+func (xl API) ListObjects(bucket string, resources BucketResourcesMetadata) ([]ObjectMetadata, BucketResourcesMetadata, *probe.Error) {
+	xl.lock.Lock()
+	defer xl.lock.Unlock()
+
+	if !IsValidBucket(bucket) {
+		return nil, BucketResourcesMetadata{IsTruncated: false}, probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+	if !IsValidPrefix(resources.Prefix) {
+		return nil, BucketResourcesMetadata{IsTruncated: false}, probe.NewError(ObjectNameInvalid{Object: resources.Prefix})
+	}
+	if !xl.storedBuckets.Exists(bucket) {
+		return nil, BucketResourcesMetadata{IsTruncated: false}, probe.NewError(BucketNotFound{Bucket: bucket})
+	}
+	var results []ObjectMetadata
+	var keys []string
+	if len(xl.config.NodeDiskMap) > 0 {
+		listObjects, err := xl.listObjects(
+			bucket,
+			resources.Prefix,
+			resources.Marker,
+			resources.Delimiter,
+			resources.Maxkeys,
+		)
+		if err != nil {
+			return nil, BucketResourcesMetadata{IsTruncated: false}, err.Trace()
+		}
+		resources.CommonPrefixes = listObjects.CommonPrefixes
+		resources.IsTruncated = listObjects.IsTruncated
+		for key := range listObjects.Objects {
+			keys = append(keys, key)
+		}
+		sort.Strings(keys)
+		for _, key := range keys {
+			results = append(results, listObjects.Objects[key])
+		}
+		if resources.IsTruncated && resources.Delimiter != "" {
+			resources.NextMarker = results[len(results)-1].Object
+		}
+		return results, resources, nil
+	}
+	storedBucket := xl.storedBuckets.Get(bucket).(storedBucket)
+	for key := range storedBucket.objectMetadata {
+		if strings.HasPrefix(key, bucket+"/") {
+			key = key[len(bucket)+1:]
+			if strings.HasPrefix(key, resources.Prefix) {
+				if key > resources.Marker {
+					keys = append(keys, key)
+				}
+			}
+		}
+	}
+	if strings.TrimSpace(resources.Prefix) != "" {
+		keys = TrimPrefix(keys, resources.Prefix)
+	}
+	var prefixes []string
+	var filteredKeys []string
+	filteredKeys = keys
+	if strings.TrimSpace(resources.Delimiter) != "" {
+		filteredKeys = HasNoDelimiter(keys, resources.Delimiter)
+		prefixes = HasDelimiter(keys, resources.Delimiter)
+		prefixes = SplitDelimiter(prefixes, resources.Delimiter)
+		prefixes = SortUnique(prefixes)
+	}
+	for _, commonPrefix := range prefixes {
+		resources.CommonPrefixes = append(resources.CommonPrefixes, resources.Prefix+commonPrefix)
+	}
+	filteredKeys = RemoveDuplicates(filteredKeys)
+	sort.Strings(filteredKeys)
+
+	for _, key := range filteredKeys {
+		if len(results) == resources.Maxkeys {
+			resources.IsTruncated = true
+			if resources.IsTruncated && resources.Delimiter != "" {
+				resources.NextMarker = results[len(results)-1].Object
+			}
+			return results, resources, nil
+		}
+		object := storedBucket.objectMetadata[bucket+"/"+resources.Prefix+key]
+		results = append(results, object)
+	}
+	resources.CommonPrefixes = RemoveDuplicates(resources.CommonPrefixes)
+	sort.Strings(resources.CommonPrefixes)
+	return results, resources, nil
+}
+
+// byBucketName is a type for sorting bucket metadata by bucket name
+type byBucketName []BucketMetadata
+
+func (b byBucketName) Len() int           { return len(b) }
+func (b byBucketName) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
+func (b byBucketName) Less(i, j int) bool { return b[i].Name < b[j].Name }
+
+// ListBuckets - List buckets from cache
+func (xl API) ListBuckets() ([]BucketMetadata, *probe.Error) {
+	xl.lock.Lock()
+	defer xl.lock.Unlock()
+
+	var results []BucketMetadata
+	if len(xl.config.NodeDiskMap) > 0 {
+		buckets, err := xl.listBuckets()
+		if err != nil {
+			return nil, err.Trace()
+		}
+		for _, bucketMetadata := range buckets {
+			results = append(results, bucketMetadata)
+		}
+		sort.Sort(byBucketName(results))
+		return results, nil
+	}
+	for _, bucket := range xl.storedBuckets.GetAll() {
+		results = append(results, bucket.(storedBucket).bucketMetadata)
+	}
+	sort.Sort(byBucketName(results))
+	return results, nil
+}
+
+// GetObjectMetadata - get object metadata from cache
+func (xl API) GetObjectMetadata(bucket, key string) (ObjectMetadata, *probe.Error) {
+	xl.lock.Lock()
+	defer xl.lock.Unlock()
+
+	// check if bucket exists
+	if !IsValidBucket(bucket) {
+		return ObjectMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+	if !IsValidObjectName(key) {
+		return ObjectMetadata{}, probe.NewError(ObjectNameInvalid{Object: key})
+	}
+	if !xl.storedBuckets.Exists(bucket) {
+		return ObjectMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
+	}
+	storedBucket := xl.storedBuckets.Get(bucket).(storedBucket)
+	objectKey := bucket + "/" + key
+	if objMetadata, ok := storedBucket.objectMetadata[objectKey]; ok == true {
+		return objMetadata, nil
+	}
+	if len(xl.config.NodeDiskMap) > 0 {
+		objMetadata, err := xl.getObjectMetadata(bucket, key)
+		if err != nil {
+			return ObjectMetadata{}, err.Trace()
+		}
+		// update
+		storedBucket.objectMetadata[objectKey] = objMetadata
+		xl.storedBuckets.Set(bucket, storedBucket)
+		return objMetadata, nil
+	}
+	return ObjectMetadata{}, probe.NewError(ObjectNotFound{Object: key})
+}
+
+// evictedObject callback function called when an item is evicted from memory
+func (xl API) evictedObject(a ...interface{}) {
+	cacheStats := xl.objects.Stats()
+	log.Printf("CurrentSize: %d, CurrentItems: %d, TotalEvicted: %d",
+		cacheStats.Bytes, cacheStats.Items, cacheStats.Evicted)
+	key := a[0].(string)
+	// loop through all buckets
+	for _, bucket := range xl.storedBuckets.GetAll() {
+		delete(bucket.(storedBucket).objectMetadata, key)
+	}
+	debug.FreeOSMemory()
+}

pkg/xl/xl-v2_test.go

@@ -0,0 +1,265 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or impliedc.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package xl
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/base64"
+	"encoding/hex"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+func TestCache(t *testing.T) { TestingT(t) }
+
+type MyCacheSuite struct {
+	root string
+}
+
+var _ = Suite(&MyCacheSuite{})
+
+var dc Interface
+
+func (s *MyCacheSuite) SetUpSuite(c *C) {
+	root, err := ioutil.TempDir(os.TempDir(), "xl-")
+	c.Assert(err, IsNil)
+	s.root = root
+
+	SetXLConfigPath(filepath.Join(root, "xl.json"))
+	dc, _ = New()
+
+	// testing empty cache
+	var buckets []BucketMetadata
+	buckets, perr := dc.ListBuckets()
+	c.Assert(perr, IsNil)
+	c.Assert(len(buckets), Equals, 0)
+}
+
+func (s *MyCacheSuite) TearDownSuite(c *C) {
+	os.RemoveAll(s.root)
+}
+
+// test make bucket without name
+func (s *MyCacheSuite) TestBucketWithoutNameFails(c *C) {
+	// fail to create new bucket without a name
+	err := dc.MakeBucket("", "private", nil, nil)
+	c.Assert(err, Not(IsNil))
+
+	err = dc.MakeBucket(" ", "private", nil, nil)
+	c.Assert(err, Not(IsNil))
+}
+
+// test empty bucket
+func (s *MyCacheSuite) TestEmptyBucket(c *C) {
+	c.Assert(dc.MakeBucket("foo1", "private", nil, nil), IsNil)
+	// check if bucket is empty
+	var resources BucketResourcesMetadata
+	resources.Maxkeys = 1
+	objectsMetadata, resources, err := dc.ListObjects("foo1", resources)
+	c.Assert(err, IsNil)
+	c.Assert(len(objectsMetadata), Equals, 0)
+	c.Assert(resources.CommonPrefixes, DeepEquals, []string{})
+	c.Assert(resources.IsTruncated, Equals, false)
+}
+
+// test bucket list
+func (s *MyCacheSuite) TestMakeBucketAndList(c *C) {
+	// create bucket
+	err := dc.MakeBucket("foo2", "private", nil, nil)
+	c.Assert(err, IsNil)
+
+	// check bucket exists
+	buckets, err := dc.ListBuckets()
+	c.Assert(err, IsNil)
+	c.Assert(len(buckets), Equals, 5)
+	c.Assert(buckets[0].ACL, Equals, BucketACL("private"))
+}
+
+// test re-create bucket
+func (s *MyCacheSuite) TestMakeBucketWithSameNameFails(c *C) {
+	err := dc.MakeBucket("foo3", "private", nil, nil)
+	c.Assert(err, IsNil)
+
+	err = dc.MakeBucket("foo3", "private", nil, nil)
+	c.Assert(err, Not(IsNil))
+}
+
+// test make multiple buckets
+func (s *MyCacheSuite) TestCreateMultipleBucketsAndList(c *C) {
+	// add a second bucket
+	err := dc.MakeBucket("foo4", "private", nil, nil)
+	c.Assert(err, IsNil)
+
+	err = dc.MakeBucket("bar1", "private", nil, nil)
+	c.Assert(err, IsNil)
+
+	buckets, err := dc.ListBuckets()
+	c.Assert(err, IsNil)
+
+	c.Assert(len(buckets), Equals, 2)
+	c.Assert(buckets[0].Name, Equals, "bar1")
+	c.Assert(buckets[1].Name, Equals, "foo4")
+
+	err = dc.MakeBucket("foobar1", "private", nil, nil)
+	c.Assert(err, IsNil)
+
+	buckets, err = dc.ListBuckets()
+	c.Assert(err, IsNil)
+
+	c.Assert(len(buckets), Equals, 3)
+	c.Assert(buckets[2].Name, Equals, "foobar1")
+}
+
+// test object create without bucket
+func (s *MyCacheSuite) TestNewObjectFailsWithoutBucket(c *C) {
+	_, err := dc.CreateObject("unknown", "obj", "", 0, nil, nil, nil)
+	c.Assert(err, Not(IsNil))
+}
+
+// test create object metadata
+func (s *MyCacheSuite) TestNewObjectMetadata(c *C) {
+	data := "Hello World"
+	hasher := md5.New()
+	hasher.Write([]byte(data))
+	expectedMd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
+	reader := ioutil.NopCloser(bytes.NewReader([]byte(data)))
+
+	err := dc.MakeBucket("foo6", "private", nil, nil)
+	c.Assert(err, IsNil)
+
+	objectMetadata, err := dc.CreateObject("foo6", "obj", expectedMd5Sum, int64(len(data)), reader, map[string]string{"contentType": "application/json"}, nil)
+	c.Assert(err, IsNil)
+	c.Assert(objectMetadata.MD5Sum, Equals, hex.EncodeToString(hasher.Sum(nil)))
+	c.Assert(objectMetadata.Metadata["contentType"], Equals, "application/json")
+}
+
+// test create object fails without name
+func (s *MyCacheSuite) TestNewObjectFailsWithEmptyName(c *C) {
+	_, err := dc.CreateObject("foo", "", "", 0, nil, nil, nil)
+	c.Assert(err, Not(IsNil))
+}
+
+// test create object
+func (s *MyCacheSuite) TestNewObjectCanBeWritten(c *C) {
+	err := dc.MakeBucket("foo", "private", nil, nil)
+	c.Assert(err, IsNil)
+
+	data := "Hello World"
+
+	hasher := md5.New()
+	hasher.Write([]byte(data))
+	expectedMd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
+	reader := ioutil.NopCloser(bytes.NewReader([]byte(data)))
+
+	actualMetadata, err := dc.CreateObject("foo", "obj", expectedMd5Sum, int64(len(data)), reader, map[string]string{"contentType": "application/octet-stream"}, nil)
+	c.Assert(err, IsNil)
+	c.Assert(actualMetadata.MD5Sum, Equals, hex.EncodeToString(hasher.Sum(nil)))
+
+	var buffer bytes.Buffer
+	size, err := dc.GetObject(&buffer, "foo", "obj", 0, 0)
+	c.Assert(err, IsNil)
+	c.Assert(size, Equals, int64(len(data)))
+	c.Assert(buffer.Bytes(), DeepEquals, []byte(data))
+
+	actualMetadata, err = dc.GetObjectMetadata("foo", "obj")
+	c.Assert(err, IsNil)
+	c.Assert(hex.EncodeToString(hasher.Sum(nil)), Equals, actualMetadata.MD5Sum)
+	c.Assert(int64(len(data)), Equals, actualMetadata.Size)
+}
+
+// test list objects
+func (s *MyCacheSuite) TestMultipleNewObjects(c *C) {
+	c.Assert(dc.MakeBucket("foo5", "private", nil, nil), IsNil)
+
+	one := ioutil.NopCloser(bytes.NewReader([]byte("one")))
+
+	_, err := dc.CreateObject("foo5", "obj1", "", int64(len("one")), one, nil, nil)
+	c.Assert(err, IsNil)
+
+	two := ioutil.NopCloser(bytes.NewReader([]byte("two")))
+	_, err = dc.CreateObject("foo5", "obj2", "", int64(len("two")), two, nil, nil)
+	c.Assert(err, IsNil)
+
+	var buffer1 bytes.Buffer
+	size, err := dc.GetObject(&buffer1, "foo5", "obj1", 0, 0)
+	c.Assert(err, IsNil)
+	c.Assert(size, Equals, int64(len([]byte("one"))))
+	c.Assert(buffer1.Bytes(), DeepEquals, []byte("one"))
+
+	var buffer2 bytes.Buffer
+	size, err = dc.GetObject(&buffer2, "foo5", "obj2", 0, 0)
+	c.Assert(err, IsNil)
+	c.Assert(size, Equals, int64(len([]byte("two"))))
+
+	c.Assert(buffer2.Bytes(), DeepEquals, []byte("two"))
+
+	/// test list of objects
+
+	// test list objects with prefix and delimiter
+	var resources BucketResourcesMetadata
+	resources.Prefix = "o"
+	resources.Delimiter = "1"
+	resources.Maxkeys = 10
+	objectsMetadata, resources, err := dc.ListObjects("foo5", resources)
+	c.Assert(err, IsNil)
+	c.Assert(resources.IsTruncated, Equals, false)
+	c.Assert(resources.CommonPrefixes[0], Equals, "obj1")
+
+	// test list objects with only delimiter
+	resources.Prefix = ""
+	resources.Delimiter = "1"
+	resources.Maxkeys = 10
+	objectsMetadata, resources, err = dc.ListObjects("foo5", resources)
+	c.Assert(err, IsNil)
+	c.Assert(objectsMetadata[0].Object, Equals, "obj2")
+	c.Assert(resources.IsTruncated, Equals, false)
+	c.Assert(resources.CommonPrefixes[0], Equals, "obj1")
+
+	// test list objects with only prefix
+	resources.Prefix = "o"
+	resources.Delimiter = ""
+	resources.Maxkeys = 10
+	objectsMetadata, resources, err = dc.ListObjects("foo5", resources)
+	c.Assert(err, IsNil)
+	c.Assert(resources.IsTruncated, Equals, false)
+	c.Assert(objectsMetadata[0].Object, Equals, "obj1")
+	c.Assert(objectsMetadata[1].Object, Equals, "obj2")
+
+	three := ioutil.NopCloser(bytes.NewReader([]byte("three")))
+	_, err = dc.CreateObject("foo5", "obj3", "", int64(len("three")), three, nil, nil)
+	c.Assert(err, IsNil)
+
+	var buffer bytes.Buffer
+	size, err = dc.GetObject(&buffer, "foo5", "obj3", 0, 0)
+	c.Assert(err, IsNil)
+	c.Assert(size, Equals, int64(len([]byte("three"))))
+	c.Assert(buffer.Bytes(), DeepEquals, []byte("three"))
+
+	// test list objects with maxkeys
+	resources.Prefix = "o"
+	resources.Delimiter = ""
+	resources.Maxkeys = 2
+	objectsMetadata, resources, err = dc.ListObjects("foo5", resources)
+	c.Assert(err, IsNil)
+	c.Assert(resources.IsTruncated, Equals, true)
+	c.Assert(len(objectsMetadata), Equals, 2)
+}