From 0475d7d0566dc2a99a3fa6ee3abb5f267e5ddc50 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sun, 5 Apr 2015 01:53:41 -0700 Subject: [PATCH 1/2] Migrate to external Donut package --- Godeps/Godeps.json | 6 +- .../src/github.com/minio-io/donut/.gitignore | 2 + .../minio-io/donut/Godeps/Godeps.json | 22 ++ .../github.com/minio-io/donut/Godeps/Readme | 5 + .../src/github.com/minio-io/donut/LICENSE | 202 ++++++++++ .../src/github.com/minio-io/donut/Makefile | 69 ++++ .../src/github.com/minio-io/donut/README.md | 3 + .../src/github.com/minio-io/donut/bucket.go | 240 ++++++++++++ .../minio-io/donut/bucket_internal.go | 184 +++++++++ .../minio-io/donut/buildscripts/checkdeps.sh | 201 ++++++++++ .../donut/buildscripts/git-commit-id.sh | 18 + .../minio-io/donut/cmd/donut-cli/.gitignore | 1 + .../cmd/donut-cli/donut-cmd-attach-disk.go | 58 +++ .../donut/cmd/donut-cli/donut-cmd-common.go | 110 ++++++ .../donut/cmd/donut-cli/donut-cmd-config.go | 105 ++++++ .../cmd/donut-cli/donut-cmd-detach-disk.go | 66 ++++ .../donut/cmd/donut-cli/donut-cmd-info.go | 84 +++++ .../donut/cmd/donut-cli/donut-cmd-list.go | 36 ++ .../donut/cmd/donut-cli/donut-cmd-make.go | 80 ++++ .../donut/cmd/donut-cli/donut-cmd-options.go | 95 +++++ .../cmd/donut-cli/donut-cmd-rebalance.go | 32 ++ .../minio-io/donut/cmd/donut-cli/donut.go | 33 ++ .../src/github.com/minio-io/donut/disk.go | 150 ++++++++ .../minio-io/donut/disk_internal.go | 63 ++++ .../src/github.com/minio-io/donut/donut.go | 200 ++++++++++ .../src/github.com/minio-io/donut/erasure.go | 85 +++++ .../github.com/minio-io/donut/interfaces.go | 106 ++++++ .../src/github.com/minio-io/donut/node.go | 68 ++++ .../src/github.com/minio-io/donut/object.go | 69 ++++ .../github.com/minio-io/donut/rebalance.go | 37 ++ pkg/drivers/donut/donut.go | 352 +++++++++++------- pkg/drivers/donut/donut_filter.go | 68 ++++ pkg/server/server.go | 4 +- pkg/storage/donut/bucket.go | 42 --- pkg/storage/donut/donut.go | 196 ---------- pkg/storage/donut/donut_test.go | 240 ------------ pkg/storage/donut/erasure.go | 245 ------------ pkg/storage/donut/interfaces.go | 51 --- pkg/storage/donut/node_local.go | 100 ----- pkg/storage/donut/object_writer.go | 91 ----- 40 files changed, 2719 insertions(+), 1100 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/.gitignore create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/Godeps/Godeps.json create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/Godeps/Readme create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/LICENSE create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/Makefile create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/README.md create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/bucket.go create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/bucket_internal.go create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/buildscripts/checkdeps.sh create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/buildscripts/git-commit-id.sh create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/.gitignore create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-attach-disk.go create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-common.go create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-config.go create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-detach-disk.go create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-info.go create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-list.go create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-make.go create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-options.go create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-rebalance.go create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut.go create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/disk.go create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/disk_internal.go create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/donut.go create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/erasure.go create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/interfaces.go create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/node.go create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/object.go create mode 100644 Godeps/_workspace/src/github.com/minio-io/donut/rebalance.go create mode 100644 pkg/drivers/donut/donut_filter.go delete mode 100644 pkg/storage/donut/bucket.go delete mode 100644 pkg/storage/donut/donut.go delete mode 100644 pkg/storage/donut/donut_test.go delete mode 100644 pkg/storage/donut/erasure.go delete mode 100644 pkg/storage/donut/interfaces.go delete mode 100644 pkg/storage/donut/node_local.go delete mode 100644 pkg/storage/donut/object_writer.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 62fa9a4dd..abcb7e765 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -1,6 +1,6 @@ { "ImportPath": "github.com/minio-io/minio", - "GoVersion": "go1.4.2", + "GoVersion": "go1.4", "Packages": [ "./..." ], @@ -22,6 +22,10 @@ "Comment": "1.2.0-100-g6d6f8d3", "Rev": "6d6f8d3cc162bfcb60379888e2f37d73ff6a6253" }, + { + "ImportPath": "github.com/minio-io/donut", + "Rev": "5647e1e6c6a95caec431610a497b15f8298d56cf" + }, { "ImportPath": "github.com/minio-io/erasure", "Rev": "2a52bdad9b271ef680374a22f0cb68513a79ebf5" diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/.gitignore b/Godeps/_workspace/src/github.com/minio-io/donut/.gitignore new file mode 100644 index 000000000..144114ecd --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/.gitignore @@ -0,0 +1,2 @@ +donut +build-constants.go \ No newline at end of file diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/Godeps/Godeps.json b/Godeps/_workspace/src/github.com/minio-io/donut/Godeps/Godeps.json new file mode 100644 index 000000000..dc3c86a04 --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/Godeps/Godeps.json @@ -0,0 +1,22 @@ +{ + "ImportPath": "github.com/minio-io/donut", + "GoVersion": "go1.4", + "Packages": [ + "./..." + ], + "Deps": [ + { + "ImportPath": "github.com/minio-io/cli", + "Comment": "1.2.0-101-g1a25bbd", + "Rev": "1a25bbdce2344b0063ea0476bceb4a4adbe4492a" + }, + { + "ImportPath": "github.com/minio-io/erasure", + "Rev": "3cece1a107115563682604b1430418e28f65dd80" + }, + { + "ImportPath": "github.com/minio-io/minio/pkg/utils/split", + "Rev": "936520e6e0fc5dd4ce8d04504ee991084555e57a" + } + ] +} diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/Godeps/Readme b/Godeps/_workspace/src/github.com/minio-io/donut/Godeps/Readme new file mode 100644 index 000000000..4cdaa53d5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/Godeps/Readme @@ -0,0 +1,5 @@ +This directory tree is generated automatically by godep. + +Please do not edit. + +See https://github.com/tools/godep for more information. diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/LICENSE b/Godeps/_workspace/src/github.com/minio-io/donut/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/Makefile b/Godeps/_workspace/src/github.com/minio-io/donut/Makefile new file mode 100644 index 000000000..b81792b5a --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/Makefile @@ -0,0 +1,69 @@ +MINIOPATH=$(GOPATH)/src/github.com/minio-io/donut + +all: getdeps install + +checkdeps: + @echo "Checking deps:" + @(env bash $(PWD)/buildscripts/checkdeps.sh) + +checkgopath: + @echo "Checking if project is at ${MINIOPATH}" + @if [ ! -d ${MINIOPATH} ]; then echo "Project not found in $GOPATH, please follow instructions provided at https://github.com/Minio-io/minio/blob/master/CONTRIBUTING.md#setup-your-minio-github-repository" && exit 1; fi + +getdeps: checkdeps checkgopath + @go get github.com/minio-io/godep && echo "Installed godep:" + @go get github.com/golang/lint/golint && echo "Installed golint:" + @go get golang.org/x/tools/cmd/vet && echo "Installed vet:" + @go get github.com/fzipp/gocyclo && echo "Installed gocyclo:" + +verifiers: getdeps vet fmt lint cyclo + +vet: + @echo "Running $@:" + @go vet ./... +fmt: + @echo "Running $@:" + @test -z "$$(gofmt -s -l . | grep -v Godeps/_workspace/src/ | tee /dev/stderr)" || \ + echo "+ please format Go code with 'gofmt -s'" +lint: + @echo "Running $@:" + @test -z "$$(golint ./... | grep -v Godeps/_workspace/src/ | tee /dev/stderr)" + +cyclo: + @echo "Running $@:" + @test -z "$$(gocyclo -over 15 . | grep -v Godeps/_workspace/src/ | tee /dev/stderr)" + +pre-build: + @echo "Running pre-build:" + @(env bash $(PWD)/buildscripts/git-commit-id.sh) + +build-all: getdeps verifiers + @echo "Building Libraries:" + @godep go generate github.com/minio-io/erasure + @godep go generate ./... + @godep go build -a ./... # have no stale packages + +test-all: pre-build build-all + @echo "Running Test Suites:" + @godep go test -race ./... + +save: restore + @godep save ./... + +restore: + @godep restore + +env: + @godep go env + +docs-deploy: + @mkdocs gh-deploy --clean + +install: test-all + @echo "Installing donut-cli:" + @godep go install -a github.com/minio-io/donut/cmd/donut-cli + @mkdir -p $(HOME)/.minio/donut + +clean: + @rm -fv cover.out + @rm -fv build-constants.go diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/README.md b/Godeps/_workspace/src/github.com/minio-io/donut/README.md new file mode 100644 index 000000000..9d977f173 --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/README.md @@ -0,0 +1,3 @@ +# Donut + +donut - Donut (do not delete) on disk format implementation released under [Apache license v2](./LICENSE). diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/bucket.go b/Godeps/_workspace/src/github.com/minio-io/donut/bucket.go new file mode 100644 index 000000000..90842651b --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/bucket.go @@ -0,0 +1,240 @@ +/* + * Minimalist Object Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package donut + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "path" + "strconv" + "strings" + "time" + + "crypto/md5" + "encoding/hex" + "encoding/json" + + "github.com/minio-io/minio/pkg/utils/split" +) + +type bucket struct { + name string + donutName string + nodes map[string]Node + objects map[string]Object +} + +// NewBucket - instantiate a new bucket +func NewBucket(bucketName, donutName string, nodes map[string]Node) (Bucket, error) { + if bucketName == "" { + return nil, errors.New("invalid argument") + } + b := bucket{} + b.name = bucketName + b.donutName = donutName + b.objects = make(map[string]Object) + b.nodes = nodes + return b, nil +} + +func (b bucket) ListNodes() (map[string]Node, error) { + return b.nodes, nil +} + +func (b bucket) ListObjects() (map[string]Object, error) { + nodeSlice := 0 + for _, node := range b.nodes { + disks, err := node.ListDisks() + if err != nil { + return nil, err + } + for _, disk := range disks { + bucketSlice := fmt.Sprintf("%s$%d$%d", b.name, nodeSlice, disk.GetOrder()) + bucketPath := path.Join(b.donutName, bucketSlice) + objects, err := disk.ListDir(bucketPath) + if err != nil { + return nil, err + } + for _, object := range objects { + newObject, err := NewObject(object.Name(), path.Join(disk.GetPath(), bucketPath)) + if err != nil { + return nil, err + } + newObjectMetadata, err := newObject.GetObjectMetadata() + if err != nil { + return nil, err + } + objectName, ok := newObjectMetadata["object"] + if !ok { + return nil, errors.New("object corrupted") + } + b.objects[objectName] = newObject + } + } + nodeSlice = nodeSlice + 1 + } + return b.objects, nil +} + +func (b bucket) GetObject(objectName string) (reader io.ReadCloser, size int64, err error) { + reader, writer := io.Pipe() + // get list of objects + objects, err := b.ListObjects() + if err != nil { + return nil, 0, err + } + // check if object exists + object, ok := objects[objectName] + if !ok { + return nil, 0, os.ErrNotExist + } + donutObjectMetadata, err := object.GetDonutObjectMetadata() + if err != nil { + return nil, 0, err + } + if objectName == "" || writer == nil || len(donutObjectMetadata) == 0 { + return nil, 0, errors.New("invalid argument") + } + size, err = strconv.ParseInt(donutObjectMetadata["size"], 10, 64) + if err != nil { + return nil, 0, err + } + go b.getObject(b.normalizeObjectName(objectName), writer, donutObjectMetadata) + return reader, size, nil +} + +func (b bucket) WriteObjectMetadata(objectName string, objectMetadata map[string]string) error { + if len(objectMetadata) == 0 { + return errors.New("invalid argument") + } + objectMetadataWriters, err := b.getDiskWriters(objectName, objectMetadataConfig) + if err != nil { + return err + } + for _, objectMetadataWriter := range objectMetadataWriters { + defer objectMetadataWriter.Close() + } + for _, objectMetadataWriter := range objectMetadataWriters { + jenc := json.NewEncoder(objectMetadataWriter) + if err := jenc.Encode(objectMetadata); err != nil { + return err + } + } + return nil +} + +func (b bucket) WriteDonutObjectMetadata(objectName string, donutObjectMetadata map[string]string) error { + if len(donutObjectMetadata) == 0 { + return errors.New("invalid argument") + } + donutObjectMetadataWriters, err := b.getDiskWriters(objectName, donutObjectMetadataConfig) + if err != nil { + return err + } + for _, donutObjectMetadataWriter := range donutObjectMetadataWriters { + defer donutObjectMetadataWriter.Close() + } + for _, donutObjectMetadataWriter := range donutObjectMetadataWriters { + jenc := json.NewEncoder(donutObjectMetadataWriter) + if err := jenc.Encode(donutObjectMetadata); err != nil { + return err + } + } + return nil +} + +// This a temporary normalization of object path, need to find a better way +func (b bucket) normalizeObjectName(objectName string) string { + // replace every '/' with '-' + return strings.Replace(objectName, "/", "-", -1) +} + +func (b bucket) PutObject(objectName, contentType string, objectData io.Reader) error { + if objectName == "" { + return errors.New("invalid argument") + } + if objectData == nil { + return errors.New("invalid argument") + } + if contentType == "" || strings.TrimSpace(contentType) == "" { + contentType = "application/octet-stream" + } + writers, err := b.getDiskWriters(b.normalizeObjectName(objectName), "data") + if err != nil { + return err + } + for _, writer := range writers { + defer writer.Close() + } + summer := md5.New() + donutObjectMetadata := make(map[string]string) + switch len(writers) == 1 { + case true: + mw := io.MultiWriter(writers[0], summer) + totalLength, err := io.Copy(mw, objectData) + if err != nil { + return err + } + donutObjectMetadata["size"] = strconv.FormatInt(totalLength, 10) + case false: + k, m, err := b.getDataAndParity(len(writers)) + if err != nil { + return err + } + chunks := split.Stream(objectData, 10*1024*1024) + encoder, err := NewEncoder(k, m, "Cauchy") + if err != nil { + return err + } + chunkCount := 0 + totalLength := 0 + for chunk := range chunks { + if chunk.Err == nil { + totalLength = totalLength + len(chunk.Data) + encodedBlocks, _ := encoder.Encode(chunk.Data) + summer.Write(chunk.Data) + for blockIndex, block := range encodedBlocks { + io.Copy(writers[blockIndex], bytes.NewBuffer(block)) + } + } + chunkCount = chunkCount + 1 + } + donutObjectMetadata["blockSize"] = strconv.Itoa(10 * 1024 * 1024) + donutObjectMetadata["chunkCount"] = strconv.Itoa(chunkCount) + donutObjectMetadata["erasureK"] = strconv.FormatUint(uint64(k), 10) + donutObjectMetadata["erasureM"] = strconv.FormatUint(uint64(m), 10) + donutObjectMetadata["erasureTechnique"] = "Cauchy" + donutObjectMetadata["size"] = strconv.Itoa(totalLength) + } + dataMd5sum := summer.Sum(nil) + donutObjectMetadata["created"] = time.Now().Format(time.RFC3339Nano) + donutObjectMetadata["md5"] = hex.EncodeToString(dataMd5sum) + if err := b.WriteDonutObjectMetadata(b.normalizeObjectName(objectName), donutObjectMetadata); err != nil { + return err + } + objectMetadata := make(map[string]string) + objectMetadata["bucket"] = b.name + objectMetadata["object"] = objectName + objectMetadata["contentType"] = strings.TrimSpace(contentType) + if err := b.WriteObjectMetadata(b.normalizeObjectName(objectName), objectMetadata); err != nil { + return err + } + return nil +} diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/bucket_internal.go b/Godeps/_workspace/src/github.com/minio-io/donut/bucket_internal.go new file mode 100644 index 000000000..c4b2ae15c --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/bucket_internal.go @@ -0,0 +1,184 @@ +/* + * Minimalist Object Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package donut + +import ( + "bytes" + "crypto/md5" + "encoding/hex" + "errors" + "fmt" + "io" + "path" + "strconv" +) + +func (b bucket) getDataAndParity(totalWriters int) (k uint8, m uint8, err error) { + if totalWriters <= 1 { + return 0, 0, errors.New("invalid argument") + } + quotient := totalWriters / 2 // not using float or abs to let integer round off to lower value + // quotient cannot be bigger than (255 / 2) = 127 + if quotient > 127 { + return 0, 0, errors.New("parity over flow") + } + remainder := totalWriters % 2 // will be 1 for odd and 0 for even numbers + k = uint8(quotient + remainder) + m = uint8(quotient) + return k, m, nil +} + +func (b bucket) getObject(objectName string, writer *io.PipeWriter, donutObjectMetadata map[string]string) { + expectedMd5sum, err := hex.DecodeString(donutObjectMetadata["md5"]) + if err != nil { + writer.CloseWithError(err) + return + } + readers, err := b.getDiskReaders(objectName, "data") + if err != nil { + writer.CloseWithError(err) + return + } + hasher := md5.New() + mwriter := io.MultiWriter(writer, hasher) + switch len(readers) == 1 { + case false: + totalChunks, totalLeft, blockSize, k, m, err := b.metadata2Values(donutObjectMetadata) + if err != nil { + writer.CloseWithError(err) + return + } + technique, ok := donutObjectMetadata["erasureTechnique"] + if !ok { + writer.CloseWithError(errors.New("missing erasure Technique")) + return + } + encoder, err := NewEncoder(uint8(k), uint8(m), technique) + if err != nil { + writer.CloseWithError(err) + return + } + for i := 0; i < totalChunks; i++ { + decodedData, err := b.decodeData(totalLeft, blockSize, readers, encoder, writer) + if err != nil { + writer.CloseWithError(err) + return + } + _, err = io.Copy(mwriter, bytes.NewBuffer(decodedData)) + if err != nil { + writer.CloseWithError(err) + return + } + totalLeft = totalLeft - int64(blockSize) + } + case true: + _, err := io.Copy(writer, readers[0]) + if err != nil { + writer.CloseWithError(err) + return + } + } + // check if decodedData md5sum matches + if !bytes.Equal(expectedMd5sum, hasher.Sum(nil)) { + writer.CloseWithError(errors.New("checksum mismatch")) + return + } + writer.Close() + return +} + +func (b bucket) decodeData(totalLeft, blockSize int64, readers []io.ReadCloser, encoder Encoder, writer *io.PipeWriter) ([]byte, error) { + var curBlockSize int64 + if blockSize < totalLeft { + curBlockSize = blockSize + } else { + curBlockSize = totalLeft // cast is safe, blockSize in if protects + } + curChunkSize, err := encoder.GetEncodedBlockLen(int(curBlockSize)) + if err != nil { + return nil, err + } + encodedBytes := make([][]byte, len(readers)) + for i, reader := range readers { + var bytesBuffer bytes.Buffer + _, err := io.CopyN(&bytesBuffer, reader, int64(curChunkSize)) + if err != nil { + return nil, err + } + encodedBytes[i] = bytesBuffer.Bytes() + } + decodedData, err := encoder.Decode(encodedBytes, int(curBlockSize)) + if err != nil { + return nil, err + } + return decodedData, nil +} + +func (b bucket) metadata2Values(donutObjectMetadata map[string]string) (totalChunks int, totalLeft, blockSize int64, k, m uint64, err error) { + totalChunks, err = strconv.Atoi(donutObjectMetadata["chunkCount"]) + totalLeft, err = strconv.ParseInt(donutObjectMetadata["size"], 10, 64) + blockSize, err = strconv.ParseInt(donutObjectMetadata["blockSize"], 10, 64) + k, err = strconv.ParseUint(donutObjectMetadata["erasureK"], 10, 8) + m, err = strconv.ParseUint(donutObjectMetadata["erasureM"], 10, 8) + return +} + +func (b bucket) getDiskReaders(objectName, objectMeta string) ([]io.ReadCloser, error) { + var readers []io.ReadCloser + nodeSlice := 0 + for _, node := range b.nodes { + disks, err := node.ListDisks() + if err != nil { + return nil, err + } + readers = make([]io.ReadCloser, len(disks)) + for _, disk := range disks { + bucketSlice := fmt.Sprintf("%s$%d$%d", b.name, nodeSlice, disk.GetOrder()) + objectPath := path.Join(b.donutName, bucketSlice, objectName, objectMeta) + objectSlice, err := disk.OpenFile(objectPath) + if err != nil { + return nil, err + } + readers[disk.GetOrder()] = objectSlice + } + nodeSlice = nodeSlice + 1 + } + return readers, nil +} + +func (b bucket) getDiskWriters(objectName, objectMeta string) ([]io.WriteCloser, error) { + var writers []io.WriteCloser + nodeSlice := 0 + for _, node := range b.nodes { + disks, err := node.ListDisks() + if err != nil { + return nil, err + } + writers = make([]io.WriteCloser, len(disks)) + for _, disk := range disks { + bucketSlice := fmt.Sprintf("%s$%d$%d", b.name, nodeSlice, disk.GetOrder()) + objectPath := path.Join(b.donutName, bucketSlice, objectName, objectMeta) + objectSlice, err := disk.MakeFile(objectPath) + if err != nil { + return nil, err + } + writers[disk.GetOrder()] = objectSlice + } + nodeSlice = nodeSlice + 1 + } + return writers, nil +} diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/buildscripts/checkdeps.sh b/Godeps/_workspace/src/github.com/minio-io/donut/buildscripts/checkdeps.sh new file mode 100644 index 000000000..3d78c0449 --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/buildscripts/checkdeps.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +# +# Minio Commander, (C) 2015 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +_init() { + ## Minimum required versions for build dependencies + GCC_VERSION="4.0" + CLANG_VERSION="3.5" + YASM_VERSION="1.2.0" + GIT_VERSION="1.0" + GO_VERSION="1.4" + OSX_VERSION="10.8" + UNAME=$(uname -sm) + + ## Check all dependencies are present + MISSING="" +} + +### +# +# Takes two arguments +# arg1: version number in `x.x.x` format +# arg2: version number in `x.x.x` format +# +# example: check_version "$version1" "$version2" +# +# returns: +# 0 - Installed version is equal to required +# 1 - Installed version is greater than required +# 2 - Installed version is lesser than required +# 3 - If args have length zero +# +#### +check_version () { + ## validate args + [[ -z "$1" ]] && return 3 + [[ -z "$2" ]] && return 3 + + if [[ $1 == $2 ]]; then + return 0 + fi + + local IFS=. + local i ver1=($1) ver2=($2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do + ver1[i]=0 + done + for ((i=0; i<${#ver1[@]}; i++)); do + if [[ -z ${ver2[i]} ]]; then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})); then + + return 1 + fi + if ((10#${ver1[i]} < 10#${ver2[i]})); then + ## Installed version is lesser than required - Bad condition + return 2 + fi + done + return 0 +} + +check_golang_env() { + echo ${GOROOT:?} 2>&1 >/dev/null + if [ $? -eq 1 ]; then + echo "ERROR" + echo "GOROOT environment variable missing, please refer to Go installation document" + echo "https://github.com/Minio-io/minio/blob/master/BUILDDEPS.md#install-go-13" + exit 1 + fi + + echo ${GOPATH:?} 2>&1 >/dev/null + if [ $? -eq 1 ]; then + echo "ERROR" + echo "GOPATH environment variable missing, please refer to Go installation document" + echo "https://github.com/Minio-io/minio/blob/master/BUILDDEPS.md#install-go-13" + exit 1 + fi +} + +is_supported_os() { + case ${UNAME%% *} in + "Linux") + os="linux" + ;; + "Darwin") + osx_host_version=$(env sw_vers -productVersion) + check_version "${osx_host_version}" "${OSX_VERSION}" + [[ $? -ge 2 ]] && die "Minimum OSX version supported is ${OSX_VERSION}" + ;; + "*") + echo "Exiting.. unsupported operating system found" + exit 1; + esac +} + +is_supported_arch() { + local supported + case ${UNAME##* } in + "x86_64") + supported=1 + ;; + *) + supported=0 + ;; + esac + if [ $supported -eq 0 ]; then + echo "Invalid arch: ${UNAME} not supported, please use x86_64/amd64" + exit 1; + fi +} + +check_deps() { + check_version "$(env go version 2>/dev/null | sed 's/^.* go\([0-9.]*\).*$/\1/')" "${GO_VERSION}" + if [ $? -ge 2 ]; then + MISSING="${MISSING} golang(1.4)" + fi + + check_version "$(env git --version 2>/dev/null | sed -e 's/^.* \([0-9.\].*\).*$/\1/' -e 's/^\([0-9.\]*\).*/\1/g')" "${GIT_VERSION}" + if [ $? -ge 2 ]; then + MISSING="${MISSING} git" + fi + + case ${UNAME%% *} in + "Linux") + check_version "$(env gcc --version 2>/dev/null | sed 's/^.* \([0-9.]*\).*$/\1/' | head -1)" "${GCC_VERSION}" + if [ $? -ge 2 ]; then + MISSING="${MISSING} build-essential" + fi + ;; + "Darwin") + check_version "$(env gcc --version 2>/dev/null | sed 's/^.* \([0-9.]*\).*$/\1/' | head -1)" "${CLANG_VERSION}" + if [ $? -ge 2 ]; then + MISSING="${MISSING} xcode-cli" + fi + ;; + "*") + ;; + esac + + check_version "$(env yasm --version 2>/dev/null | sed 's/^.* \([0-9.]*\).*$/\1/' | head -1)" "${YASM_VERSION}" + if [ $? -ge 2 ]; then + MISSING="${MISSING} yasm(1.2.0)" + fi + + env mkdocs help >/dev/null 2>&1 + if [ $? -ne 0 ]; then + MISSING="${MISSING} mkdocs" + fi +} + +main() { + echo -n "Check for supported arch.. " + is_supported_arch + + echo -n "Check for supported os.. " + is_supported_os + + echo -n "Checking if proper environment variables are set.. " + check_golang_env + + echo "Done" + echo "Using GOPATH=${GOPATH} and GOROOT=${GOROOT}" + + echo -n "Checking dependencies for Minio.. " + check_deps + + ## If dependencies are missing, warn the user and abort + if [ "x${MISSING}" != "x" ]; then + echo "ERROR" + echo + echo "The following build tools are missing:" + echo + echo "** ${MISSING} **" + echo + echo "Please install them " + echo "${MISSING}" + echo + echo "Follow https://github.com/Minio-io/minio/blob/master/BUILDDEPS.md for further instructions" + exit 1 + fi + echo "Done" +} + +_init && main "$@" diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/buildscripts/git-commit-id.sh b/Godeps/_workspace/src/github.com/minio-io/donut/buildscripts/git-commit-id.sh new file mode 100644 index 000000000..bdcc76391 --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/buildscripts/git-commit-id.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +CONST_FILE=${PWD}/cmd/donut-cli/build-constants.go + +cat > $CONST_FILE < is needed for attach") + } + if _, ok := mcDonutConfigData.Donuts[donutName]; !ok { + log.Fatalf("Requested donut name %s does not exist, please use ``mc donut make`` first\n", donutName) + } + if _, ok := mcDonutConfigData.Donuts[donutName].Node["localhost"]; !ok { + log.Fatalln("Corrupted donut config, please consult donut experts") + } + activeDisks := mcDonutConfigData.Donuts[donutName].Node["localhost"].ActiveDisks + inactiveDisks := mcDonutConfigData.Donuts[donutName].Node["localhost"].InactiveDisks + for _, disk := range disks { + activeDisks = appendUniq(activeDisks, disk) + inactiveDisks = deleteFromSlice(inactiveDisks, disk) + } + + mcDonutConfigData.Donuts[donutName].Node["localhost"] = nodeConfig{ + ActiveDisks: activeDisks, + InactiveDisks: inactiveDisks, + } + if err := saveDonutConfig(mcDonutConfigData); err != nil { + log.Fatalln(err) + } +} diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-common.go b/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-common.go new file mode 100644 index 000000000..d79243dfe --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-common.go @@ -0,0 +1,110 @@ +/* + * Minimalist Object Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "strings" + + "net/url" +) + +// url2Object converts URL to bucket and objectname +func url2Object(urlStr string) (bucketName, objectName string, err error) { + u, err := url.Parse(urlStr) + if u.Path == "" { + // No bucket name passed. It is a valid case + return "", "", nil + } + splits := strings.SplitN(u.Path, "/", 3) + switch len(splits) { + case 0, 1: + bucketName = "" + objectName = "" + case 2: + bucketName = splits[1] + objectName = "" + case 3: + bucketName = splits[1] + objectName = splits[2] + } + return bucketName, objectName, nil +} + +func isStringInSlice(items []string, item string) bool { + for _, s := range items { + if s == item { + return true + } + } + return false +} + +func deleteFromSlice(items []string, item string) []string { + var newitems []string + for _, s := range items { + if s == item { + continue + } + newitems = append(newitems, s) + } + return newitems +} + +func appendUniq(slice []string, i string) []string { + for _, ele := range slice { + if ele == i { + return slice + } + } + return append(slice, i) +} + +// Is alphanumeric? +func isalnum(c rune) bool { + return '0' <= c && c <= '9' || 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' +} + +// isValidDonutName - verify donutName to be valid +func isValidDonutName(donutName string) bool { + if len(donutName) > 1024 || len(donutName) == 0 { + return false + } + for _, char := range donutName { + if isalnum(char) { + continue + } + switch char { + case '-': + case '.': + case '_': + case '~': + continue + default: + return false + } + } + return true +} + +// getNodeMap - get a node and disk map through nodeConfig struct +func getNodeMap(node map[string]nodeConfig) map[string][]string { + nodes := make(map[string][]string) + for k, v := range node { + nodes[k] = v.ActiveDisks + } + return nodes +} diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-config.go b/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-config.go new file mode 100644 index 000000000..f5aada16a --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-config.go @@ -0,0 +1,105 @@ +/* + * Minimalist Object Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + "log" + "os" + "os/user" + "path" + + "encoding/json" + "io/ioutil" +) + +const ( + donutConfigDir = ".minio/donut" + donutConfigFilename = "donuts.json" +) + +type nodeConfig struct { + ActiveDisks []string + InactiveDisks []string +} + +type donutConfig struct { + Node map[string]nodeConfig +} + +type mcDonutConfig struct { + Donuts map[string]donutConfig +} + +func getDonutConfigDir() string { + u, err := user.Current() + if err != nil { + msg := fmt.Sprintf("Unable to obtain user's home directory. \nError: %s", err) + log.Fatalln(msg) + } + + return path.Join(u.HomeDir, donutConfigDir) +} + +func getDonutConfigFilename() string { + return path.Join(getDonutConfigDir(), "donuts.json") +} + +// saveDonutConfig writes configuration data in json format to donut config file. +func saveDonutConfig(donutConfigData *mcDonutConfig) error { + jsonConfig, err := json.MarshalIndent(donutConfigData, "", "\t") + if err != nil { + return err + } + + err = os.MkdirAll(getDonutConfigDir(), 0755) + if !os.IsExist(err) && err != nil { + return err + } + + configFile, err := os.OpenFile(getDonutConfigFilename(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + return err + } + defer configFile.Close() + + _, err = configFile.Write(jsonConfig) + if err != nil { + return err + } + return nil +} + +func loadDonutConfig() (donutConfigData *mcDonutConfig, err error) { + configFile := getDonutConfigFilename() + _, err = os.Stat(configFile) + if err != nil { + return nil, err + } + + configBytes, err := ioutil.ReadFile(configFile) + if err != nil { + return nil, err + } + + err = json.Unmarshal(configBytes, &donutConfigData) + if err != nil { + return nil, err + } + + return donutConfigData, nil +} diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-detach-disk.go b/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-detach-disk.go new file mode 100644 index 000000000..1f6ab79cf --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-detach-disk.go @@ -0,0 +1,66 @@ +/* + * Minimalist Object Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + "log" + + "github.com/minio-io/cli" +) + +func doDetachDiskCmd(c *cli.Context) { + if !c.Args().Present() { + log.Fatalln("no args?") + } + disks := c.Args() + mcDonutConfigData, err := loadDonutConfig() + if err != nil { + log.Fatalln(err.Error()) + } + donutName := c.String("name") + if donutName == "" { + log.Fatalln("Invalid --donut is needed for attach") + } + if _, ok := mcDonutConfigData.Donuts[donutName]; !ok { + msg := fmt.Sprintf("Requested donut name <%s> does not exist, please use ``mc donut make`` first", donutName) + log.Fatalln(msg) + } + if _, ok := mcDonutConfigData.Donuts[donutName].Node["localhost"]; !ok { + msg := fmt.Sprintf("Corrupted donut config, please consult donut experts") + log.Fatalln(msg) + } + + inactiveDisks := mcDonutConfigData.Donuts[donutName].Node["localhost"].InactiveDisks + activeDisks := mcDonutConfigData.Donuts[donutName].Node["localhost"].ActiveDisks + for _, disk := range disks { + if isStringInSlice(activeDisks, disk) { + activeDisks = deleteFromSlice(activeDisks, disk) + inactiveDisks = appendUniq(inactiveDisks, disk) + } else { + msg := fmt.Sprintf("Cannot detach disk: <%s>, not part of donut <%s>", disk, donutName) + log.Println(msg) + } + } + mcDonutConfigData.Donuts[donutName].Node["localhost"] = nodeConfig{ + ActiveDisks: activeDisks, + InactiveDisks: inactiveDisks, + } + if err := saveDonutConfig(mcDonutConfigData); err != nil { + log.Fatalln(err.Error()) + } +} diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-info.go b/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-info.go new file mode 100644 index 000000000..2db70f5c9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-info.go @@ -0,0 +1,84 @@ +/* + * Minimalist Object Storage, (C) 2014,2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "os" + "strings" + + "text/tabwriter" + "text/template" + + "github.com/minio-io/cli" + "github.com/minio-io/donut" +) + +var infoTemplate = ` +{{range $donutName, $nodes := .}} +DONUTNAME: {{$donutName}} +{{range $nodeName, $disks := $nodes}} +NODE: {{$nodeName}} + DISKS: {{$disks}} +{{end}} +{{end}} +` + +var infoPrinter = func(templ string, data interface{}) { + funcMap := template.FuncMap{ + "join": strings.Join, + } + + w := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', 0) + t := template.Must(template.New("help").Funcs(funcMap).Parse(templ)) + err := t.Execute(w, data) + if err != nil { + panic(err) + } + w.Flush() +} + +// doInfoDonutCmd +func doInfoDonutCmd(c *cli.Context) { + if !c.Args().Present() { + log.Fatalln("no args?") + } + if len(c.Args()) != 1 { + log.Fatalln("invalid number of args") + } + donutName := c.Args().First() + if !isValidDonutName(donutName) { + log.Fatalln("Invalid donutName") + } + mcDonutConfigData, err := loadDonutConfig() + if err != nil { + log.Fatalln(err) + } + if _, ok := mcDonutConfigData.Donuts[donutName]; !ok { + log.Fatalln("donut does not exist") + } + d, err := donut.NewDonut(donutName, getNodeMap(mcDonutConfigData.Donuts[donutName].Node)) + if err != nil { + log.Fatalln(err) + } + donutNodes := make(map[string]map[string][]string) + donutNodes[donutName], err = d.Info() + if err != nil { + log.Fatalln(err) + } + infoPrinter(infoTemplate, donutNodes) +} diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-list.go b/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-list.go new file mode 100644 index 000000000..a692bf77e --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-list.go @@ -0,0 +1,36 @@ +/* + * Minimalist Object Storage, (C) 2014,2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + "log" + + "github.com/minio-io/cli" +) + +// doListDonutCmd creates a new bucket +func doListDonutCmd(c *cli.Context) { + mcDonutConfigData, err := loadDonutConfig() + if err != nil { + log.Fatalln(err) + } + + for k := range mcDonutConfigData.Donuts { + fmt.Println(k) + } +} diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-make.go b/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-make.go new file mode 100644 index 000000000..8e0d06a5a --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-make.go @@ -0,0 +1,80 @@ +/* + * Minimalist Object Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + "log" + "os" + + "github.com/minio-io/cli" +) + +func newDonutConfig(donutName string) (*mcDonutConfig, error) { + mcDonutConfigData := new(mcDonutConfig) + mcDonutConfigData.Donuts = make(map[string]donutConfig) + mcDonutConfigData.Donuts[donutName] = donutConfig{ + Node: make(map[string]nodeConfig), + } + mcDonutConfigData.Donuts[donutName].Node["localhost"] = nodeConfig{ + ActiveDisks: make([]string, 0), + InactiveDisks: make([]string, 0), + } + return mcDonutConfigData, nil +} + +// doMakeDonutCmd creates a new donut +func doMakeDonutCmd(c *cli.Context) { + if !c.Args().Present() { + log.Fatalln("no args?") + } + if len(c.Args()) != 1 { + log.Fatalln("invalid number of args") + } + donutName := c.Args().First() + if !isValidDonutName(donutName) { + log.Fatalln("Invalid donutName") + } + mcDonutConfigData, err := loadDonutConfig() + if os.IsNotExist(err) { + mcDonutConfigData, err = newDonutConfig(donutName) + if err != nil { + log.Fatalln(err) + } + if err := saveDonutConfig(mcDonutConfigData); err != nil { + log.Fatalln(err) + } + return + } else if err != nil { + log.Fatalln(err) + } + if _, ok := mcDonutConfigData.Donuts[donutName]; !ok { + mcDonutConfigData.Donuts[donutName] = donutConfig{ + Node: make(map[string]nodeConfig), + } + mcDonutConfigData.Donuts[donutName].Node["localhost"] = nodeConfig{ + ActiveDisks: make([]string, 0), + InactiveDisks: make([]string, 0), + } + if err := saveDonutConfig(mcDonutConfigData); err != nil { + log.Fatalln(err) + } + } else { + msg := fmt.Sprintf("donut: %s already exists", donutName) + log.Println(msg) + } +} diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-options.go b/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-options.go new file mode 100644 index 000000000..5fa40d48e --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-options.go @@ -0,0 +1,95 @@ +/* + * Minimalist Object Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "github.com/minio-io/cli" +) + +var makeDonutCmd = cli.Command{ + Name: "make", + Usage: "make donut", + Description: "Make a new donut", + Action: doMakeDonutCmd, +} + +var listDonutCmd = cli.Command{ + Name: "list", + Usage: "list donuts", + Description: "list all donuts locally or remote", + Action: doListDonutCmd, +} + +var attachDiskCmd = cli.Command{ + Name: "attach", + Usage: "attach disk", + Description: "Attach disk to an existing donut", + Action: doAttachDiskCmd, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "name", + Usage: "Donut name", + }, + }, +} + +var detachDiskCmd = cli.Command{ + Name: "detach", + Usage: "detach disk", + Description: "Detach disk from an existing donut", + Action: doDetachDiskCmd, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "name", + Usage: "Donut name", + }, + }, +} + +var healDonutCmd = cli.Command{ + Name: "heal", + Usage: "heal donut", + Description: "Heal donut with any errors", + Action: doHealDonutCmd, +} + +var rebalanceDonutCmd = cli.Command{ + Name: "rebalance", + Usage: "rebalance donut", + Description: "Rebalance data on donut after adding disks", + Action: doRebalanceDonutCmd, +} + +var infoDonutCmd = cli.Command{ + Name: "info", + Usage: "information about donut", + Description: "Pretty print donut information", + Action: doInfoDonutCmd, +} + +var donutOptions = []cli.Command{ + makeDonutCmd, + listDonutCmd, + attachDiskCmd, + detachDiskCmd, + healDonutCmd, + rebalanceDonutCmd, + infoDonutCmd, +} + +func doHealDonutCmd(c *cli.Context) { +} diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-rebalance.go b/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-rebalance.go new file mode 100644 index 000000000..fcfa5779f --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut-cmd-rebalance.go @@ -0,0 +1,32 @@ +package main + +import ( + "log" + + "github.com/minio-io/cli" + "github.com/minio-io/donut" +) + +func doRebalanceDonutCmd(c *cli.Context) { + if !c.Args().Present() { + log.Fatalln("no args?") + } + donutName := c.Args().First() + if !isValidDonutName(donutName) { + log.Fatalln("Invalid donutName") + } + mcDonutConfigData, err := loadDonutConfig() + if err != nil { + log.Fatalln(err) + } + if _, ok := mcDonutConfigData.Donuts[donutName]; !ok { + log.Fatalln("donut does not exist") + } + d, err := donut.NewDonut(donutName, getNodeMap(mcDonutConfigData.Donuts[donutName].Node)) + if err != nil { + log.Fatalln(err) + } + if err := d.Rebalance(); err != nil { + log.Fatalln(err) + } +} diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut.go b/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut.go new file mode 100644 index 000000000..21cb1d53c --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/cmd/donut-cli/donut.go @@ -0,0 +1,33 @@ +/* + * Minimalist Object Storage, (C) 2014,2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "os" + + "github.com/minio-io/cli" +) + +func main() { + app := cli.NewApp() + app.Usage = "" + app.Version = gitCommitHash + app.Commands = donutOptions + app.Author = "Minio.io" + app.EnableBashCompletion = true + app.Run(os.Args) +} diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/disk.go b/Godeps/_workspace/src/github.com/minio-io/donut/disk.go new file mode 100644 index 000000000..78f909d3c --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/disk.go @@ -0,0 +1,150 @@ +/* + * Minimalist Object Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package donut + +import ( + "errors" + "os" + "path" + "syscall" + + "io/ioutil" +) + +type disk struct { + root string + order int + filesystem map[string]string +} + +// NewDisk - instantiate new disk +func NewDisk(diskPath string, diskOrder int) (Disk, error) { + if diskPath == "" || diskOrder < 0 { + return nil, errors.New("invalid argument") + } + s := syscall.Statfs_t{} + err := syscall.Statfs(diskPath, &s) + if err != nil { + return nil, err + } + st, err := os.Stat(diskPath) + if err != nil { + return nil, err + } + if !st.IsDir() { + return nil, syscall.ENOTDIR + } + d := disk{ + root: diskPath, + order: diskOrder, + filesystem: make(map[string]string), + } + if fsType := d.getFSType(s.Type); fsType != "UNKNOWN" { + d.filesystem["FSType"] = fsType + d.filesystem["MountPoint"] = d.root + return d, nil + } + return nil, errors.New("unsupported filesystem") +} + +func (d disk) GetPath() string { + return d.root +} + +func (d disk) GetOrder() int { + return d.order +} + +func (d disk) GetFSInfo() map[string]string { + s := syscall.Statfs_t{} + err := syscall.Statfs(d.root, &s) + if err != nil { + return nil + } + d.filesystem["Total"] = d.formatBytes(s.Bsize * int64(s.Blocks)) + d.filesystem["Free"] = d.formatBytes(s.Bsize * int64(s.Bfree)) + return d.filesystem +} + +func (d disk) MakeDir(dirname string) error { + return os.MkdirAll(path.Join(d.root, dirname), 0700) +} + +func (d disk) ListDir(dirname string) ([]os.FileInfo, error) { + contents, err := ioutil.ReadDir(path.Join(d.root, dirname)) + if err != nil { + return nil, err + } + var directories []os.FileInfo + for _, content := range contents { + // Include only directories, ignore everything else + if content.IsDir() { + directories = append(directories, content) + } + } + return directories, nil +} + +func (d disk) ListFiles(dirname string) ([]os.FileInfo, error) { + contents, err := ioutil.ReadDir(path.Join(d.root, dirname)) + if err != nil { + return nil, err + } + var files []os.FileInfo + for _, content := range contents { + // Include only regular files, ignore everything else + if content.Mode().IsRegular() { + files = append(files, content) + } + } + return files, nil +} + +func (d disk) MakeFile(filename string) (*os.File, error) { + if filename == "" { + return nil, errors.New("Invalid argument") + } + filePath := path.Join(d.root, filename) + // Create directories if they don't exist + if err := os.MkdirAll(path.Dir(filePath), 0700); err != nil { + return nil, err + } + dataFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return nil, err + } + return dataFile, nil +} + +func (d disk) OpenFile(filename string) (*os.File, error) { + if filename == "" { + return nil, errors.New("Invalid argument") + } + dataFile, err := os.Open(path.Join(d.root, filename)) + if err != nil { + return nil, err + } + return dataFile, nil +} + +func (d disk) SaveConfig() error { + return errors.New("Not Implemented") +} + +func (d disk) LoadConfig() error { + return errors.New("Not Implemented") +} diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/disk_internal.go b/Godeps/_workspace/src/github.com/minio-io/donut/disk_internal.go new file mode 100644 index 000000000..0f78abc66 --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/disk_internal.go @@ -0,0 +1,63 @@ +/* + * Minimalist Object Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package donut + +import ( + "fmt" + "strconv" + "strings" +) + +// Convert bytes to human readable string. Like a 2 MB, 64.2 KB, 52 B +func (d disk) formatBytes(i int64) (result string) { + switch { + case i > (1024 * 1024 * 1024 * 1024): + result = fmt.Sprintf("%.02f TB", float64(i)/1024/1024/1024/1024) + case i > (1024 * 1024 * 1024): + result = fmt.Sprintf("%.02f GB", float64(i)/1024/1024/1024) + case i > (1024 * 1024): + result = fmt.Sprintf("%.02f MB", float64(i)/1024/1024) + case i > 1024: + result = fmt.Sprintf("%.02f KB", float64(i)/1024) + default: + result = fmt.Sprintf("%d B", i) + } + result = strings.Trim(result, " ") + return +} + +var fsType2StringMap = map[string]string{ + "137d": "EXT", + "ef51": "EXT2OLD", + "ef53": "EXT4", + "4244": "HFS", + "5346544e": "NTFS", + "4d44": "MSDOS", + "52654973": "REISERFS", + "6969": "NFS", + "01021994": "TMPFS", + "58465342": "XFS", +} + +func (d disk) getFSType(fsType int64) string { + fsTypeHex := strconv.FormatInt(fsType, 16) + fsTypeString, ok := fsType2StringMap[fsTypeHex] + if ok == false { + return "UNKNOWN" + } + return fsTypeString +} diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/donut.go b/Godeps/_workspace/src/github.com/minio-io/donut/donut.go new file mode 100644 index 000000000..0fba7fdd5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/donut.go @@ -0,0 +1,200 @@ +/* + * Minimalist Object Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package donut + +import ( + "encoding/json" + "errors" + "fmt" + "path" + "strings" +) + +type donut struct { + name string + buckets map[string]Bucket + nodes map[string]Node +} + +// attachDonutNode - wrapper function to instantiate a new node for associated donut +// based on the configuration +func (d donut) attachDonutNode(hostname string, disks []string) error { + node, err := NewNode(hostname) + if err != nil { + return err + } + for i, disk := range disks { + // Order is necessary for maps, keep order number separately + newDisk, err := NewDisk(disk, i) + if err != nil { + return err + } + if err := newDisk.MakeDir(d.name); err != nil { + return err + } + if err := node.AttachDisk(newDisk); err != nil { + return err + } + } + if err := d.AttachNode(node); err != nil { + return err + } + return nil +} + +// NewDonut - instantiate a new donut +func NewDonut(donutName string, nodeDiskMap map[string][]string) (Donut, error) { + if donutName == "" || len(nodeDiskMap) == 0 { + return nil, errors.New("invalid argument") + } + nodes := make(map[string]Node) + buckets := make(map[string]Bucket) + d := donut{ + name: donutName, + nodes: nodes, + buckets: buckets, + } + for k, v := range nodeDiskMap { + if len(v) == 0 { + return nil, errors.New("invalid number of disks per node") + } + err := d.attachDonutNode(k, v) + if err != nil { + return nil, err + } + } + return d, nil +} + +func (d donut) MakeBucket(bucketName string) error { + if bucketName == "" || strings.TrimSpace(bucketName) == "" { + return errors.New("invalid argument") + } + if _, ok := d.buckets[bucketName]; ok { + return errors.New("bucket exists") + } + bucket, err := NewBucket(bucketName, d.name, d.nodes) + if err != nil { + return err + } + nodeNumber := 0 + d.buckets[bucketName] = bucket + for _, node := range d.nodes { + disks, err := node.ListDisks() + if err != nil { + return err + } + for _, disk := range disks { + bucketSlice := fmt.Sprintf("%s$%d$%d", bucketName, nodeNumber, disk.GetOrder()) + err := disk.MakeDir(path.Join(d.name, bucketSlice)) + if err != nil { + return err + } + } + nodeNumber = nodeNumber + 1 + } + return nil +} + +func (d donut) ListBuckets() (map[string]Bucket, error) { + for _, node := range d.nodes { + disks, err := node.ListDisks() + if err != nil { + return nil, err + } + for _, disk := range disks { + dirs, err := disk.ListDir(d.name) + if err != nil { + return nil, err + } + for _, dir := range dirs { + splitDir := strings.Split(dir.Name(), "$") + if len(splitDir) < 3 { + return nil, errors.New("corrupted backend") + } + bucketName := splitDir[0] + // we dont need this NewBucket once we cache these + bucket, err := NewBucket(bucketName, d.name, d.nodes) + if err != nil { + return nil, err + } + d.buckets[bucketName] = bucket + } + } + } + return d.buckets, nil +} + +func (d donut) Heal() error { + return errors.New("Not Implemented") +} + +func (d donut) Info() (nodeDiskMap map[string][]string, err error) { + nodeDiskMap = make(map[string][]string) + for nodeName, node := range d.nodes { + disks, err := node.ListDisks() + if err != nil { + return nil, err + } + diskList := make([]string, len(disks)) + for diskName, disk := range disks { + diskList[disk.GetOrder()] = diskName + } + nodeDiskMap[nodeName] = diskList + } + return nodeDiskMap, nil +} + +func (d donut) AttachNode(node Node) error { + if node == nil { + return errors.New("invalid argument") + } + d.nodes[node.GetNodeName()] = node + return nil +} +func (d donut) DetachNode(node Node) error { + delete(d.nodes, node.GetNodeName()) + return nil +} + +func (d donut) SaveConfig() error { + nodeDiskMap := make(map[string][]string) + for hostname, node := range d.nodes { + disks, err := node.ListDisks() + if err != nil { + return err + } + for _, disk := range disks { + donutConfigPath := path.Join(d.name, donutConfig) + donutConfigWriter, err := disk.MakeFile(donutConfigPath) + defer donutConfigWriter.Close() + if err != nil { + return err + } + nodeDiskMap[hostname][disk.GetOrder()] = disk.GetPath() + jenc := json.NewEncoder(donutConfigWriter) + if err := jenc.Encode(nodeDiskMap); err != nil { + return err + } + } + } + return nil +} + +func (d donut) LoadConfig() error { + return errors.New("Not Implemented") +} diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/erasure.go b/Godeps/_workspace/src/github.com/minio-io/donut/erasure.go new file mode 100644 index 000000000..9eb41dca9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/erasure.go @@ -0,0 +1,85 @@ +/* + * Minimalist Object Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package donut + +import ( + "errors" + + encoding "github.com/minio-io/erasure" +) + +type encoder struct { + encoder *encoding.Erasure + k, m uint8 + technique encoding.Technique +} + +// getErasureTechnique - convert technique string into Technique type +func getErasureTechnique(technique string) (encoding.Technique, error) { + switch true { + case technique == "Cauchy": + return encoding.Cauchy, nil + case technique == "Vandermonde": + return encoding.Cauchy, nil + default: + return encoding.None, errors.New("Invalid erasure technique") + } +} + +// NewEncoder - instantiate a new encoder +func NewEncoder(k, m uint8, technique string) (Encoder, error) { + e := encoder{} + t, err := getErasureTechnique(technique) + if err != nil { + return nil, err + } + params, err := encoding.ValidateParams(k, m, t) + if err != nil { + return nil, err + } + e.encoder = encoding.NewErasure(params) + e.k = k + e.m = m + e.technique = t + return e, nil +} + +func (e encoder) GetEncodedBlockLen(dataLength int) (int, error) { + if dataLength == 0 { + return 0, errors.New("invalid argument") + } + return encoding.GetEncodedBlockLen(dataLength, e.k), nil +} + +func (e encoder) Encode(data []byte) (encodedData [][]byte, err error) { + if data == nil { + return nil, errors.New("invalid argument") + } + encodedData, err = e.encoder.Encode(data) + if err != nil { + return nil, err + } + return encodedData, nil +} + +func (e encoder) Decode(encodedData [][]byte, dataLength int) (data []byte, err error) { + decodedData, err := e.encoder.Decode(encodedData, dataLength) + if err != nil { + return nil, err + } + return decodedData, nil +} diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/interfaces.go b/Godeps/_workspace/src/github.com/minio-io/donut/interfaces.go new file mode 100644 index 000000000..750fbdf4d --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/interfaces.go @@ -0,0 +1,106 @@ +/* + * Minimalist Object Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package donut + +import ( + "io" + "os" +) + +// Collection of Donut specification interfaces + +// Donut interface +type Donut interface { + Storage + Management +} + +// Storage object storage interface +type Storage interface { + MakeBucket(bucket string) error + ListBuckets() (map[string]Bucket, error) +} + +// Management is a donut management system interface +type Management interface { + Heal() error + Rebalance() error + Info() (map[string][]string, error) + + AttachNode(node Node) error + DetachNode(node Node) error + + SaveConfig() error + LoadConfig() error +} + +// Encoder interface +type Encoder interface { + GetEncodedBlockLen(dataLength int) (int, error) + Encode(data []byte) (encodedData [][]byte, err error) + Decode(encodedData [][]byte, dataLength int) (data []byte, err error) +} + +// Bucket interface +type Bucket interface { + ListNodes() (map[string]Node, error) + ListObjects() (map[string]Object, error) + + GetObject(object string) (io.ReadCloser, int64, error) + PutObject(object, contentType string, contents io.Reader) error + + WriteDonutObjectMetadata(object string, donutMetadata map[string]string) error + WriteObjectMetadata(object string, objectMetadata map[string]string) error +} + +// Object interface +type Object interface { + GetObjectMetadata() (map[string]string, error) + GetDonutObjectMetadata() (map[string]string, error) +} + +// Node interface +type Node interface { + ListDisks() (map[string]Disk, error) + AttachDisk(disk Disk) error + DetachDisk(disk Disk) error + + GetNodeName() string + SaveConfig() error + LoadConfig() error +} + +// Disk interface +type Disk interface { + MakeDir(dirname string) error + + ListDir(dirname string) ([]os.FileInfo, error) + ListFiles(dirname string) ([]os.FileInfo, error) + + MakeFile(path string) (*os.File, error) + OpenFile(path string) (*os.File, error) + + GetPath() string + GetOrder() int + GetFSInfo() map[string]string +} + +const ( + donutObjectMetadataConfig = "donutObjectMetadata.json" + objectMetadataConfig = "objectMetadata.json" + donutConfig = "donutMetadata.json" +) diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/node.go b/Godeps/_workspace/src/github.com/minio-io/donut/node.go new file mode 100644 index 000000000..25a7d28f3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/node.go @@ -0,0 +1,68 @@ +/* + * Minimalist Object Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package donut + +import ( + "errors" +) + +type node struct { + hostname string + disks map[string]Disk +} + +// NewNode - instantiates a new node +func NewNode(hostname string) (Node, error) { + if hostname == "" { + return nil, errors.New("invalid argument") + } + disks := make(map[string]Disk) + n := node{ + hostname: hostname, + disks: disks, + } + return n, nil +} + +func (n node) GetNodeName() string { + return n.hostname +} + +func (n node) ListDisks() (map[string]Disk, error) { + return n.disks, nil +} + +func (n node) AttachDisk(disk Disk) error { + if disk == nil { + return errors.New("Invalid argument") + } + n.disks[disk.GetPath()] = disk + return nil +} + +func (n node) DetachDisk(disk Disk) error { + delete(n.disks, disk.GetPath()) + return nil +} + +func (n node) SaveConfig() error { + return errors.New("Not Implemented") +} + +func (n node) LoadConfig() error { + return errors.New("Not Implemented") +} diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/object.go b/Godeps/_workspace/src/github.com/minio-io/donut/object.go new file mode 100644 index 000000000..ecc2a884f --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/object.go @@ -0,0 +1,69 @@ +/* + * Minimalist Object Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package donut + +import ( + "errors" + "path" + + "encoding/json" + "io/ioutil" +) + +type object struct { + name string + objectPath string + objectMetadata map[string]string + donutObjectMetadata map[string]string +} + +// NewObject - instantiate a new object +func NewObject(objectName, p string) (Object, error) { + if objectName == "" { + return nil, errors.New("invalid argument") + } + o := object{} + o.name = objectName + o.objectPath = path.Join(p, objectName) + return o, nil +} + +func (o object) GetObjectMetadata() (map[string]string, error) { + objectMetadata := make(map[string]string) + objectMetadataBytes, err := ioutil.ReadFile(path.Join(o.objectPath, objectMetadataConfig)) + if err != nil { + return nil, err + } + if err := json.Unmarshal(objectMetadataBytes, &objectMetadata); err != nil { + return nil, err + } + o.objectMetadata = objectMetadata + return objectMetadata, nil +} + +func (o object) GetDonutObjectMetadata() (map[string]string, error) { + donutObjectMetadata := make(map[string]string) + donutObjectMetadataBytes, err := ioutil.ReadFile(path.Join(o.objectPath, donutObjectMetadataConfig)) + if err != nil { + return nil, err + } + if err := json.Unmarshal(donutObjectMetadataBytes, &donutObjectMetadata); err != nil { + return nil, err + } + o.donutObjectMetadata = donutObjectMetadata + return donutObjectMetadata, nil +} diff --git a/Godeps/_workspace/src/github.com/minio-io/donut/rebalance.go b/Godeps/_workspace/src/github.com/minio-io/donut/rebalance.go new file mode 100644 index 000000000..6fe87f336 --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/donut/rebalance.go @@ -0,0 +1,37 @@ +package donut + +import ( + "fmt" + "os" + "strings" +) + +func (d donut) Rebalance() error { + var totalOffSetLength int + var newDisks []Disk + var existingDirs []os.FileInfo + for _, node := range d.nodes { + disks, err := node.ListDisks() + if err != nil { + return err + } + totalOffSetLength = len(disks) + fmt.Println(totalOffSetLength) + for _, disk := range disks { + dirs, err := disk.ListDir(d.name) + if err != nil { + return err + } + if len(dirs) == 0 { + newDisks = append(newDisks, disk) + } + existingDirs = append(existingDirs, dirs...) + } + } + for _, dir := range existingDirs { + splits := strings.Split(dir.Name(), "$") + bucketName, segment, offset := splits[0], splits[1], splits[2] + fmt.Println(bucketName, segment, offset) + } + return nil +} diff --git a/pkg/drivers/donut/donut.go b/pkg/drivers/donut/donut.go index a088a6c96..5497df602 100644 --- a/pkg/drivers/donut/donut.go +++ b/pkg/drivers/donut/donut.go @@ -19,6 +19,8 @@ package donut import ( "errors" "io" + "os" + "path" "sort" "strconv" "strings" @@ -26,9 +28,9 @@ import ( "io/ioutil" + "github.com/minio-io/donut" "github.com/minio-io/iodine" "github.com/minio-io/minio/pkg/drivers" - "github.com/minio-io/minio/pkg/storage/donut" "github.com/minio-io/minio/pkg/utils/log" ) @@ -41,20 +43,44 @@ const ( blockSize = 10 * 1024 * 1024 ) +// This is a dummy nodeDiskMap which is going to be deprecated soon +// once the Management API is standardized, this map is useful for now +// to show multi disk API correctness behavior. +// +// This should be obtained from donut configuration file +func createNodeDiskMap(p string) map[string][]string { + nodes := make(map[string][]string) + nodes["localhost"] = make([]string, 16) + for i := 0; i < len(nodes["localhost"]); i++ { + diskPath := path.Join(p, strconv.Itoa(i)) + if _, err := os.Stat(diskPath); err != nil { + if os.IsNotExist(err) { + os.MkdirAll(diskPath, 0700) + } + } + nodes["localhost"][i] = diskPath + } + return nodes +} + // Start a single disk subsystem func Start(path string) (chan<- string, <-chan error, drivers.Driver) { ctrlChannel := make(chan string) errorChannel := make(chan error) - s := new(donutDriver) + errParams := map[string]string{"path": path} - // TODO donut driver should be passed in as Start param and driven by config - var err error - s.donut, err = donut.NewDonut(path) - err = iodine.New(err, map[string]string{"path": path}) + // Soon to be user configurable, when Management API + // is finished we remove "default" to something + // which is passed down from configuration + donut, err := donut.NewDonut("default", createNodeDiskMap(path)) if err != nil { + err = iodine.New(err, errParams) log.Error.Println(err) } + s := new(donutDriver) + s.donut = donut + go start(ctrlChannel, errorChannel, s) return ctrlChannel, errorChannel, s } @@ -63,26 +89,37 @@ func start(ctrlChannel <-chan string, errorChannel chan<- error, s *donutDriver) close(errorChannel) } +// byBucketName is a type for sorting bucket metadata by bucket name +type byBucketName []drivers.BucketMetadata + +func (b byBucketName) Len() int { return len(b) } +func (b byBucketName) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b byBucketName) Less(i, j int) bool { return b[i].Name < b[j].Name } + // ListBuckets returns a list of buckets func (d donutDriver) ListBuckets() (results []drivers.BucketMetadata, err error) { buckets, err := d.donut.ListBuckets() if err != nil { return nil, err } - for _, bucket := range buckets { + for name := range buckets { result := drivers.BucketMetadata{ - Name: bucket, + Name: name, // TODO Add real created date Created: time.Now(), } results = append(results, result) } + sort.Sort(byBucketName(results)) return results, nil } // CreateBucket creates a new bucket -func (d donutDriver) CreateBucket(bucket string) error { - return d.donut.CreateBucket(bucket) +func (d donutDriver) CreateBucket(bucketName string) error { + if drivers.IsValidBucket(bucketName) && !strings.Contains(bucketName, ".") { + return d.donut.MakeBucket(bucketName) + } + return errors.New("Invalid bucket") } // GetBucketMetadata retrieves an bucket's metadata @@ -101,30 +138,75 @@ func (d donutDriver) GetBucketPolicy(bucket string) (drivers.BucketPolicy, error } // GetObject retrieves an object and writes it to a writer -func (d donutDriver) GetObject(target io.Writer, bucket, key string) (int64, error) { - reader, err := d.donut.GetObjectReader(bucket, key) - if err != nil { - return 0, drivers.ObjectNotFound{ - Bucket: bucket, - Object: key, - } +func (d donutDriver) GetObject(target io.Writer, bucketName, objectName string) (int64, error) { + errParams := map[string]string{ + "bucketName": bucketName, + "objectName": objectName, } - return io.Copy(target, reader) + if bucketName == "" || strings.TrimSpace(bucketName) == "" { + return 0, iodine.New(errors.New("invalid argument"), errParams) + } + if objectName == "" || strings.TrimSpace(objectName) == "" { + return 0, iodine.New(errors.New("invalid argument"), errParams) + } + buckets, err := d.donut.ListBuckets() + if err != nil { + return 0, iodine.New(err, nil) + } + if _, ok := buckets[bucketName]; !ok { + return 0, iodine.New(errors.New("bucket does not exist"), errParams) + } + reader, size, err := buckets[bucketName].GetObject(objectName) + if os.IsNotExist(err) { + return 0, drivers.ObjectNotFound{ + Bucket: bucketName, + Object: objectName, + } + } else if err != nil { + return 0, iodine.New(err, errParams) + } + n, err := io.CopyN(target, reader, size) + return n, iodine.New(err, errParams) } // GetPartialObject retrieves an object range and writes it to a writer -func (d donutDriver) GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error) { +func (d donutDriver) GetPartialObject(w io.Writer, bucketName, objectName string, start, length int64) (int64, error) { // TODO more efficient get partial object with proper donut support errParams := map[string]string{ - "bucket": bucket, - "object": object, - "start": strconv.FormatInt(start, 10), - "length": strconv.FormatInt(length, 10), + "bucketName": bucketName, + "objectName": objectName, + "start": strconv.FormatInt(start, 10), + "length": strconv.FormatInt(length, 10), } - reader, err := d.donut.GetObjectReader(bucket, object) + + if bucketName == "" || strings.TrimSpace(bucketName) == "" { + return 0, iodine.New(errors.New("invalid argument"), errParams) + } + if objectName == "" || strings.TrimSpace(objectName) == "" { + return 0, iodine.New(errors.New("invalid argument"), errParams) + } + if start < 0 { + return 0, iodine.New(errors.New("invalid argument"), errParams) + } + buckets, err := d.donut.ListBuckets() if err != nil { + return 0, iodine.New(err, nil) + } + if _, ok := buckets[bucketName]; !ok { + return 0, iodine.New(errors.New("bucket does not exist"), errParams) + } + reader, size, err := buckets[bucketName].GetObject(objectName) + if os.IsNotExist(err) { + return 0, drivers.ObjectNotFound{ + Bucket: bucketName, + Object: objectName, + } + } else if err != nil { return 0, iodine.New(err, errParams) } + if start > size || (start+length-1) > size { + return 0, iodine.New(errors.New("invalid range"), errParams) + } _, err = io.CopyN(ioutil.Discard, reader, start) if err != nil { return 0, iodine.New(err, errParams) @@ -134,156 +216,160 @@ func (d donutDriver) GetPartialObject(w io.Writer, bucket, object string, start, } // GetObjectMetadata retrieves an object's metadata -func (d donutDriver) GetObjectMetadata(bucket, key string, prefix string) (drivers.ObjectMetadata, error) { - metadata, err := d.donut.GetObjectMetadata(bucket, key) +func (d donutDriver) GetObjectMetadata(bucketName, objectName, prefixName string) (drivers.ObjectMetadata, error) { + errParams := map[string]string{ + "bucketName": bucketName, + "objectName": objectName, + "prefixName": prefixName, + } + buckets, err := d.donut.ListBuckets() if err != nil { + return drivers.ObjectMetadata{}, iodine.New(err, errParams) + } + if _, ok := buckets[bucketName]; !ok { + return drivers.ObjectMetadata{}, iodine.New(errors.New("bucket does not exist"), errParams) + } + objectList, err := buckets[bucketName].ListObjects() + if err != nil { + return drivers.ObjectMetadata{}, iodine.New(err, errParams) + } + donutObjectMetadata, err := objectList[objectName].GetDonutObjectMetadata() + if os.IsNotExist(err) { // return ObjectNotFound quickly on an error, API needs this to handle invalid requests return drivers.ObjectMetadata{}, drivers.ObjectNotFound{ - Bucket: bucket, - Object: key, + Bucket: bucketName, + Object: objectName, } + } else if err != nil { + return drivers.ObjectMetadata{}, iodine.New(err, errParams) } - created, err := time.Parse(time.RFC3339Nano, metadata["sys.created"]) + objectMetadata, err := objectList[objectName].GetObjectMetadata() + if os.IsNotExist(err) { + // return ObjectNotFound quickly on an error, API needs this to handle invalid requests + return drivers.ObjectMetadata{}, drivers.ObjectNotFound{ + Bucket: bucketName, + Object: objectName, + } + } else if err != nil { + return drivers.ObjectMetadata{}, iodine.New(err, errParams) + } + created, err := time.Parse(time.RFC3339Nano, donutObjectMetadata["created"]) if err != nil { - return drivers.ObjectMetadata{}, err + return drivers.ObjectMetadata{}, iodine.New(err, nil) } - size, err := strconv.ParseInt(metadata["sys.size"], 10, 64) + size, err := strconv.ParseInt(donutObjectMetadata["size"], 10, 64) if err != nil { - return drivers.ObjectMetadata{}, err + return drivers.ObjectMetadata{}, iodine.New(err, nil) } - objectMetadata := drivers.ObjectMetadata{ - Bucket: bucket, - Key: key, + driversObjectMetadata := drivers.ObjectMetadata{ + Bucket: bucketName, + Key: objectName, - ContentType: metadata["contentType"], + ContentType: objectMetadata["contentType"], Created: created, - Md5: metadata["sys.md5"], + Md5: donutObjectMetadata["md5"], Size: size, } - return objectMetadata, nil + return driversObjectMetadata, nil } +type byObjectKey []drivers.ObjectMetadata + +func (b byObjectKey) Len() int { return len(b) } +func (b byObjectKey) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b byObjectKey) Less(i, j int) bool { return b[i].Key < b[j].Key } + // ListObjects - returns list of objects -func (d donutDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) { - // TODO Fix IsPrefixSet && IsDelimiterSet and use them - objects, err := d.donut.ListObjects(bucket) +func (d donutDriver) ListObjects(bucketName string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) { + errParams := map[string]string{ + "bucketName": bucketName, + } + buckets, err := d.donut.ListBuckets() if err != nil { - return nil, drivers.BucketResourcesMetadata{}, err + return nil, drivers.BucketResourcesMetadata{}, iodine.New(err, errParams) + } + if _, ok := buckets[bucketName]; !ok { + return nil, drivers.BucketResourcesMetadata{}, iodine.New(errors.New("bucket does not exist"), errParams) + } + objectList, err := buckets[bucketName].ListObjects() + if err != nil { + return nil, drivers.BucketResourcesMetadata{}, iodine.New(err, errParams) + } + var objects []string + for key := range objectList { + objects = append(objects, key) } sort.Strings(objects) - if resources.Prefix != "" { - objects = filterPrefix(objects, resources.Prefix) - objects = removePrefix(objects, resources.Prefix) - } + if resources.Maxkeys <= 0 || resources.Maxkeys > 1000 { resources.Maxkeys = 1000 } - - var actualObjects []string - var commonPrefixes []string - if strings.TrimSpace(resources.Delimiter) != "" { - actualObjects = filterDelimited(objects, resources.Delimiter) - commonPrefixes = filterNotDelimited(objects, resources.Delimiter) - commonPrefixes = extractDir(commonPrefixes, resources.Delimiter) - commonPrefixes = uniqueObjects(commonPrefixes) - resources.CommonPrefixes = commonPrefixes - } else { - actualObjects = objects - } + // Populate filtering mode + resources.Mode = drivers.GetMode(resources) + // filter objects based on resources.Prefix and resources.Delimiter + actualObjects, commonPrefixes := d.filter(objects, resources) + resources.CommonPrefixes = commonPrefixes var results []drivers.ObjectMetadata - for _, object := range actualObjects { + for _, objectName := range actualObjects { if len(results) >= resources.Maxkeys { resources.IsTruncated = true break } - metadata, err := d.GetObjectMetadata(bucket, resources.Prefix+object, "") + if _, ok := objectList[objectName]; !ok { + return nil, drivers.BucketResourcesMetadata{}, iodine.New(errors.New("object corrupted"), errParams) + } + objectMetadata, err := objectList[objectName].GetDonutObjectMetadata() if err != nil { - return nil, drivers.BucketResourcesMetadata{}, err + return nil, drivers.BucketResourcesMetadata{}, iodine.New(err, errParams) + } + t, err := time.Parse(time.RFC3339Nano, objectMetadata["created"]) + if err != nil { + return nil, drivers.BucketResourcesMetadata{}, iodine.New(err, nil) + } + size, err := strconv.ParseInt(objectMetadata["size"], 10, 64) + if err != nil { + return nil, drivers.BucketResourcesMetadata{}, iodine.New(err, nil) + } + metadata := drivers.ObjectMetadata{ + Key: objectName, + Created: t, + Size: size, } results = append(results, metadata) } + sort.Sort(byObjectKey(results)) return results, resources, nil } -func filterPrefix(objects []string, prefix string) []string { - var results []string - for _, object := range objects { - if strings.HasPrefix(object, prefix) { - results = append(results, object) - } - } - return results -} - -func removePrefix(objects []string, prefix string) []string { - var results []string - for _, object := range objects { - results = append(results, strings.TrimPrefix(object, prefix)) - } - return results -} - -func filterDelimited(objects []string, delim string) []string { - var results []string - for _, object := range objects { - if !strings.Contains(object, delim) { - results = append(results, object) - } - } - return results -} -func filterNotDelimited(objects []string, delim string) []string { - var results []string - for _, object := range objects { - if strings.Contains(object, delim) { - results = append(results, object) - } - } - return results -} - -func extractDir(objects []string, delim string) []string { - var results []string - for _, object := range objects { - parts := strings.Split(object, delim) - results = append(results, parts[0]+"/") - } - return results -} - -func uniqueObjects(objects []string) []string { - objectMap := make(map[string]string) - for _, v := range objects { - objectMap[v] = v - } - var results []string - for k := range objectMap { - results = append(results, k) - } - sort.Strings(results) - return results -} - // CreateObject creates a new object -func (d donutDriver) CreateObject(bucketKey, objectKey, contentType, expectedMd5sum string, reader io.Reader) error { - writer, err := d.donut.GetObjectWriter(bucketKey, objectKey) - if err != nil { - return err +func (d donutDriver) CreateObject(bucketName, objectName, contentType, expectedMd5sum string, reader io.Reader) error { + errParams := map[string]string{ + "bucketName": bucketName, + "objectName": objectName, + "contentType": contentType, } - if _, err := io.Copy(writer, reader); err != nil { - return err + if bucketName == "" || strings.TrimSpace(bucketName) == "" { + return iodine.New(errors.New("invalid argument"), errParams) + } + if objectName == "" || strings.TrimSpace(objectName) == "" { + return iodine.New(errors.New("invalid argument"), errParams) + } + buckets, err := d.donut.ListBuckets() + if err != nil { + return iodine.New(err, errParams) + } + if _, ok := buckets[bucketName]; !ok { + return iodine.New(errors.New("bucket does not exist"), errParams) } if contentType == "" { contentType = "application/octet-stream" } contentType = strings.TrimSpace(contentType) - metadata := make(map[string]string) - metadata["bucket"] = bucketKey - metadata["object"] = objectKey - metadata["contentType"] = contentType - if err = writer.SetMetadata(metadata); err != nil { - return err + err = buckets[bucketName].PutObject(objectName, contentType, reader) + if err != nil { + return iodine.New(err, errParams) } - return writer.Close() + // handle expectedMd5sum + return nil } diff --git a/pkg/drivers/donut/donut_filter.go b/pkg/drivers/donut/donut_filter.go new file mode 100644 index 000000000..396fb0aa5 --- /dev/null +++ b/pkg/drivers/donut/donut_filter.go @@ -0,0 +1,68 @@ +package donut + +import ( + "bufio" + "bytes" + "strings" + + "github.com/minio-io/minio/pkg/drivers" +) + +func delimiter(object, delimiter string) string { + readBuffer := bytes.NewBufferString(object) + reader := bufio.NewReader(readBuffer) + stringReader := strings.NewReader(delimiter) + delimited, _ := stringReader.ReadByte() + delimitedStr, _ := reader.ReadString(delimited) + return delimitedStr +} + +func appendUniq(slice []string, i string) []string { + for _, ele := range slice { + if ele == i { + return slice + } + } + return append(slice, i) +} + +func (d donutDriver) filter(objects []string, resources drivers.BucketResourcesMetadata) ([]string, []string) { + var actualObjects []string + var commonPrefixes []string + for _, name := range objects { + switch true { + // Both delimiter and Prefix is present + case resources.IsDelimiterPrefixSet(): + if strings.HasPrefix(name, resources.Prefix) { + trimmedName := strings.TrimPrefix(name, resources.Prefix) + delimitedName := delimiter(trimmedName, resources.Delimiter) + if delimitedName != "" { + if delimitedName == resources.Delimiter { + commonPrefixes = appendUniq(commonPrefixes, resources.Prefix+delimitedName) + } else { + commonPrefixes = appendUniq(commonPrefixes, delimitedName) + } + if trimmedName == delimitedName { + actualObjects = appendUniq(actualObjects, name) + } + } + } + // Delimiter present and Prefix is absent + case resources.IsDelimiterSet(): + delimitedName := delimiter(name, resources.Delimiter) + switch true { + case delimitedName == name: + actualObjects = appendUniq(actualObjects, name) + case delimitedName != "": + commonPrefixes = appendUniq(commonPrefixes, delimitedName) + } + case resources.IsPrefixSet(): + if strings.HasPrefix(name, resources.Prefix) { + actualObjects = appendUniq(actualObjects, name) + } + case resources.IsDefault(): + return objects, nil + } + } + return actualObjects, commonPrefixes +} diff --git a/pkg/server/server.go b/pkg/server/server.go index b4ee1b286..c1852e2e6 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -17,6 +17,7 @@ package server import ( + "fmt" "os/user" "path" "reflect" @@ -187,7 +188,8 @@ func Start(configs []Config) { for _, ch := range ctrlChans { close(ch) } - log.Fatal(value.Interface()) + msg := fmt.Sprintf("%q", value.Interface()) + log.Fatal(iodine.New(errors.New(msg), nil)) } case false: // Channel closed, remove from list diff --git a/pkg/storage/donut/bucket.go b/pkg/storage/donut/bucket.go deleted file mode 100644 index f81e464b0..000000000 --- a/pkg/storage/donut/bucket.go +++ /dev/null @@ -1,42 +0,0 @@ -package donut - -import ( - "errors" - "strconv" - "strings" - - "github.com/minio-io/iodine" -) - -type donutBucket struct { - nodes []string - objects map[string][]byte -} - -// GetNodes - get list of associated nodes for a given bucket -func (b donutBucket) GetNodes() ([]string, error) { - var nodes []string - for _, node := range b.nodes { - nodes = append(nodes, node) - } - return nodes, nil -} - -// AddNode - adds a node to a bucket -func (b donutBucket) AddNode(nodeID, bucketID string) error { - errParams := map[string]string{"node": nodeID, "bucketID": bucketID} - tokens := strings.Split(bucketID, ":") - if len(tokens) != 3 { - return iodine.New(errors.New("Bucket ID malformeD: "+bucketID), errParams) - - } - // bucketName := tokens[0] - // aggregate := tokens[1] - // aggregate := "0" - part, err := strconv.Atoi(tokens[2]) - if err != nil { - return iodine.New(errors.New("Part malformed: "+tokens[2]), errParams) - } - b.nodes[part] = nodeID - return nil -} diff --git a/pkg/storage/donut/donut.go b/pkg/storage/donut/donut.go deleted file mode 100644 index 510ea95cf..000000000 --- a/pkg/storage/donut/donut.go +++ /dev/null @@ -1,196 +0,0 @@ -package donut - -import ( - "errors" - "io" - "sort" - "strconv" - "strings" - - "github.com/minio-io/iodine" -) - -type donut struct { - buckets map[string]Bucket - nodes map[string]Node -} - -// NewDonut - instantiate new donut driver -func NewDonut(root string) (Donut, error) { - nodes := make(map[string]Node) - nodes["localhost"] = &localDirectoryNode{root: root} - driver := &donut{ - buckets: make(map[string]Bucket), - nodes: nodes, - } - for nodeID, node := range nodes { - bucketIDs, err := node.GetBuckets() - if err != nil { - return nil, iodine.New(err, map[string]string{"root": root}) - } - for _, bucketID := range bucketIDs { - tokens := strings.Split(bucketID, ":") - if _, ok := driver.buckets[tokens[0]]; !ok { - bucket := donutBucket{ - nodes: make([]string, 16), - } - // TODO catch errors - driver.buckets[tokens[0]] = bucket - } - if err = driver.buckets[tokens[0]].AddNode(nodeID, bucketID); err != nil { - return nil, iodine.New(err, map[string]string{"root": root}) - } - } - } - return driver, nil -} - -// CreateBucket - create a new bucket -func (d donut) CreateBucket(bucketName string) error { - if _, ok := d.buckets[bucketName]; ok == false { - bucketName = strings.TrimSpace(bucketName) - if bucketName == "" { - return iodine.New(errors.New("Cannot create bucket with no name"), map[string]string{"bucket": bucketName}) - } - // assign nodes - // TODO assign other nodes - nodes := make([]string, 16) - for i := 0; i < 16; i++ { - nodes[i] = "localhost" - if node, ok := d.nodes["localhost"]; ok { - err := node.CreateBucket(bucketName + ":0:" + strconv.Itoa(i)) - if err != nil { - return iodine.New(err, map[string]string{"node": nodes[i], "bucket": bucketName}) - } - } - } - bucket := donutBucket{ - nodes: nodes, - } - d.buckets[bucketName] = bucket - return nil - } - return iodine.New(errors.New("Bucket exists"), map[string]string{"bucket": bucketName}) -} - -// ListBuckets - list all buckets -func (d donut) ListBuckets() ([]string, error) { - var buckets []string - for bucket := range d.buckets { - buckets = append(buckets, bucket) - } - sort.Strings(buckets) - return buckets, nil -} - -// GetObjectWriter - get a new writer interface for a new object -func (d donut) GetObjectWriter(bucketName, objectName string) (ObjectWriter, error) { - if bucket, ok := d.buckets[bucketName]; ok == true { - writers := make([]Writer, 16) - nodes, err := bucket.GetNodes() - if err != nil { - return nil, iodine.New(err, map[string]string{"bucket": bucketName, "object": objectName}) - } - for i, nodeID := range nodes { - if node, ok := d.nodes[nodeID]; ok == true { - bucketID := bucketName + ":0:" + strconv.Itoa(i) - writer, err := node.GetWriter(bucketID, objectName) - if err != nil { - for _, writerToClose := range writers { - if writerToClose != nil { - writerToClose.CloseWithError(iodine.New(err, nil)) - } - } - return nil, iodine.New(err, map[string]string{"bucketid": bucketID}) - } - writers[i] = writer - } - } - return newErasureWriter(writers), nil - } - return nil, iodine.New(errors.New("Bucket not found"), map[string]string{"bucket": bucketName}) -} - -// GetObjectReader - get a new reader interface for a new object -func (d donut) GetObjectReader(bucketName, objectName string) (io.ReadCloser, error) { - errParams := map[string]string{"bucket": bucketName, "object": objectName} - r, w := io.Pipe() - if bucket, ok := d.buckets[bucketName]; ok == true { - readers := make([]io.ReadCloser, 16) - nodes, err := bucket.GetNodes() - if err != nil { - return nil, iodine.New(err, errParams) - } - var metadata map[string]string - for i, nodeID := range nodes { - if node, ok := d.nodes[nodeID]; ok == true { - bucketID := bucketName + ":0:" + strconv.Itoa(i) - reader, err := node.GetReader(bucketID, objectName) - if err != nil { - errParams["node"] = nodeID - return nil, iodine.New(err, errParams) - } - readers[i] = reader - if metadata == nil { - metadata, err = node.GetDonutMetadata(bucketID, objectName) - if err != nil { - errParams["node"] = nodeID - return nil, iodine.New(err, errParams) - } - } - } - } - go erasureReader(readers, metadata, w) - return r, nil - } - return nil, iodine.New(errors.New("Bucket not found"), errParams) -} - -// GetObjectMetadata returns metadata for a given object in a bucket -func (d donut) GetObjectMetadata(bucketName, object string) (map[string]string, error) { - errParams := map[string]string{"bucket": bucketName, "object": object} - if bucket, ok := d.buckets[bucketName]; ok { - nodes, err := bucket.GetNodes() - if err != nil { - return nil, iodine.New(err, errParams) - } - if node, ok := d.nodes[nodes[0]]; ok { - bucketID := bucketName + ":0:0" - metadata, err := node.GetMetadata(bucketID, object) - if err != nil { - errParams["bucketID"] = bucketID - return nil, iodine.New(err, errParams) - } - donutMetadata, err := node.GetDonutMetadata(bucketID, object) - if err != nil { - errParams["bucketID"] = bucketID - return nil, iodine.New(err, errParams) - } - metadata["sys.created"] = donutMetadata["created"] - metadata["sys.md5"] = donutMetadata["md5"] - metadata["sys.size"] = donutMetadata["size"] - return metadata, nil - } - errParams["node"] = nodes[0] - return nil, iodine.New(errors.New("Cannot connect to node: "+nodes[0]), errParams) - } - return nil, errors.New("Bucket not found") -} - -// ListObjects - list all the available objects in a bucket -func (d donut) ListObjects(bucketName string) ([]string, error) { - errParams := map[string]string{"bucket": bucketName} - if bucket, ok := d.buckets[bucketName]; ok { - nodes, err := bucket.GetNodes() - if err != nil { - return nil, iodine.New(err, errParams) - } - if node, ok := d.nodes[nodes[0]]; ok { - bucketID := bucketName + ":0:0" - objects, err := node.ListObjects(bucketID) - errParams["bucketID"] = bucketID - return objects, iodine.New(err, errParams) - } - } - return nil, iodine.New(errors.New("Bucket not found"), errParams) -} diff --git a/pkg/storage/donut/donut_test.go b/pkg/storage/donut/donut_test.go deleted file mode 100644 index 0f982b79f..000000000 --- a/pkg/storage/donut/donut_test.go +++ /dev/null @@ -1,240 +0,0 @@ -package donut - -import ( - "bytes" - "io" - "io/ioutil" - "os" - "testing" - "time" - - . "github.com/minio-io/check" -) - -func Test(t *testing.T) { TestingT(t) } - -type MySuite struct{} - -var _ = Suite(&MySuite{}) - -func (s *MySuite) TestEmptyBucket(c *C) { - root, err := ioutil.TempDir(os.TempDir(), "donut-") - c.Assert(err, IsNil) - defer os.RemoveAll(root) - donut, err := NewDonut(root) - c.Assert(err, IsNil) - - // check buckets are empty - buckets, err := donut.ListBuckets() - c.Assert(err, IsNil) - c.Assert(buckets, IsNil) -} - -func (s *MySuite) TestBucketWithoutNameFails(c *C) { - root, err := ioutil.TempDir(os.TempDir(), "donut-") - c.Assert(err, IsNil) - defer os.RemoveAll(root) - donut, err := NewDonut(root) - c.Assert(err, IsNil) - // fail to create new bucket without a name - err = donut.CreateBucket("") - c.Assert(err, Not(IsNil)) - - err = donut.CreateBucket(" ") - c.Assert(err, Not(IsNil)) -} - -func (s *MySuite) TestCreateBucketAndList(c *C) { - root, err := ioutil.TempDir(os.TempDir(), "donut-") - c.Assert(err, IsNil) - defer os.RemoveAll(root) - donut, err := NewDonut(root) - c.Assert(err, IsNil) - // create bucket - err = donut.CreateBucket("foo") - c.Assert(err, IsNil) - - // check bucket exists - buckets, err := donut.ListBuckets() - c.Assert(err, IsNil) - c.Assert(buckets, DeepEquals, []string{"foo"}) -} - -func (s *MySuite) TestCreateBucketWithSameNameFails(c *C) { - root, err := ioutil.TempDir(os.TempDir(), "donut-") - c.Assert(err, IsNil) - defer os.RemoveAll(root) - donut, err := NewDonut(root) - c.Assert(err, IsNil) - err = donut.CreateBucket("foo") - c.Assert(err, IsNil) - - err = donut.CreateBucket("foo") - c.Assert(err, Not(IsNil)) -} - -func (s *MySuite) TestCreateMultipleBucketsAndList(c *C) { - root, err := ioutil.TempDir(os.TempDir(), "donut-") - c.Assert(err, IsNil) - defer os.RemoveAll(root) - donut, err := NewDonut(root) - c.Assert(err, IsNil) - // add a second bucket - err = donut.CreateBucket("foo") - c.Assert(err, IsNil) - - err = donut.CreateBucket("bar") - c.Assert(err, IsNil) - - buckets, err := donut.ListBuckets() - c.Assert(err, IsNil) - c.Assert(buckets, DeepEquals, []string{"bar", "foo"}) - - err = donut.CreateBucket("foobar") - c.Assert(err, IsNil) - - buckets, err = donut.ListBuckets() - c.Assert(err, IsNil) - c.Assert(buckets, DeepEquals, []string{"bar", "foo", "foobar"}) -} - -func (s *MySuite) TestNewObjectFailsWithoutBucket(c *C) { - root, err := ioutil.TempDir(os.TempDir(), "donut-") - c.Assert(err, IsNil) - defer os.RemoveAll(root) - donut, err := NewDonut(root) - c.Assert(err, IsNil) - - writer, err := donut.GetObjectWriter("foo", "obj") - c.Assert(err, Not(IsNil)) - c.Assert(writer, IsNil) -} - -func (s *MySuite) TestNewObjectFailsWithEmptyName(c *C) { - root, err := ioutil.TempDir(os.TempDir(), "donut-") - c.Assert(err, IsNil) - defer os.RemoveAll(root) - donut, err := NewDonut(root) - c.Assert(err, IsNil) - - writer, err := donut.GetObjectWriter("foo", "") - c.Assert(err, Not(IsNil)) - c.Assert(writer, IsNil) - - writer, err = donut.GetObjectWriter("foo", " ") - c.Assert(err, Not(IsNil)) - c.Assert(writer, IsNil) -} - -func (s *MySuite) TestNewObjectCanBeWritten(c *C) { - root, err := ioutil.TempDir(os.TempDir(), "donut-") - c.Assert(err, IsNil) - defer os.RemoveAll(root) - donut, err := NewDonut(root) - c.Assert(err, IsNil) - - err = donut.CreateBucket("foo") - c.Assert(err, IsNil) - - writer, err := donut.GetObjectWriter("foo", "obj") - c.Assert(err, IsNil) - - data := "Hello World" - length, err := writer.Write([]byte(data)) - c.Assert(length, Equals, len(data)) - - expectedMetadata := map[string]string{ - "foo": "bar", - "created": "one", - "hello": "world", - } - - err = writer.SetMetadata(expectedMetadata) - c.Assert(err, IsNil) - - err = writer.Close() - c.Assert(err, IsNil) - - actualWriterMetadata, err := writer.GetMetadata() - c.Assert(err, IsNil) - c.Assert(actualWriterMetadata, DeepEquals, expectedMetadata) - - c.Assert(err, IsNil) - - reader, err := donut.GetObjectReader("foo", "obj") - c.Assert(err, IsNil) - - var actualData bytes.Buffer - _, err = io.Copy(&actualData, reader) - c.Assert(err, IsNil) - c.Assert(actualData.Bytes(), DeepEquals, []byte(data)) - - actualMetadata, err := donut.GetObjectMetadata("foo", "obj") - c.Assert(err, IsNil) - expectedMetadata["sys.md5"] = "b10a8db164e0754105b7a99be72e3fe5" - expectedMetadata["sys.size"] = "11" - _, err = time.Parse(time.RFC3339Nano, actualMetadata["sys.created"]) - c.Assert(err, IsNil) - expectedMetadata["sys.created"] = actualMetadata["sys.created"] - c.Assert(actualMetadata, DeepEquals, expectedMetadata) -} - -func (s *MySuite) TestMultipleNewObjects(c *C) { - root, err := ioutil.TempDir(os.TempDir(), "donut-") - c.Assert(err, IsNil) - defer os.RemoveAll(root) - donut, err := NewDonut(root) - c.Assert(err, IsNil) - - c.Assert(donut.CreateBucket("foo"), IsNil) - writer, err := donut.GetObjectWriter("foo", "obj1") - c.Assert(err, IsNil) - writer.Write([]byte("one")) - writer.Close() - - writer, err = donut.GetObjectWriter("foo", "obj2") - c.Assert(err, IsNil) - writer.Write([]byte("two")) - writer.Close() - - // c.Skip("not complete") - - reader, err := donut.GetObjectReader("foo", "obj1") - c.Assert(err, IsNil) - var readerBuffer1 bytes.Buffer - _, err = io.Copy(&readerBuffer1, reader) - c.Assert(err, IsNil) - // c.Skip("Not Implemented") - c.Assert(readerBuffer1.Bytes(), DeepEquals, []byte("one")) - - reader, err = donut.GetObjectReader("foo", "obj2") - c.Assert(err, IsNil) - var readerBuffer2 bytes.Buffer - _, err = io.Copy(&readerBuffer2, reader) - c.Assert(err, IsNil) - c.Assert(readerBuffer2.Bytes(), DeepEquals, []byte("two")) - - // test list objects - listObjects, err := donut.ListObjects("foo") - c.Assert(err, IsNil) - c.Assert(listObjects, DeepEquals, []string{"obj1", "obj2"}) -} - -func (s *MySuite) TestSysPrefixShouldFail(c *C) { - root, err := ioutil.TempDir(os.TempDir(), "donut-") - c.Assert(err, IsNil) - defer os.RemoveAll(root) - donut, err := NewDonut(root) - c.Assert(err, IsNil) - - c.Assert(donut.CreateBucket("foo"), IsNil) - writer, err := donut.GetObjectWriter("foo", "obj1") - c.Assert(err, IsNil) - writer.Write([]byte("one")) - metadata := make(map[string]string) - metadata["foo"] = "bar" - metadata["sys.hello"] = "world" - err = writer.SetMetadata(metadata) - c.Assert(err, Not(IsNil)) - writer.Close() -} diff --git a/pkg/storage/donut/erasure.go b/pkg/storage/donut/erasure.go deleted file mode 100644 index f23ed844f..000000000 --- a/pkg/storage/donut/erasure.go +++ /dev/null @@ -1,245 +0,0 @@ -package donut - -import ( - "bytes" - "errors" - "hash" - "io" - "strconv" - "strings" - "time" - - "crypto/md5" - "encoding/hex" - - encoding "github.com/minio-io/erasure" - "github.com/minio-io/iodine" - "github.com/minio-io/minio/pkg/utils/split" -) - -// getErasureTechnique - convert technique string into Technique type -func getErasureTechnique(technique string) (encoding.Technique, error) { - switch true { - case technique == "Cauchy": - return encoding.Cauchy, nil - case technique == "Vandermonde": - return encoding.Cauchy, nil - default: - return encoding.None, iodine.New(errors.New("Invalid erasure technique: "+technique), nil) - } -} - -// erasureReader - returns aligned streaming reads over a PipeWriter -func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, writer *io.PipeWriter) { - totalChunks, err := strconv.Atoi(donutMetadata["chunkCount"]) - if err != nil { - writer.CloseWithError(iodine.New(err, donutMetadata)) - return - } - totalLeft, err := strconv.ParseInt(donutMetadata["size"], 10, 64) - if err != nil { - writer.CloseWithError(iodine.New(err, donutMetadata)) - return - } - blockSize, err := strconv.Atoi(donutMetadata["blockSize"]) - if err != nil { - writer.CloseWithError(iodine.New(err, donutMetadata)) - return - } - parsedk, err := strconv.ParseUint(donutMetadata["erasureK"], 10, 8) - if err != nil { - writer.CloseWithError(iodine.New(err, donutMetadata)) - return - } - k := uint8(parsedk) - parsedm, err := strconv.ParseUint(donutMetadata["erasureM"], 10, 8) - if err != nil { - writer.CloseWithError(iodine.New(err, donutMetadata)) - return - } - m := uint8(parsedm) - expectedMd5sum, err := hex.DecodeString(donutMetadata["md5"]) - if err != nil { - writer.CloseWithError(iodine.New(err, donutMetadata)) - return - } - technique, err := getErasureTechnique(donutMetadata["erasureTechnique"]) - if err != nil { - writer.CloseWithError(iodine.New(err, donutMetadata)) - return - } - hasher := md5.New() - params, err := encoding.ValidateParams(k, m, technique) - if err != nil { - writer.CloseWithError(iodine.New(err, donutMetadata)) - } - encoder := encoding.NewErasure(params) - for i := 0; i < totalChunks; i++ { - totalLeft, err = decodeChunk(writer, readers, encoder, hasher, k, m, totalLeft, blockSize) - if err != nil { - errParams := map[string]string{ - "totalLeft": strconv.FormatInt(totalLeft, 10), - } - for k, v := range donutMetadata { - errParams[k] = v - } - writer.CloseWithError(iodine.New(err, errParams)) - } - } - actualMd5sum := hasher.Sum(nil) - if bytes.Compare(expectedMd5sum, actualMd5sum) != 0 { - writer.CloseWithError(iodine.New(errors.New("decoded md5sum did not match. expected: "+string(expectedMd5sum)+" actual: "+string(actualMd5sum)), donutMetadata)) - return - } - writer.Close() - return -} - -func decodeChunk(writer *io.PipeWriter, readers []io.ReadCloser, encoder *encoding.Erasure, hasher hash.Hash, k, m uint8, totalLeft int64, blockSize int) (int64, error) { - curBlockSize := 0 - if int64(blockSize) < totalLeft { - curBlockSize = blockSize - } else { - curBlockSize = int(totalLeft) // cast is safe, blockSize in if protects - } - - curChunkSize := encoding.GetEncodedBlockLen(curBlockSize, uint8(k)) - encodedBytes := make([][]byte, 16) - for i, reader := range readers { - var bytesBuffer bytes.Buffer - written, err := io.CopyN(&bytesBuffer, reader, int64(curChunkSize)) - if err != nil { - errParams := map[string]string{} - errParams["part"] = strconv.FormatInt(written, 10) - errParams["block.written"] = strconv.FormatInt(written, 10) - errParams["block.length"] = strconv.Itoa(curChunkSize) - return totalLeft, iodine.New(err, errParams) - } - encodedBytes[i] = bytesBuffer.Bytes() - } - decodedData, err := encoder.Decode(encodedBytes, curBlockSize) - if err != nil { - errParams := map[string]string{} - errParams["block.length"] = strconv.Itoa(curChunkSize) - return totalLeft, iodine.New(err, errParams) - } - _, err = hasher.Write(decodedData) // not expecting errors from hash, will also catch further down on .Sum mismatch in parent - if err != nil { - errParams := map[string]string{} - errParams["block.length"] = strconv.Itoa(curChunkSize) - return totalLeft, iodine.New(err, errParams) - } - _, err = io.Copy(writer, bytes.NewBuffer(decodedData)) - if err != nil { - errParams := map[string]string{} - errParams["block.length"] = strconv.Itoa(curChunkSize) - return totalLeft, iodine.New(err, errParams) - } - totalLeft = totalLeft - int64(blockSize) - return totalLeft, nil -} - -// erasure writer - -type erasureWriter struct { - writers []Writer - metadata map[string]string - donutMetadata map[string]string // not exposed - erasureWriter *io.PipeWriter - isClosed <-chan bool -} - -// newErasureWriter - get a new writer -func newErasureWriter(writers []Writer) ObjectWriter { - r, w := io.Pipe() - isClosed := make(chan bool) - writer := erasureWriter{ - writers: writers, - metadata: make(map[string]string), - erasureWriter: w, - isClosed: isClosed, - } - go erasureGoroutine(r, writer, isClosed) - return writer -} - -func erasureGoroutine(r *io.PipeReader, eWriter erasureWriter, isClosed chan<- bool) { - chunks := split.Stream(r, 10*1024*1024) - params, _ := encoding.ValidateParams(8, 8, encoding.Cauchy) - encoder := encoding.NewErasure(params) - chunkCount := 0 - totalLength := 0 - summer := md5.New() - for chunk := range chunks { - if chunk.Err == nil { - totalLength = totalLength + len(chunk.Data) - encodedBlocks, _ := encoder.Encode(chunk.Data) - summer.Write(chunk.Data) - for blockIndex, block := range encodedBlocks { - io.Copy(eWriter.writers[blockIndex], bytes.NewBuffer(block)) - } - } - chunkCount = chunkCount + 1 - } - dataMd5sum := summer.Sum(nil) - metadata := make(map[string]string) - metadata["blockSize"] = strconv.Itoa(10 * 1024 * 1024) - metadata["chunkCount"] = strconv.Itoa(chunkCount) - metadata["created"] = time.Now().Format(time.RFC3339Nano) - metadata["erasureK"] = "8" - metadata["erasureM"] = "8" - metadata["erasureTechnique"] = "Cauchy" - metadata["md5"] = hex.EncodeToString(dataMd5sum) - metadata["size"] = strconv.Itoa(totalLength) - for _, nodeWriter := range eWriter.writers { - if nodeWriter != nil { - nodeWriter.SetMetadata(eWriter.metadata) - nodeWriter.SetDonutMetadata(metadata) - nodeWriter.Close() - } - } - isClosed <- true -} - -func (eWriter erasureWriter) Write(data []byte) (int, error) { - io.Copy(eWriter.erasureWriter, bytes.NewBuffer(data)) - return len(data), nil -} - -func (eWriter erasureWriter) Close() error { - eWriter.erasureWriter.Close() - <-eWriter.isClosed - return nil -} - -func (eWriter erasureWriter) CloseWithError(err error) error { - for _, writer := range eWriter.writers { - if writer != nil { - writer.CloseWithError(err) - } - } - return nil -} - -func (eWriter erasureWriter) SetMetadata(metadata map[string]string) error { - for k := range metadata { - if strings.HasPrefix(k, "sys.") { - return errors.New("Invalid key '" + k + "', cannot start with sys.'") - } - } - for k := range eWriter.metadata { - delete(eWriter.metadata, k) - } - for k, v := range metadata { - eWriter.metadata[k] = v - } - return nil -} - -func (eWriter erasureWriter) GetMetadata() (map[string]string, error) { - metadata := make(map[string]string) - for k, v := range eWriter.metadata { - metadata[k] = v - } - return metadata, nil -} diff --git a/pkg/storage/donut/interfaces.go b/pkg/storage/donut/interfaces.go deleted file mode 100644 index dcd822e45..000000000 --- a/pkg/storage/donut/interfaces.go +++ /dev/null @@ -1,51 +0,0 @@ -package donut - -import ( - "io" -) - -// Collection of Donut specification interfaces - -// Donut interface -type Donut interface { - CreateBucket(bucket string) error - GetObjectReader(bucket, object string) (io.ReadCloser, error) - GetObjectWriter(bucket, object string) (ObjectWriter, error) - GetObjectMetadata(bucket, object string) (map[string]string, error) - ListBuckets() ([]string, error) - ListObjects(bucket string) ([]string, error) -} - -// Bucket interface -type Bucket interface { - GetNodes() ([]string, error) - AddNode(nodeID, bucketID string) error -} - -// Node interface -type Node interface { - CreateBucket(bucket string) error - GetBuckets() ([]string, error) - GetDonutMetadata(bucket, object string) (map[string]string, error) - GetMetadata(bucket, object string) (map[string]string, error) - GetReader(bucket, object string) (io.ReadCloser, error) - GetWriter(bucket, object string) (Writer, error) - ListObjects(bucket string) ([]string, error) -} - -// ObjectWriter interface -type ObjectWriter interface { - Close() error - CloseWithError(error) error - GetMetadata() (map[string]string, error) - SetMetadata(map[string]string) error - Write([]byte) (int, error) -} - -// Writer interface -type Writer interface { - ObjectWriter - - GetDonutMetadata() (map[string]string, error) - SetDonutMetadata(map[string]string) error -} diff --git a/pkg/storage/donut/node_local.go b/pkg/storage/donut/node_local.go deleted file mode 100644 index 0427917fc..000000000 --- a/pkg/storage/donut/node_local.go +++ /dev/null @@ -1,100 +0,0 @@ -package donut - -import ( - "io" - "os" - "path" - "sort" - "strings" - - "encoding/json" - "io/ioutil" - "path/filepath" - - "github.com/minio-io/iodine" -) - -type localDirectoryNode struct { - root string -} - -func (d localDirectoryNode) CreateBucket(bucket string) error { - objectPath := path.Join(d.root, bucket) - return iodine.New(os.MkdirAll(objectPath, 0700), map[string]string{"bucket": bucket}) -} - -func (d localDirectoryNode) GetBuckets() ([]string, error) { - files, err := ioutil.ReadDir(d.root) - if err != nil { - return nil, iodine.New(err, nil) - } - var results []string - for _, file := range files { - if file.IsDir() { - results = append(results, file.Name()) - } - } - return results, nil -} - -func (d localDirectoryNode) GetWriter(bucket, object string) (Writer, error) { - errParams := map[string]string{"bucket": bucket, "object": object} - objectPath := path.Join(d.root, bucket, object) - err := os.MkdirAll(objectPath, 0700) - if err != nil { - return nil, iodine.New(err, errParams) - } - writer, err := newDonutObjectWriter(objectPath) - return writer, iodine.New(err, errParams) -} - -func (d localDirectoryNode) GetReader(bucket, object string) (io.ReadCloser, error) { - reader, err := os.Open(path.Join(d.root, bucket, object, "data")) - return reader, iodine.New(err, map[string]string{"bucket": bucket, "object": object}) -} - -func (d localDirectoryNode) GetMetadata(bucket, object string) (map[string]string, error) { - m, err := d.getMetadata(bucket, object, "metadata.json") - return m, iodine.New(err, map[string]string{"bucket": bucket, "object": object}) -} -func (d localDirectoryNode) GetDonutMetadata(bucket, object string) (map[string]string, error) { - m, err := d.getMetadata(bucket, object, "donutMetadata.json") - return m, iodine.New(err, map[string]string{"bucket": bucket, "object": object}) -} - -func (d localDirectoryNode) getMetadata(bucket, object, fileName string) (map[string]string, error) { - errParams := map[string]string{"bucket": bucket, "object": object, "file": fileName} - file, err := os.Open(path.Join(d.root, bucket, object, fileName)) - defer file.Close() - if err != nil { - return nil, iodine.New(err, errParams) - } - metadata := make(map[string]string) - decoder := json.NewDecoder(file) - if err := decoder.Decode(&metadata); err != nil { - return nil, iodine.New(err, errParams) - } - return metadata, nil - -} - -func (d localDirectoryNode) ListObjects(bucketName string) ([]string, error) { - errParams := map[string]string{"bucket": bucketName} - prefix := path.Join(d.root, bucketName) - var objects []string - if err := filepath.Walk(prefix, func(path string, info os.FileInfo, err error) error { - if err != nil { - return iodine.New(err, errParams) - } - if !info.IsDir() && strings.HasSuffix(path, "data") { - object := strings.TrimPrefix(path, prefix+"/") - object = strings.TrimSuffix(object, "/data") - objects = append(objects, object) - } - return nil - }); err != nil { - return nil, iodine.New(err, errParams) - } - sort.Strings(objects) - return objects, nil -} diff --git a/pkg/storage/donut/object_writer.go b/pkg/storage/donut/object_writer.go deleted file mode 100644 index 7f506673b..000000000 --- a/pkg/storage/donut/object_writer.go +++ /dev/null @@ -1,91 +0,0 @@ -package donut - -import ( - "encoding/json" - "io/ioutil" - "os" - "path" - - "github.com/minio-io/iodine" -) - -func newDonutObjectWriter(objectDir string) (Writer, error) { - dataFile, err := os.OpenFile(path.Join(objectDir, "data"), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) - if err != nil { - return nil, iodine.New(err, map[string]string{"objectDir": objectDir}) - } - return donutObjectWriter{ - root: objectDir, - file: dataFile, - metadata: make(map[string]string), - donutMetadata: make(map[string]string), - }, nil -} - -type donutObjectWriter struct { - root string - file *os.File - metadata map[string]string - donutMetadata map[string]string - err error -} - -func (d donutObjectWriter) Write(data []byte) (int, error) { - written, err := d.file.Write(data) - return written, iodine.New(err, nil) -} - -func (d donutObjectWriter) Close() error { - if d.err != nil { - return iodine.New(d.err, nil) - } - metadata, _ := json.Marshal(d.metadata) - ioutil.WriteFile(path.Join(d.root, "metadata.json"), metadata, 0600) - donutMetadata, _ := json.Marshal(d.donutMetadata) - ioutil.WriteFile(path.Join(d.root, "donutMetadata.json"), donutMetadata, 0600) - - return iodine.New(d.file.Close(), nil) -} - -func (d donutObjectWriter) CloseWithError(err error) error { - if d.err != nil { - d.err = err - } - return iodine.New(d.Close(), nil) -} - -func (d donutObjectWriter) SetMetadata(metadata map[string]string) error { - for k := range d.metadata { - delete(d.metadata, k) - } - for k, v := range metadata { - d.metadata[k] = v - } - return nil -} - -func (d donutObjectWriter) GetMetadata() (map[string]string, error) { - metadata := make(map[string]string) - for k, v := range d.metadata { - metadata[k] = v - } - return metadata, nil -} - -func (d donutObjectWriter) SetDonutMetadata(metadata map[string]string) error { - for k := range d.donutMetadata { - delete(d.donutMetadata, k) - } - for k, v := range metadata { - d.donutMetadata[k] = v - } - return nil -} - -func (d donutObjectWriter) GetDonutMetadata() (map[string]string, error) { - donutMetadata := make(map[string]string) - for k, v := range d.donutMetadata { - donutMetadata[k] = v - } - return donutMetadata, nil -} From d324c4b061c101db88922e8f09d9b57b0565ab06 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sun, 5 Apr 2015 15:52:31 -0700 Subject: [PATCH 2/2] Return proper return for all drivers when BucketNotFound --- main.go | 6 +----- pkg/api/api_test.go | 5 +++-- pkg/drivers/donut/donut.go | 38 ++++++++++++++++----------------- pkg/drivers/file/file_object.go | 6 ++++-- pkg/drivers/memory/memory.go | 7 ++++++ 5 files changed, 34 insertions(+), 28 deletions(-) diff --git a/main.go b/main.go index 0d5451a38..1908c0bd1 100644 --- a/main.go +++ b/main.go @@ -201,9 +201,5 @@ func main() { } return nil } - err := app.Run(os.Args) - if err != nil { - log.Error.Println(err) - } - + app.RunAndExitOnError() } diff --git a/pkg/api/api_test.go b/pkg/api/api_test.go index ea659cb01..c83a6f92e 100644 --- a/pkg/api/api_test.go +++ b/pkg/api/api_test.go @@ -379,14 +379,15 @@ func (s *MySuite) TestHeader(c *C) { testServer := httptest.NewServer(httpHandler) defer testServer.Close() + typedDriver.On("CreateBucket", "bucket").Return(nil).Once() + driver.CreateBucket("bucket") + typedDriver.On("GetObjectMetadata", "bucket", "object", "").Return(drivers.ObjectMetadata{}, drivers.ObjectNotFound{}).Once() response, err := http.Get(testServer.URL + "/bucket/object") c.Assert(err, IsNil) verifyError(c, response, "NoSuchKey", "The specified key does not exist.", http.StatusNotFound) buffer := bytes.NewBufferString("hello world") - typedDriver.On("CreateBucket", "bucket").Return(nil).Once() - driver.CreateBucket("bucket") typedDriver.On("CreateObject", "bucket", "object", "", "", mock.Anything).Return(nil).Once() driver.CreateObject("bucket", "object", "", "", buffer) diff --git a/pkg/drivers/donut/donut.go b/pkg/drivers/donut/donut.go index 5497df602..d29ca2f33 100644 --- a/pkg/drivers/donut/donut.go +++ b/pkg/drivers/donut/donut.go @@ -154,16 +154,14 @@ func (d donutDriver) GetObject(target io.Writer, bucketName, objectName string) return 0, iodine.New(err, nil) } if _, ok := buckets[bucketName]; !ok { - return 0, iodine.New(errors.New("bucket does not exist"), errParams) + return 0, drivers.BucketNotFound{Bucket: bucketName} } reader, size, err := buckets[bucketName].GetObject(objectName) - if os.IsNotExist(err) { + if err != nil { return 0, drivers.ObjectNotFound{ Bucket: bucketName, Object: objectName, } - } else if err != nil { - return 0, iodine.New(err, errParams) } n, err := io.CopyN(target, reader, size) return n, iodine.New(err, errParams) @@ -193,16 +191,14 @@ func (d donutDriver) GetPartialObject(w io.Writer, bucketName, objectName string return 0, iodine.New(err, nil) } if _, ok := buckets[bucketName]; !ok { - return 0, iodine.New(errors.New("bucket does not exist"), errParams) + return 0, drivers.BucketNotFound{Bucket: bucketName} } reader, size, err := buckets[bucketName].GetObject(objectName) - if os.IsNotExist(err) { + if err != nil { return 0, drivers.ObjectNotFound{ Bucket: bucketName, Object: objectName, } - } else if err != nil { - return 0, iodine.New(err, errParams) } if start > size || (start+length-1) > size { return 0, iodine.New(errors.New("invalid range"), errParams) @@ -227,31 +223,35 @@ func (d donutDriver) GetObjectMetadata(bucketName, objectName, prefixName string return drivers.ObjectMetadata{}, iodine.New(err, errParams) } if _, ok := buckets[bucketName]; !ok { - return drivers.ObjectMetadata{}, iodine.New(errors.New("bucket does not exist"), errParams) + return drivers.ObjectMetadata{}, drivers.BucketNotFound{Bucket: bucketName} } objectList, err := buckets[bucketName].ListObjects() if err != nil { return drivers.ObjectMetadata{}, iodine.New(err, errParams) } - donutObjectMetadata, err := objectList[objectName].GetDonutObjectMetadata() - if os.IsNotExist(err) { + object, ok := objectList[objectName] + if !ok { // return ObjectNotFound quickly on an error, API needs this to handle invalid requests return drivers.ObjectMetadata{}, drivers.ObjectNotFound{ Bucket: bucketName, Object: objectName, } - } else if err != nil { - return drivers.ObjectMetadata{}, iodine.New(err, errParams) } - objectMetadata, err := objectList[objectName].GetObjectMetadata() - if os.IsNotExist(err) { + donutObjectMetadata, err := object.GetDonutObjectMetadata() + if err != nil { + // return ObjectNotFound quickly on an error, API needs this to handle invalid requests + return drivers.ObjectMetadata{}, drivers.ObjectNotFound{ + Bucket: bucketName, + Object: objectName, + } + } + objectMetadata, err := object.GetObjectMetadata() + if err != nil { // return ObjectNotFound quickly on an error, API needs this to handle invalid requests return drivers.ObjectMetadata{}, drivers.ObjectNotFound{ Bucket: bucketName, Object: objectName, } - } else if err != nil { - return drivers.ObjectMetadata{}, iodine.New(err, errParams) } created, err := time.Parse(time.RFC3339Nano, donutObjectMetadata["created"]) if err != nil { @@ -289,7 +289,7 @@ func (d donutDriver) ListObjects(bucketName string, resources drivers.BucketReso return nil, drivers.BucketResourcesMetadata{}, iodine.New(err, errParams) } if _, ok := buckets[bucketName]; !ok { - return nil, drivers.BucketResourcesMetadata{}, iodine.New(errors.New("bucket does not exist"), errParams) + return nil, drivers.BucketResourcesMetadata{}, drivers.BucketNotFound{Bucket: bucketName} } objectList, err := buckets[bucketName].ListObjects() if err != nil { @@ -360,7 +360,7 @@ func (d donutDriver) CreateObject(bucketName, objectName, contentType, expectedM return iodine.New(err, errParams) } if _, ok := buckets[bucketName]; !ok { - return iodine.New(errors.New("bucket does not exist"), errParams) + return drivers.BucketNotFound{Bucket: bucketName} } if contentType == "" { contentType = "application/octet-stream" diff --git a/pkg/drivers/file/file_object.go b/pkg/drivers/file/file_object.go index 02537b51e..bd70fdf47 100644 --- a/pkg/drivers/file/file_object.go +++ b/pkg/drivers/file/file_object.go @@ -134,11 +134,13 @@ func (file *fileDriver) GetObjectMetadata(bucket, object, prefix string) (driver if drivers.IsValidBucket(bucket) == false { return drivers.ObjectMetadata{}, drivers.BucketNameInvalid{Bucket: bucket} } - if drivers.IsValidObject(object) == false { return drivers.ObjectMetadata{}, drivers.ObjectNameInvalid{Bucket: bucket, Object: bucket} } - + // check bucket exists + if _, err := os.Stat(path.Join(file.root, bucket)); os.IsNotExist(err) { + return drivers.ObjectMetadata{}, drivers.BucketNotFound{Bucket: bucket} + } // Do not use path.Join() since path.Join strips off any object names with '/', use them as is // in a static manner so that we can send a proper 'ObjectNotFound' reply back upon os.Stat() objectPath := file.root + "/" + bucket + "/" + object diff --git a/pkg/drivers/memory/memory.go b/pkg/drivers/memory/memory.go index 7ec382ce0..bc66f924c 100644 --- a/pkg/drivers/memory/memory.go +++ b/pkg/drivers/memory/memory.go @@ -71,6 +71,9 @@ func start(ctrlChannel <-chan string, errorChannel chan<- error) { // GetObject - GET object from memory buffer func (memory memoryDriver) GetObject(w io.Writer, bucket string, object string) (int64, error) { + if _, ok := memory.bucketdata[bucket]; ok == false { + return 0, drivers.BucketNotFound{Bucket: bucket} + } // get object key := object if val, ok := memory.objectdata[key]; ok { @@ -269,6 +272,10 @@ func (memory memoryDriver) ListBuckets() ([]drivers.BucketMetadata, error) { // GetObjectMetadata - get object metadata from memory func (memory memoryDriver) GetObjectMetadata(bucket, key, prefix string) (drivers.ObjectMetadata, error) { + // check if bucket exists + if _, ok := memory.bucketdata[bucket]; ok == false { + return drivers.ObjectMetadata{}, drivers.BucketNotFound{Bucket: bucket} + } if object, ok := memory.objectdata[key]; ok == true { return object.metadata, nil }