MyFavMirrors
/
minio
mirror of https://github.com/minio/minio.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #676 from harshavardhana/pr_out_add_new_metadata_definitions_and_use_them_wip

This commit is contained in:
Harshavardhana 2015-06-26 20:38:02 +00:00
parent 0cb3f76a91 767d3743ee
commit e3d8a9e0f1
4 changed files with 159 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

163
pkg/storage/donut/bucket.go
View File

@ -33,6 +33,7 @@ import (
"encoding/json" "encoding/json"
"github.com/minio/minio/pkg/iodine" "github.com/minio/minio/pkg/iodine"
"github.com/minio/minio/pkg/utils/crypto/sha512"
"github.com/minio/minio/pkg/utils/split" "github.com/minio/minio/pkg/utils/split"
) )
@ -72,6 +73,10 @@ func newBucket(bucketName, aclType, donutName string, nodes map[string]node) (bu
return b, bucketMetadata, nil return b, bucketMetadata, nil
} }
func (b bucket) getBucketName() string {
return b.name
}
func (b bucket) getObjectName(fileName, diskPath, bucketPath string) (string, error) { func (b bucket) getObjectName(fileName, diskPath, bucketPath string) (string, error) {
newObject, err := newObject(fileName, filepath.Join(diskPath, bucketPath)) newObject, err := newObject(fileName, filepath.Join(diskPath, bucketPath))
if err != nil { if err != nil {
@ -81,15 +86,14 @@ func (b bucket) getObjectName(fileName, diskPath, bucketPath string) (string, er
if err != nil { if err != nil {
return "", iodine.New(err, nil) return "", iodine.New(err, nil)
} }
objectName, ok := newObjectMetadata["object"] if newObjectMetadata.Object == "" {
if !ok {
return "", iodine.New(ObjectCorrupted{Object: newObject.name}, nil) return "", iodine.New(ObjectCorrupted{Object: newObject.name}, nil)
} }
b.objects[objectName] = newObject b.objects[newObjectMetadata.Object] = newObject
return objectName, nil return newObjectMetadata.Object, nil
} }
func (b bucket) GetObjectMetadata(objectName string) (map[string]string, error) { func (b bucket) GetObjectMetadata(objectName string) (*objectMetadata, error) {
return b.objects[objectName].GetObjectMetadata() return b.objects[objectName].GetObjectMetadata()
} }
@ -179,26 +183,14 @@ func (b bucket) ReadObject(objectName string) (reader io.ReadCloser, size int64,
if !ok { if !ok {
return nil, 0, iodine.New(ObjectNotFound{Object: objectName}, nil) return nil, 0, iodine.New(ObjectNotFound{Object: objectName}, nil)
} }
// verify if objectMetadata is readable, before we serve the request
objectMetadata, err := object.GetObjectMetadata()
if err != nil {
return nil, 0, iodine.New(err, nil)
}
if objectName == "" || writer == nil || len(objectMetadata) == 0 {
return nil, 0, iodine.New(InvalidArgument{}, nil)
}
size, err = strconv.ParseInt(objectMetadata["size"], 10, 64)
if err != nil {
return nil, 0, iodine.New(err, nil)
}
// verify if donutObjectMetadata is readable, before we server the request // verify if donutObjectMetadata is readable, before we server the request
donutObjectMetadata, err := object.GetDonutObjectMetadata() donutObjMetadata, err := object.GetDonutObjectMetadata()
if err != nil { if err != nil {
return nil, 0, iodine.New(err, nil) return nil, 0, iodine.New(err, nil)
} }
// read and reply back to GetObject() request in a go-routine // read and reply back to GetObject() request in a go-routine
go b.readEncodedData(b.normalizeObjectName(objectName), writer, donutObjectMetadata) go b.readEncodedData(b.normalizeObjectName(objectName), writer, donutObjMetadata)
return reader, size, nil return reader, donutObjMetadata.Size, nil
} }
// WriteObject - write a new object into bucket // WriteObject - write a new object into bucket
@ -212,11 +204,13 @@ func (b bucket) WriteObject(objectName string, objectData io.Reader, expectedMD5
if err != nil { if err != nil {
return "", iodine.New(err, nil) return "", iodine.New(err, nil)
} }
summer := md5.New() sumMD5 := md5.New()
objectMetadata := make(map[string]string) sum512 := sha512.New()
donutObjectMetadata := make(map[string]string)
objectMetadata["version"] = objectMetadataVersion objectMetadata := new(objectMetadata)
donutObjectMetadata["version"] = donutObjectMetadataVersion donutObjectMetadata := new(donutObjectMetadata)
objectMetadata.Version = objectMetadataVersion
donutObjectMetadata.Version = donutObjectMetadataVersion
size := metadata["contentLength"] size := metadata["contentLength"]
sizeInt, err := strconv.ParseInt(size, 10, 64) sizeInt, err := strconv.ParseInt(size, 10, 64)
if err != nil { if err != nil {
@ -226,13 +220,13 @@ func (b bucket) WriteObject(objectName string, objectData io.Reader, expectedMD5
// if total writers are only '1' do not compute erasure // if total writers are only '1' do not compute erasure
switch len(writers) == 1 { switch len(writers) == 1 {
case true: case true:
mw := io.MultiWriter(writers[0], summer) mw := io.MultiWriter(writers[0], sumMD5, sum512)
totalLength, err := io.CopyN(mw, objectData, sizeInt) totalLength, err := io.CopyN(mw, objectData, sizeInt)
if err != nil { if err != nil {
return "", iodine.New(err, nil) return "", iodine.New(err, nil)
} }
donutObjectMetadata["sys.size"] = strconv.FormatInt(totalLength, 10) donutObjectMetadata.Size = totalLength
objectMetadata["size"] = strconv.FormatInt(totalLength, 10) objectMetadata.Size = totalLength
case false: case false:
// calculate data and parity dictated by total number of writers // calculate data and parity dictated by total number of writers
k, m, err := b.getDataAndParity(len(writers)) k, m, err := b.getDataAndParity(len(writers))
@ -240,37 +234,39 @@ func (b bucket) WriteObject(objectName string, objectData io.Reader, expectedMD5
return "", iodine.New(err, nil) return "", iodine.New(err, nil)
} }
// encoded data with k, m and write // encoded data with k, m and write
chunkCount, totalLength, err := b.writeEncodedData(k, m, writers, objectData, summer) chunkCount, totalLength, err := b.writeEncodedData(k, m, writers, objectData, sumMD5, sum512)
if err != nil { if err != nil {
return "", iodine.New(err, nil) return "", iodine.New(err, nil)
} }
/// donutMetadata section /// donutMetadata section
donutObjectMetadata["sys.blockSize"] = strconv.Itoa(10 * 1024 * 1024) donutObjectMetadata.BlockSize = 10 * 1024 * 1024
donutObjectMetadata["sys.chunkCount"] = strconv.Itoa(chunkCount) donutObjectMetadata.ChunkCount = chunkCount
donutObjectMetadata["sys.erasureK"] = strconv.FormatUint(uint64(k), 10) donutObjectMetadata.DataDisks = k
donutObjectMetadata["sys.erasureM"] = strconv.FormatUint(uint64(m), 10) donutObjectMetadata.ParityDisks = m
donutObjectMetadata["sys.erasureTechnique"] = "Cauchy" donutObjectMetadata.ErasureTechnique = "Cauchy"
donutObjectMetadata["sys.size"] = strconv.Itoa(totalLength) donutObjectMetadata.Size = int64(totalLength)
// keep size inside objectMetadata as well for Object API requests // keep size inside objectMetadata as well for Object API requests
objectMetadata["size"] = strconv.Itoa(totalLength) objectMetadata.Size = int64(totalLength)
} }
objectMetadata["bucket"] = b.name objectMetadata.Bucket = b.getBucketName()
objectMetadata["object"] = objectName objectMetadata.Object = objectName
// store all user provided metadata objectMetadata.Metadata = metadata
for k, v := range metadata { dataMD5sum := sumMD5.Sum(nil)
objectMetadata[k] = v dataSHA512sum := sum512.Sum(nil)
} objectMetadata.Created = time.Now().UTC()
dataMd5sum := summer.Sum(nil)
objectMetadata["created"] = time.Now().UTC().Format(time.RFC3339Nano)
// keeping md5sum for the object in two different places // keeping md5sum for the object in two different places
// one for object storage and another is for internal use // one for object storage and another is for internal use
objectMetadata["md5"] = hex.EncodeToString(dataMd5sum) hexMD5Sum := hex.EncodeToString(dataMD5sum)
donutObjectMetadata["sys.md5"] = hex.EncodeToString(dataMd5sum) hex512Sum := hex.EncodeToString(dataSHA512sum)
objectMetadata.MD5Sum = hexMD5Sum
objectMetadata.SHA512Sum = hex512Sum
donutObjectMetadata.MD5Sum = hexMD5Sum
donutObjectMetadata.SHA512Sum = hex512Sum
// Verify if the written object is equal to what is expected, only if it is requested as such // Verify if the written object is equal to what is expected, only if it is requested as such
if strings.TrimSpace(expectedMD5Sum) != "" { if strings.TrimSpace(expectedMD5Sum) != "" {
if err := b.isMD5SumEqual(strings.TrimSpace(expectedMD5Sum), objectMetadata["md5"]); err != nil { if err := b.isMD5SumEqual(strings.TrimSpace(expectedMD5Sum), objectMetadata.MD5Sum); err != nil {
return "", iodine.New(err, nil) return "", iodine.New(err, nil)
} }
} }
@ -286,7 +282,7 @@ func (b bucket) WriteObject(objectName string, objectData io.Reader, expectedMD5
for _, writer := range writers { for _, writer := range writers {
writer.Close() writer.Close()
} }
return objectMetadata["md5"], nil return objectMetadata.MD5Sum, nil
} }
// isMD5SumEqual - returns error if md5sum mismatches, other its `nil` // isMD5SumEqual - returns error if md5sum mismatches, other its `nil`
@ -309,8 +305,8 @@ func (b bucket) isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) error {
} }
// writeObjectMetadata - write additional object metadata // writeObjectMetadata - write additional object metadata
func (b bucket) writeObjectMetadata(objectName string, objectMetadata map[string]string) error { func (b bucket) writeObjectMetadata(objectName string, objectMetadata *objectMetadata) error {
if len(objectMetadata) == 0 { if objectMetadata == nil {
return iodine.New(InvalidArgument{}, nil) return iodine.New(InvalidArgument{}, nil)
} }
objectMetadataWriters, err := b.getDiskWriters(objectName, objectMetadataConfig) objectMetadataWriters, err := b.getDiskWriters(objectName, objectMetadataConfig)
@ -330,20 +326,20 @@ func (b bucket) writeObjectMetadata(objectName string, objectMetadata map[string
} }
// writeDonutObjectMetadata - write donut related object metadata // writeDonutObjectMetadata - write donut related object metadata
func (b bucket) writeDonutObjectMetadata(objectName string, objectMetadata map[string]string) error { func (b bucket) writeDonutObjectMetadata(objectName string, donutObjectMetadata *donutObjectMetadata) error {
if len(objectMetadata) == 0 { if donutObjectMetadata == nil {
return iodine.New(InvalidArgument{}, nil) return iodine.New(InvalidArgument{}, nil)
} }
objectMetadataWriters, err := b.getDiskWriters(objectName, donutObjectMetadataConfig) donutObjectMetadataWriters, err := b.getDiskWriters(objectName, donutObjectMetadataConfig)
if err != nil { if err != nil {
return iodine.New(err, nil) return iodine.New(err, nil)
} }
for _, objectMetadataWriter := range objectMetadataWriters { for _, donutObjectMetadataWriter := range donutObjectMetadataWriters {
defer objectMetadataWriter.Close() defer donutObjectMetadataWriter.Close()
} }
for _, objectMetadataWriter := range objectMetadataWriters { for _, donutObjectMetadataWriter := range donutObjectMetadataWriters {
jenc := json.NewEncoder(objectMetadataWriter) jenc := json.NewEncoder(donutObjectMetadataWriter)
if err := jenc.Encode(objectMetadata); err != nil { if err := jenc.Encode(donutObjectMetadata); err != nil {
return iodine.New(err, nil) return iodine.New(err, nil)
} }
} }
@ -380,7 +376,7 @@ func (b bucket) getDataAndParity(totalWriters int) (k uint8, m uint8, err error)
} }
// writeEncodedData - // writeEncodedData -
func (b bucket) writeEncodedData(k, m uint8, writers []io.WriteCloser, objectData io.Reader, summer hash.Hash) (int, int, error) { func (b bucket) writeEncodedData(k, m uint8, writers []io.WriteCloser, objectData io.Reader, sumMD5, sum512 hash.Hash) (int, int, error) {
chunks := split.Stream(objectData, 10*1024*1024) chunks := split.Stream(objectData, 10*1024*1024)
encoder, err := newEncoder(k, m, "Cauchy") encoder, err := newEncoder(k, m, "Cauchy")
if err != nil { if err != nil {
@ -392,7 +388,8 @@ func (b bucket) writeEncodedData(k, m uint8, writers []io.WriteCloser, objectDat
if chunk.Err == nil { if chunk.Err == nil {
totalLength = totalLength + len(chunk.Data) totalLength = totalLength + len(chunk.Data)
encodedBlocks, _ := encoder.Encode(chunk.Data) encodedBlocks, _ := encoder.Encode(chunk.Data)
summer.Write(chunk.Data) sumMD5.Write(chunk.Data)
sum512.Write(chunk.Data)
for blockIndex, block := range encodedBlocks { for blockIndex, block := range encodedBlocks {
_, err := io.Copy(writers[blockIndex], bytes.NewBuffer(block)) _, err := io.Copy(writers[blockIndex], bytes.NewBuffer(block))
if err != nil { if err != nil {
@ -406,8 +403,8 @@ func (b bucket) writeEncodedData(k, m uint8, writers []io.WriteCloser, objectDat
} }
// readEncodedData - // readEncodedData -
func (b bucket) readEncodedData(objectName string, writer *io.PipeWriter, donutObjectMetadata map[string]string) { func (b bucket) readEncodedData(objectName string, writer *io.PipeWriter, donutObjMetadata *donutObjectMetadata) {
expectedMd5sum, err := hex.DecodeString(donutObjectMetadata["sys.md5"]) expectedMd5sum, err := hex.DecodeString(donutObjMetadata.MD5Sum)
if err != nil { if err != nil {
writer.CloseWithError(iodine.New(err, nil)) writer.CloseWithError(iodine.New(err, nil))
return return
@ -424,23 +421,18 @@ func (b bucket) readEncodedData(objectName string, writer *io.PipeWriter, donutO
mwriter := io.MultiWriter(writer, hasher) mwriter := io.MultiWriter(writer, hasher)
switch len(readers) == 1 { switch len(readers) == 1 {
case false: case false:
totalChunks, totalLeft, blockSize, k, m, err := b.donutMetadata2Values(donutObjectMetadata) if donutObjMetadata.ErasureTechnique == "" {
if err != nil {
writer.CloseWithError(iodine.New(err, nil))
return
}
technique, ok := donutObjectMetadata["sys.erasureTechnique"]
if !ok {
writer.CloseWithError(iodine.New(MissingErasureTechnique{}, nil)) writer.CloseWithError(iodine.New(MissingErasureTechnique{}, nil))
return return
} }
encoder, err := newEncoder(uint8(k), uint8(m), technique) encoder, err := newEncoder(donutObjMetadata.DataDisks, donutObjMetadata.ParityDisks, donutObjMetadata.ErasureTechnique)
if err != nil { if err != nil {
writer.CloseWithError(iodine.New(err, nil)) writer.CloseWithError(iodine.New(err, nil))
return return
} }
for i := 0; i < totalChunks; i++ { totalLeft := donutObjMetadata.Size
decodedData, err := b.decodeEncodedData(totalLeft, blockSize, readers, encoder, writer) for i := 0; i < donutObjMetadata.ChunkCount; i++ {
decodedData, err := b.decodeEncodedData(totalLeft, int64(donutObjMetadata.BlockSize), readers, encoder, writer)
if err != nil { if err != nil {
writer.CloseWithError(iodine.New(err, nil)) writer.CloseWithError(iodine.New(err, nil))
return return
@ -450,7 +442,7 @@ func (b bucket) readEncodedData(objectName string, writer *io.PipeWriter, donutO
writer.CloseWithError(iodine.New(err, nil)) writer.CloseWithError(iodine.New(err, nil))
return return
} }
totalLeft = totalLeft - int64(blockSize) totalLeft = totalLeft - int64(donutObjMetadata.BlockSize)
} }
case true: case true:
_, err := io.Copy(writer, readers[0]) _, err := io.Copy(writer, readers[0])
@ -496,31 +488,6 @@ func (b bucket) decodeEncodedData(totalLeft, blockSize int64, readers []io.ReadC
return decodedData, nil return decodedData, nil
} }
// donutMetadata2Values -
func (b bucket) donutMetadata2Values(donutObjectMetadata map[string]string) (totalChunks int, totalLeft, blockSize int64, k, m uint64, err error) {
totalChunks, err = strconv.Atoi(donutObjectMetadata["sys.chunkCount"])
if err != nil {
return 0, 0, 0, 0, 0, iodine.New(err, nil)
}
totalLeft, err = strconv.ParseInt(donutObjectMetadata["sys.size"], 10, 64)
if err != nil {
return 0, 0, 0, 0, 0, iodine.New(err, nil)
}
blockSize, err = strconv.ParseInt(donutObjectMetadata["sys.blockSize"], 10, 64)
if err != nil {
return 0, 0, 0, 0, 0, iodine.New(err, nil)
}
k, err = strconv.ParseUint(donutObjectMetadata["sys.erasureK"], 10, 8)
if err != nil {
return 0, 0, 0, 0, 0, iodine.New(err, nil)
}
m, err = strconv.ParseUint(donutObjectMetadata["sys.erasureM"], 10, 8)
if err != nil {
return 0, 0, 0, 0, 0, iodine.New(err, nil)
}
return totalChunks, totalLeft, blockSize, k, m, nil
}
// getDiskReaders - // getDiskReaders -
func (b bucket) getDiskReaders(objectName, objectMeta string) ([]io.ReadCloser, error) { func (b bucket) getDiskReaders(objectName, objectMeta string) ([]io.ReadCloser, error) {
var readers []io.ReadCloser var readers []io.ReadCloser

70
pkg/storage/donut/definitions.go Normal file
View File

@ -0,0 +1,70 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package donut
import "time"
// objectMetadata object specific metadata per object
type objectMetadata struct {
// version
Version string `json:"version"`
// object metadata
Size int64 `json:"size"`
Created time.Time `json:"created"`
Bucket string `json:"bucket"`
Object string `json:"object"`
// checksums
MD5Sum string `json:"md5sum"`
SHA512Sum string `json:"sha512sum"`
// additional metadata
Metadata map[string]string `json:"metadata"`
}
// donutObjectMetadata container for donut specific internal metadata per object
type donutObjectMetadata struct {
// version
Version string `json:"version"`
// erasure
DataDisks uint8 `json:"sys.erasureK"`
ParityDisks uint8 `json:"sys.erasureM"`
ErasureTechnique string `json:"sys.erasureTechnique"`
// object metadata
Size int64 `json:"sys.size"`
BlockSize int `json:"sys.blockSize"`
ChunkCount int `json:"sys.chunkCount"`
// checksums
MD5Sum string `json:"sys.md5sum"`
SHA512Sum string `json:"sys.sha512sum"`
}
// donutMetadata container for donut level metadata
type donutMetadata struct {
Version string `json:"version"`
}
// bucketMetadata container for bucket level metadata
type bucketMetadata struct {
Version string `json:"version"`
ACL string `json:"acl"`
Metadata map[string]string `json:"metadata"`
}

15
pkg/storage/donut/donut.go
View File

@ -25,6 +25,7 @@ import (
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
"time"
"github.com/minio/minio/pkg/iodine" "github.com/minio/minio/pkg/iodine"
) )
@ -256,7 +257,19 @@ func (dt donut) GetObjectMetadata(bucket, object string) (map[string]string, err
} }
for _, objectName := range objectList { for _, objectName := range objectList {
if objectName == object { if objectName == object {
return dt.buckets[bucket].GetObjectMetadata(object) objectMetadataMap := make(map[string]string)
objectMetadata, err := dt.buckets[bucket].GetObjectMetadata(object)
if err != nil {
return nil, iodine.New(err, nil)
}
objectMetadataMap["created"] = objectMetadata.Created.Format(time.RFC3339Nano)
objectMetadataMap["size"] = strconv.FormatInt(objectMetadata.Size, 10)
objectMetadataMap["md5"] = objectMetadata.MD5Sum
objectMetadataMap["version"] = objectMetadata.Version
for k, v := range objectMetadata.Metadata {
objectMetadataMap[k] = v
}
return objectMetadataMap, nil
} }
} }
return nil, iodine.New(ObjectNotFound{Object: object}, errParams) return nil, iodine.New(ObjectNotFound{Object: object}, errParams)

24
pkg/storage/donut/object.go
View File

@ -26,10 +26,8 @@ import (
// object internal struct // object internal struct
type object struct { type object struct {
name string name string
objectPath string objectPath string
objectMetadata map[string]string
donutObjectMetadata map[string]string
} }
// newObject - instantiate a new object // newObject - instantiate a new object
@ -43,28 +41,26 @@ func newObject(objectName, p string) (object, error) {
return o, nil return o, nil
} }
func (o object) GetObjectMetadata() (map[string]string, error) { func (o object) GetObjectMetadata() (*objectMetadata, error) {
objectMetadata := make(map[string]string) objectMetadata := new(objectMetadata)
objectMetadataBytes, err := ioutil.ReadFile(filepath.Join(o.objectPath, objectMetadataConfig)) objectMetadataBytes, err := ioutil.ReadFile(filepath.Join(o.objectPath, objectMetadataConfig))
if err != nil { if err != nil {
return nil, iodine.New(ObjectNotFound{Object: o.name}, nil) return nil, iodine.New(ObjectNotFound{Object: o.name}, nil)
} }
if err := json.Unmarshal(objectMetadataBytes, &objectMetadata); err != nil { if err := json.Unmarshal(objectMetadataBytes, objectMetadata); err != nil {
return nil, iodine.New(err, nil) return nil, iodine.New(err, nil)
} }
o.objectMetadata = objectMetadata return objectMetadata, nil
return o.objectMetadata, nil
} }
func (o object) GetDonutObjectMetadata() (map[string]string, error) { func (o object) GetDonutObjectMetadata() (*donutObjectMetadata, error) {
donutObjectMetadata := make(map[string]string) donutObjectMetadata := new(donutObjectMetadata)
donutObjectMetadataBytes, err := ioutil.ReadFile(filepath.Join(o.objectPath, donutObjectMetadataConfig)) donutObjectMetadataBytes, err := ioutil.ReadFile(filepath.Join(o.objectPath, donutObjectMetadataConfig))
if err != nil { if err != nil {
return nil, iodine.New(ObjectNotFound{Object: o.name}, nil) return nil, iodine.New(ObjectNotFound{Object: o.name}, nil)
} }
if err := json.Unmarshal(donutObjectMetadataBytes, &donutObjectMetadata); err != nil { if err := json.Unmarshal(donutObjectMetadataBytes, donutObjectMetadata); err != nil {
return nil, iodine.New(err, nil) return nil, iodine.New(err, nil)
} }
o.donutObjectMetadata = donutObjectMetadata return donutObjectMetadata, nil
return o.donutObjectMetadata, nil
} }