From 856e0100c07448a6711ed77b6a2cf3439a424892 Mon Sep 17 00:00:00 2001 From: "Frederick F. Kautz IV" Date: Fri, 6 Mar 2015 16:37:44 -0800 Subject: [PATCH] Erasure layer now writes using new technique --- pkg/storage/donut/erasure/erasure1/erasure.go | 98 ++++++++++++++++ .../{erasure_v1 => erasure1}/erasure_test.go | 23 ++-- .../donut/erasure/erasure_v1/erasure.go | 106 ------------------ 3 files changed, 108 insertions(+), 119 deletions(-) create mode 100644 pkg/storage/donut/erasure/erasure1/erasure.go rename pkg/storage/donut/erasure/{erasure_v1 => erasure1}/erasure_test.go (75%) delete mode 100644 pkg/storage/donut/erasure/erasure_v1/erasure.go diff --git a/pkg/storage/donut/erasure/erasure1/erasure.go b/pkg/storage/donut/erasure/erasure1/erasure.go new file mode 100644 index 000000000..b4719f94a --- /dev/null +++ b/pkg/storage/donut/erasure/erasure1/erasure.go @@ -0,0 +1,98 @@ +/* + * Mini Object Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package erasure1 + +import ( + "bytes" + "encoding/binary" + "encoding/gob" + "errors" + "io" +) + +// DataHeader represents the structure serialized to gob. +type DataHeader struct { + // object + block stored + Key string + // chunk index of encoded block + ChunkIndex uint8 + // Original Length of the block output + OriginalLength uint32 + // Data Blocks + EncoderK uint8 + // Parity Blocks + EncoderM uint8 + // Matrix Technique + EncoderTechnique EncoderTechnique +} + +// EncoderTechnique specified the Matrix type used in encoding +type EncoderTechnique int + +const ( + // Vandermonde matrix type + Vandermonde EncoderTechnique = iota + // Cauchy matrix type + Cauchy +) + +// validate populated header +func validateHeader(header DataHeader) error { + if header.Key == "" { + return errors.New("Empty Key") + } + + if header.EncoderTechnique > 1 { + return errors.New("Invalid encoder technique") + } + + return nil +} + +// WriteData returns error upon any failure +func Write(target io.Writer, key string, part uint8, length uint32, k, m uint8, technique EncoderTechnique, data io.Reader) error { + header := DataHeader{ + Key: key, + ChunkIndex: part, + OriginalLength: length, + EncoderK: k, + EncoderM: m, + EncoderTechnique: technique, + } + + if err := validateHeader(header); err != nil { + return err + } + + var headerBuffer bytes.Buffer + // encode header + encoder := gob.NewEncoder(&headerBuffer) + encoder.Encode(header) + + // write version + binary.Write(target, binary.LittleEndian, uint32(1)) + + // write encoded header + if _, err := io.Copy(target, &headerBuffer); err != nil { + return err + } + // write data + if _, err := io.Copy(target, data); err != nil { + return err + } + return nil +} diff --git a/pkg/storage/donut/erasure/erasure_v1/erasure_test.go b/pkg/storage/donut/erasure/erasure1/erasure_test.go similarity index 75% rename from pkg/storage/donut/erasure/erasure_v1/erasure_test.go rename to pkg/storage/donut/erasure/erasure1/erasure_test.go index a5bcce02a..8111aeab8 100644 --- a/pkg/storage/donut/erasure/erasure_v1/erasure_test.go +++ b/pkg/storage/donut/erasure/erasure1/erasure_test.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package erasure +package erasure1 import ( "bytes" @@ -35,19 +35,16 @@ var _ = Suite(&MySuite{}) func (s *MySuite) TestSingleWrite(c *C) { var testBuffer bytes.Buffer testData := "Hello, World" - encoderParams := EncoderParams{ - Length: uint32(len(testData)), - K: 8, - M: 8, - Technique: Cauchy, + testHeader := DataHeader{ + Key: "testobj", + ChunkIndex: 1, + OriginalLength: uint32(len(testData)), + EncoderK: 8, + EncoderM: 8, + EncoderTechnique: Cauchy, } - metadata := make(Metadata) - metadata["Content-Type"] = "application/octet-stream" - metadata["Content-MD5"] = "testing" - header := NewHeader("testobj", 1, metadata, encoderParams) - - err := WriteData(&testBuffer, header, bytes.NewBufferString(testData)) + err := Write(&testBuffer, testHeader.Key, testHeader.ChunkIndex, testHeader.OriginalLength, testHeader.EncoderK, testHeader.EncoderM, testHeader.EncoderTechnique, bytes.NewBufferString(testData)) c.Assert(err, IsNil) actualVersion := make([]byte, 4) @@ -60,7 +57,7 @@ func (s *MySuite) TestSingleWrite(c *C) { decoder := gob.NewDecoder(&testBuffer) decoder.Decode(&actualHeader) - c.Assert(actualHeader, DeepEquals, header) + c.Assert(actualHeader, DeepEquals, testHeader) var actualData bytes.Buffer diff --git a/pkg/storage/donut/erasure/erasure_v1/erasure.go b/pkg/storage/donut/erasure/erasure_v1/erasure.go deleted file mode 100644 index 16ba4ba4a..000000000 --- a/pkg/storage/donut/erasure/erasure_v1/erasure.go +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Mini Object Storage, (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package erasure - -import ( - "bytes" - "encoding/binary" - "encoding/gob" - "fmt" - "io" -) - -// Metadata map -type Metadata map[string]string - -// DataHeader struct -type DataHeader struct { - Key string - Part uint8 - Metadata Metadata - EncoderParams EncoderParams -} - -// EncoderTechnique type -type EncoderTechnique int - -// EncoderTechniques -const ( - Vandermonde EncoderTechnique = iota - Cauchy -) - -// EncoderParams struct -type EncoderParams struct { - Length uint32 - K uint8 - M uint8 - Technique EncoderTechnique -} - -// NewHeader populate new header -func NewHeader(key string, part uint8, metadata Metadata, encoderParams EncoderParams) DataHeader { - header := DataHeader{} - header.Key = key - header.Part = part - header.Metadata = metadata - header.EncoderParams = EncoderParams{ - Length: encoderParams.Length, - K: encoderParams.K, - M: encoderParams.M, - Technique: encoderParams.Technique, - } - return header -} - -// ValidateHeader validate populated header -func ValidateHeader(header DataHeader) bool { - if header.Key == "" || header.Part < 0 || len(header.Metadata) < 2 { - return false - } - - if header.EncoderParams.Length < 0 || header.EncoderParams.Technique > 1 { - return false - } - - return true -} - -// WriteData write data, returns error upon any failure -func WriteData(target io.Writer, header DataHeader, data io.Reader) error { - if !ValidateHeader(header) { - return fmt.Errorf("Invalid header") - } - - var headerBuffer bytes.Buffer - // encode header - encoder := gob.NewEncoder(&headerBuffer) - encoder.Encode(header) - - // write version - binary.Write(target, binary.LittleEndian, uint32(1)) - - // write encoded header - if _, err := io.Copy(target, &headerBuffer); err != nil { - return err - } - // write data - if _, err := io.Copy(target, data); err != nil { - return err - } - return nil -}