mirror of
https://github.com/minio/minio.git
synced 2024-12-25 06:35:56 -05:00
Erasure layer now writes using new technique
This commit is contained in:
Frederick F. Kautz IV
parent
15552a59f9
commit
856e0100c0
98
pkg/storage/donut/erasure/erasure1/erasure.go
Normal file
98
pkg/storage/donut/erasure/erasure1/erasure.go
Normal file
@ -0,0 +1,98 @@
|
||||
/*
|
||||
* Mini Object Storage, (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package erasure1
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"encoding/gob"
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
// DataHeader represents the structure serialized to gob.
|
||||
type DataHeader struct {
|
||||
// object + block stored
|
||||
Key string
|
||||
// chunk index of encoded block
|
||||
ChunkIndex uint8
|
||||
// Original Length of the block output
|
||||
OriginalLength uint32
|
||||
// Data Blocks
|
||||
EncoderK uint8
|
||||
// Parity Blocks
|
||||
EncoderM uint8
|
||||
// Matrix Technique
|
||||
EncoderTechnique EncoderTechnique
|
||||
}
|
||||
|
||||
// EncoderTechnique specified the Matrix type used in encoding
|
||||
type EncoderTechnique int
|
||||
|
||||
const (
|
||||
// Vandermonde matrix type
|
||||
Vandermonde EncoderTechnique = iota
|
||||
// Cauchy matrix type
|
||||
Cauchy
|
||||
)
|
||||
|
||||
// validate populated header
|
||||
func validateHeader(header DataHeader) error {
|
||||
if header.Key == "" {
|
||||
return errors.New("Empty Key")
|
||||
}
|
||||
|
||||
if header.EncoderTechnique > 1 {
|
||||
return errors.New("Invalid encoder technique")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteData returns error upon any failure
|
||||
func Write(target io.Writer, key string, part uint8, length uint32, k, m uint8, technique EncoderTechnique, data io.Reader) error {
|
||||
header := DataHeader{
|
||||
Key: key,
|
||||
ChunkIndex: part,
|
||||
OriginalLength: length,
|
||||
EncoderK: k,
|
||||
EncoderM: m,
|
||||
EncoderTechnique: technique,
|
||||
}
|
||||
|
||||
if err := validateHeader(header); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var headerBuffer bytes.Buffer
|
||||
// encode header
|
||||
encoder := gob.NewEncoder(&headerBuffer)
|
||||
encoder.Encode(header)
|
||||
|
||||
// write version
|
||||
binary.Write(target, binary.LittleEndian, uint32(1))
|
||||
|
||||
// write encoded header
|
||||
if _, err := io.Copy(target, &headerBuffer); err != nil {
|
||||
return err
|
||||
}
|
||||
// write data
|
||||
if _, err := io.Copy(target, data); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -14,7 +14,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package erasure
|
||||
package erasure1
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@ -35,19 +35,16 @@ var _ = Suite(&MySuite{})
|
||||
func (s *MySuite) TestSingleWrite(c *C) {
|
||||
var testBuffer bytes.Buffer
|
||||
testData := "Hello, World"
|
||||
encoderParams := EncoderParams{
|
||||
Length: uint32(len(testData)),
|
||||
K: 8,
|
||||
M: 8,
|
||||
Technique: Cauchy,
|
||||
testHeader := DataHeader{
|
||||
Key: "testobj",
|
||||
ChunkIndex: 1,
|
||||
OriginalLength: uint32(len(testData)),
|
||||
EncoderK: 8,
|
||||
EncoderM: 8,
|
||||
EncoderTechnique: Cauchy,
|
||||
}
|
||||
metadata := make(Metadata)
|
||||
metadata["Content-Type"] = "application/octet-stream"
|
||||
metadata["Content-MD5"] = "testing"
|
||||
|
||||
header := NewHeader("testobj", 1, metadata, encoderParams)
|
||||
|
||||
err := WriteData(&testBuffer, header, bytes.NewBufferString(testData))
|
||||
err := Write(&testBuffer, testHeader.Key, testHeader.ChunkIndex, testHeader.OriginalLength, testHeader.EncoderK, testHeader.EncoderM, testHeader.EncoderTechnique, bytes.NewBufferString(testData))
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
actualVersion := make([]byte, 4)
|
||||
@ -60,7 +57,7 @@ func (s *MySuite) TestSingleWrite(c *C) {
|
||||
decoder := gob.NewDecoder(&testBuffer)
|
||||
decoder.Decode(&actualHeader)
|
||||
|
||||
c.Assert(actualHeader, DeepEquals, header)
|
||||
c.Assert(actualHeader, DeepEquals, testHeader)
|
||||
|
||||
var actualData bytes.Buffer
|
||||
|
||||
@ -1,106 +0,0 @@
|
||||
/*
|
||||
* Mini Object Storage, (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package erasure
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"encoding/gob"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// Metadata map
|
||||
type Metadata map[string]string
|
||||
|
||||
// DataHeader struct
|
||||
type DataHeader struct {
|
||||
Key string
|
||||
Part uint8
|
||||
Metadata Metadata
|
||||
EncoderParams EncoderParams
|
||||
}
|
||||
|
||||
// EncoderTechnique type
|
||||
type EncoderTechnique int
|
||||
|
||||
// EncoderTechniques
|
||||
const (
|
||||
Vandermonde EncoderTechnique = iota
|
||||
Cauchy
|
||||
)
|
||||
|
||||
// EncoderParams struct
|
||||
type EncoderParams struct {
|
||||
Length uint32
|
||||
K uint8
|
||||
M uint8
|
||||
Technique EncoderTechnique
|
||||
}
|
||||
|
||||
// NewHeader populate new header
|
||||
func NewHeader(key string, part uint8, metadata Metadata, encoderParams EncoderParams) DataHeader {
|
||||
header := DataHeader{}
|
||||
header.Key = key
|
||||
header.Part = part
|
||||
header.Metadata = metadata
|
||||
header.EncoderParams = EncoderParams{
|
||||
Length: encoderParams.Length,
|
||||
K: encoderParams.K,
|
||||
M: encoderParams.M,
|
||||
Technique: encoderParams.Technique,
|
||||
}
|
||||
return header
|
||||
}
|
||||
|
||||
// ValidateHeader validate populated header
|
||||
func ValidateHeader(header DataHeader) bool {
|
||||
if header.Key == "" || header.Part < 0 || len(header.Metadata) < 2 {
|
||||
return false
|
||||
}
|
||||
|
||||
if header.EncoderParams.Length < 0 || header.EncoderParams.Technique > 1 {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// WriteData write data, returns error upon any failure
|
||||
func WriteData(target io.Writer, header DataHeader, data io.Reader) error {
|
||||
if !ValidateHeader(header) {
|
||||
return fmt.Errorf("Invalid header")
|
||||
}
|
||||
|
||||
var headerBuffer bytes.Buffer
|
||||
// encode header
|
||||
encoder := gob.NewEncoder(&headerBuffer)
|
||||
encoder.Encode(header)
|
||||
|
||||
// write version
|
||||
binary.Write(target, binary.LittleEndian, uint32(1))
|
||||
|
||||
// write encoded header
|
||||
if _, err := io.Copy(target, &headerBuffer); err != nil {
|
||||
return err
|
||||
}
|
||||
// write data
|
||||
if _, err := io.Copy(target, data); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user