mirror of
https://github.com/minio/minio.git
synced 2024-12-25 06:35:56 -05:00
Removing old donut format
This commit is contained in:
Frederick F. Kautz IV
parent
2f4b92eb8d
commit
348c4d9a88
@ -1,38 +0,0 @@
|
||||
package erasure
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"github.com/minio-io/minio/pkg/storage/donut/erasure/erasure1"
|
||||
)
|
||||
|
||||
// EncoderTechnique - encoder matrix type
|
||||
type EncoderTechnique int
|
||||
|
||||
const (
|
||||
// Vandermonde matrix type
|
||||
Vandermonde EncoderTechnique = iota
|
||||
// Cauchy matrix type
|
||||
Cauchy
|
||||
)
|
||||
|
||||
// Write latest donut format
|
||||
func Write(target io.Writer, length uint32, k, m uint8, technique EncoderTechnique, data io.Reader) error {
|
||||
var versionedTechnique erasure1.EncoderTechnique
|
||||
switch {
|
||||
case technique == Vandermonde:
|
||||
versionedTechnique = erasure1.Vandermonde
|
||||
case technique == Cauchy:
|
||||
versionedTechnique = erasure1.Cauchy
|
||||
default:
|
||||
errors.New("Unknown encoder technique")
|
||||
}
|
||||
return erasure1.Write(target, length, k, m, versionedTechnique, data)
|
||||
}
|
||||
|
||||
// Read any donut format
|
||||
func Read(reader io.Reader) (io.Reader, error) {
|
||||
// when version2 is created, create a method in version2 that can transform version1 structure to version2
|
||||
return nil, errors.New("Not Implemented")
|
||||
}
|
||||
@ -1,108 +0,0 @@
|
||||
/*
|
||||
* Mini Object Storage, (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package erasure1
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"encoding/gob"
|
||||
"errors"
|
||||
"io"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// Version - format version
|
||||
const (
|
||||
Version = uint32(1)
|
||||
)
|
||||
|
||||
// DataHeader represents the structure serialized to gob.
|
||||
type DataHeader struct {
|
||||
OriginalLength uint32
|
||||
// Data Blocks
|
||||
EncoderK uint8
|
||||
// Parity Blocks
|
||||
EncoderM uint8
|
||||
// Matrix Technique
|
||||
EncoderTechnique EncoderTechnique
|
||||
}
|
||||
|
||||
// EncoderTechnique specified the Matrix type used in encoding
|
||||
type EncoderTechnique int
|
||||
|
||||
const (
|
||||
// Vandermonde matrix type
|
||||
Vandermonde EncoderTechnique = iota
|
||||
// Cauchy matrix type
|
||||
Cauchy
|
||||
)
|
||||
|
||||
// validate populated header
|
||||
func validateHeader(header DataHeader) error {
|
||||
if header.EncoderTechnique > 1 {
|
||||
return errors.New("Invalid encoder technique")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write returns error upon any failure
|
||||
func Write(target io.Writer, length uint32, k, m uint8, technique EncoderTechnique, data io.Reader) error {
|
||||
header := DataHeader{
|
||||
OriginalLength: length,
|
||||
EncoderK: k,
|
||||
EncoderM: m,
|
||||
EncoderTechnique: technique,
|
||||
}
|
||||
|
||||
if err := validateHeader(header); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var headerBuffer bytes.Buffer
|
||||
// encode header
|
||||
encoder := gob.NewEncoder(&headerBuffer)
|
||||
encoder.Encode(header)
|
||||
|
||||
// write version
|
||||
binary.Write(target, binary.LittleEndian, Version)
|
||||
|
||||
// write encoded header
|
||||
if _, err := io.Copy(target, &headerBuffer); err != nil {
|
||||
return err
|
||||
}
|
||||
// write data
|
||||
if _, err := io.Copy(target, data); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReadHeader - read an erasure block
|
||||
func ReadHeader(reader io.Reader) (dataHeader DataHeader, err error) {
|
||||
versionArray := make([]byte, 4)
|
||||
if err := binary.Read(reader, binary.LittleEndian, versionArray); err != nil {
|
||||
return dataHeader, err
|
||||
}
|
||||
version := binary.LittleEndian.Uint32(versionArray)
|
||||
if version > Version || version == 0 {
|
||||
return dataHeader, errors.New("Unknown version: " + strconv.FormatUint(uint64(version), 10))
|
||||
}
|
||||
decoder := gob.NewDecoder(reader)
|
||||
err = decoder.Decode(&dataHeader)
|
||||
return dataHeader, err
|
||||
}
|
||||
@ -1,86 +0,0 @@
|
||||
/*
|
||||
* Mini Object Storage, (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package erasure1
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"encoding/gob"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
. "gopkg.in/check.v1"
|
||||
)
|
||||
|
||||
func Test(t *testing.T) { TestingT(t) }
|
||||
|
||||
type MySuite struct{}
|
||||
|
||||
var _ = Suite(&MySuite{})
|
||||
|
||||
func (s *MySuite) TestSingleWrite(c *C) {
|
||||
var testBuffer bytes.Buffer
|
||||
testData := "Hello, World"
|
||||
testHeader := DataHeader{
|
||||
OriginalLength: uint32(len(testData)),
|
||||
EncoderK: 8,
|
||||
EncoderM: 8,
|
||||
EncoderTechnique: Cauchy,
|
||||
}
|
||||
|
||||
err := Write(&testBuffer, testHeader.OriginalLength, testHeader.EncoderK, testHeader.EncoderM, testHeader.EncoderTechnique, bytes.NewBufferString(testData))
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
actualVersion := make([]byte, 4)
|
||||
_, err = testBuffer.Read(actualVersion)
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(binary.LittleEndian.Uint32(actualVersion), DeepEquals, uint32(1))
|
||||
|
||||
actualHeader := DataHeader{}
|
||||
|
||||
decoder := gob.NewDecoder(&testBuffer)
|
||||
decoder.Decode(&actualHeader)
|
||||
|
||||
c.Assert(actualHeader, DeepEquals, testHeader)
|
||||
|
||||
var actualData bytes.Buffer
|
||||
|
||||
dataLength, err := io.Copy(&actualData, &testBuffer)
|
||||
c.Assert(dataLength, Equals, int64(len(testData)))
|
||||
c.Assert(actualData.Bytes(), DeepEquals, []byte(testData))
|
||||
c.Assert(err, IsNil)
|
||||
}
|
||||
|
||||
func (s *MySuite) TestReadWrite(c *C) {
|
||||
var testBuffer bytes.Buffer
|
||||
testData := "Hello, World"
|
||||
testHeader := DataHeader{
|
||||
OriginalLength: uint32(len(testData)),
|
||||
EncoderK: 8,
|
||||
EncoderM: 8,
|
||||
EncoderTechnique: Cauchy,
|
||||
}
|
||||
|
||||
err := Write(&testBuffer, testHeader.OriginalLength, testHeader.EncoderK, testHeader.EncoderM, testHeader.EncoderTechnique, bytes.NewBufferString(testData))
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
header, err := ReadHeader(&testBuffer)
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(header, DeepEquals, testHeader)
|
||||
c.Assert(testBuffer.Len(), Equals, len(testData))
|
||||
c.Assert(testBuffer.Bytes(), DeepEquals, []byte(testData))
|
||||
}
|
||||
@ -1,222 +0,0 @@
|
||||
/*
|
||||
* Mini Object Storage, (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package fragment
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"github.com/minio-io/minio/pkg/utils/checksum/crc32c"
|
||||
"github.com/minio-io/minio/pkg/utils/crypto/sha512"
|
||||
)
|
||||
|
||||
/*
|
||||
|
||||
DONUT v1 Spec
|
||||
**********************
|
||||
BlockStart uint32 // Magic="MINI"=1229867341
|
||||
VersionMajor uint32
|
||||
Reserved uint64
|
||||
DataLen uint64
|
||||
HeaderCrc32c uint32
|
||||
BlockData uint32 // Magic="DATA"=1096040772
|
||||
Data io.Reader // matches length
|
||||
HeaderCrc32c uint32
|
||||
DataSha512 [64]byte
|
||||
BlockLen uint64 // length of entire frame, inclusive of MINI and INIM
|
||||
BlockEnd uint32 // Magic="INIM"=1296649801
|
||||
|
||||
*/
|
||||
|
||||
// Magic list
|
||||
var (
|
||||
MagicMINI = binary.LittleEndian.Uint32([]byte{'M', 'I', 'N', 'I'})
|
||||
MagicDATA = binary.LittleEndian.Uint32([]byte{'D', 'A', 'T', 'A'})
|
||||
MagicINIM = binary.LittleEndian.Uint32([]byte{'I', 'N', 'I', 'M'})
|
||||
)
|
||||
|
||||
// DonutFrameHeader -
|
||||
// --------------
|
||||
// BlockStart uint32
|
||||
// VersionMajor uint32
|
||||
// Reserved uint64
|
||||
// DataLen uint64
|
||||
// --------------
|
||||
type DonutFrameHeader struct {
|
||||
MagicMINI uint32
|
||||
Version uint32
|
||||
Reserved uint64
|
||||
DataLength uint64
|
||||
}
|
||||
|
||||
// Crc32c checksum
|
||||
type Crc32c uint32
|
||||
|
||||
// Sha512 checksum
|
||||
type Sha512 [sha512.Size]byte
|
||||
|
||||
// DonutFrameFooter -
|
||||
// --------------
|
||||
// DataSha512 [64]byte
|
||||
// BlockLen uint64
|
||||
// BlockEnd uint32
|
||||
// --------------
|
||||
type DonutFrameFooter struct {
|
||||
DataSha512 Sha512
|
||||
OffsetToMINI uint64
|
||||
MagicINIM uint32
|
||||
}
|
||||
|
||||
// Data buffer
|
||||
type Data bytes.Buffer
|
||||
|
||||
// Write - write donut format to input io.Writer, returns error upon any failure
|
||||
func Write(target io.Writer, reader io.Reader, length uint64) error {
|
||||
// write header
|
||||
header := DonutFrameHeader{
|
||||
MagicMINI: MagicMINI,
|
||||
Version: 1,
|
||||
Reserved: 0,
|
||||
DataLength: length,
|
||||
}
|
||||
var headerBytes bytes.Buffer
|
||||
binary.Write(&headerBytes, binary.LittleEndian, header)
|
||||
headerCrc := crc32c.Sum32(headerBytes.Bytes())
|
||||
|
||||
binary.Write(&headerBytes, binary.LittleEndian, headerCrc)
|
||||
binary.Write(&headerBytes, binary.LittleEndian, MagicDATA)
|
||||
// write header
|
||||
headerLen, err := io.Copy(target, &headerBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// write DATA
|
||||
// create sha512 tee
|
||||
sumReader, sumWriter := io.Pipe()
|
||||
defer sumWriter.Close()
|
||||
checksumChannel := make(chan checksumValue)
|
||||
go generateChecksum(sumReader, checksumChannel)
|
||||
teeReader := io.TeeReader(reader, sumWriter)
|
||||
dataLength, err := io.Copy(target, teeReader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if uint64(dataLength) != length {
|
||||
return errors.New("Specified data length and amount written mismatched")
|
||||
}
|
||||
sumWriter.Close()
|
||||
dataChecksum := <-checksumChannel
|
||||
if dataChecksum.err != nil {
|
||||
return dataChecksum.err
|
||||
}
|
||||
// generate footer
|
||||
frameFooter := DonutFrameFooter{
|
||||
DataSha512: dataChecksum.checksum,
|
||||
OffsetToMINI: length + uint64(headerLen) + uint64(80), /*footer size*/
|
||||
MagicINIM: MagicINIM,
|
||||
}
|
||||
var frameFooterBytes bytes.Buffer
|
||||
binary.Write(&frameFooterBytes, binary.LittleEndian, frameFooter)
|
||||
// write footer crc
|
||||
footerChecksum := crc32c.Sum32(frameFooterBytes.Bytes())
|
||||
if err := binary.Write(target, binary.LittleEndian, footerChecksum); err != nil {
|
||||
return err
|
||||
}
|
||||
// write write footer
|
||||
_, err = io.Copy(target, &frameFooterBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read - reads a donut fragment
|
||||
func Read(reader io.Reader) (io.Reader, error) {
|
||||
header, err := ReadHeader(reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sumReader, sumWriter := io.Pipe()
|
||||
|
||||
teeReader := io.TeeReader(reader, sumWriter)
|
||||
|
||||
defer sumWriter.Close()
|
||||
checksumChannel := make(chan checksumValue)
|
||||
go generateChecksum(sumReader, checksumChannel)
|
||||
|
||||
data := make([]byte, header.DataLength)
|
||||
teeReader.Read(data)
|
||||
sumWriter.Close()
|
||||
|
||||
// read crc
|
||||
footerBuffer := make([]byte, 80)
|
||||
reader.Read(footerBuffer)
|
||||
expectedCrc := binary.LittleEndian.Uint32(footerBuffer[:4])
|
||||
actualCrc := crc32c.Sum32(footerBuffer[4:])
|
||||
|
||||
if expectedCrc != actualCrc {
|
||||
// TODO perhaps we should return data and still report error?
|
||||
return nil, errors.New("Expected CRC doesn't match for footer")
|
||||
}
|
||||
|
||||
footer := DonutFrameFooter{}
|
||||
err = binary.Read(bytes.NewBuffer(footerBuffer[4:]), binary.LittleEndian, &footer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return bytes.NewBuffer(data), nil
|
||||
}
|
||||
|
||||
// ReadHeader - reads the header of a donut
|
||||
func ReadHeader(reader io.Reader) (header DonutFrameHeader, err error) {
|
||||
headerSlice := make([]byte, 32)
|
||||
headerLength, err := reader.Read(headerSlice)
|
||||
if err != nil {
|
||||
return header, err
|
||||
}
|
||||
if headerLength != 32 {
|
||||
return header, errors.New("EOF found while reading donut header")
|
||||
}
|
||||
|
||||
actualCrc := crc32c.Sum32(headerSlice[:24])
|
||||
expectedCrc := binary.LittleEndian.Uint32(headerSlice[24:28])
|
||||
if actualCrc != expectedCrc {
|
||||
return header, errors.New("CRC for donut did not match")
|
||||
}
|
||||
|
||||
err = binary.Read(bytes.NewBuffer(headerSlice[0:24]), binary.LittleEndian, &header)
|
||||
return header, nil
|
||||
}
|
||||
|
||||
type checksumValue struct {
|
||||
checksum Sha512
|
||||
err error
|
||||
}
|
||||
|
||||
// calculate sha512 over channel
|
||||
func generateChecksum(reader io.Reader, c chan<- checksumValue) {
|
||||
checksum, err := sha512.SumStream(reader)
|
||||
result := checksumValue{
|
||||
checksum: checksum,
|
||||
err: err,
|
||||
}
|
||||
c <- result
|
||||
}
|
||||
@ -1,160 +0,0 @@
|
||||
/*
|
||||
* Mini Object Storage, (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package fragment
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha512"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/minio-io/minio/pkg/utils/checksum/crc32c"
|
||||
. "gopkg.in/check.v1"
|
||||
)
|
||||
|
||||
func Test(t *testing.T) { TestingT(t) }
|
||||
|
||||
type MySuite struct{}
|
||||
|
||||
var _ = Suite(&MySuite{})
|
||||
|
||||
func (s *MySuite) TestSingleWrite(c *C) {
|
||||
//var b io.ReadWriteSeeker
|
||||
var testBuffer bytes.Buffer
|
||||
|
||||
testData := "Hello, World"
|
||||
testLength := uint64(len(testData))
|
||||
err := Write(&testBuffer, bytes.NewBufferString(testData), testLength)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
testBufferLength := uint64(testBuffer.Len())
|
||||
|
||||
// we test our crc here too
|
||||
headerBytes := testBuffer.Bytes()[:24]
|
||||
expectedCrc := crc32c.Sum32(headerBytes)
|
||||
|
||||
// magic mini
|
||||
magicMini := make([]byte, 4)
|
||||
testBuffer.Read(magicMini)
|
||||
c.Assert(magicMini, DeepEquals, []byte{'M', 'I', 'N', 'I'})
|
||||
|
||||
// major version
|
||||
version := make([]byte, 4)
|
||||
testBuffer.Read(version)
|
||||
c.Assert(binary.LittleEndian.Uint32(version), DeepEquals, uint32(1))
|
||||
|
||||
// reserved
|
||||
reserved := make([]byte, 8)
|
||||
testBuffer.Read(reserved)
|
||||
c.Assert(binary.LittleEndian.Uint64(reserved), DeepEquals, uint64(0))
|
||||
|
||||
// data length
|
||||
length := make([]byte, 8)
|
||||
testBuffer.Read(length)
|
||||
c.Assert(binary.LittleEndian.Uint64(length), DeepEquals, testLength)
|
||||
|
||||
// test crc
|
||||
bufCrc := make([]byte, 4)
|
||||
testBuffer.Read(bufCrc)
|
||||
c.Assert(binary.LittleEndian.Uint32(bufCrc), DeepEquals, expectedCrc)
|
||||
|
||||
// magic DATA
|
||||
magicData := make([]byte, 4)
|
||||
testBuffer.Read(magicData)
|
||||
c.Assert(magicData, DeepEquals, []byte{'D', 'A', 'T', 'A'})
|
||||
|
||||
// data
|
||||
actualData := make([]byte, int32(testLength))
|
||||
testBuffer.Read(actualData)
|
||||
c.Assert(string(actualData), DeepEquals, testData)
|
||||
|
||||
// extract footer crc32c
|
||||
actualFooterCrc := make([]byte, 4)
|
||||
testBuffer.Read(actualFooterCrc)
|
||||
remainingBytes := testBuffer.Bytes()
|
||||
remainingSum := crc32c.Sum32(remainingBytes)
|
||||
c.Assert(binary.LittleEndian.Uint32(actualFooterCrc), DeepEquals, remainingSum)
|
||||
|
||||
// sha512
|
||||
expectedSha512 := sha512.Sum512([]byte(testData))
|
||||
actualSha512 := make([]byte, 64)
|
||||
testBuffer.Read(actualSha512)
|
||||
c.Assert(actualSha512, DeepEquals, expectedSha512[:])
|
||||
|
||||
// length
|
||||
actualLength := make([]byte, 8)
|
||||
testBuffer.Read(actualLength)
|
||||
c.Assert(testBufferLength, DeepEquals, binary.LittleEndian.Uint64(actualLength))
|
||||
|
||||
// magic INIM
|
||||
magicInim := make([]byte, 4)
|
||||
testBuffer.Read(magicInim)
|
||||
c.Assert(magicInim, DeepEquals, []byte{'I', 'N', 'I', 'M'})
|
||||
|
||||
// ensure no extra data is in the file
|
||||
c.Assert(testBuffer.Len(), Equals, 0)
|
||||
}
|
||||
|
||||
func (s *MySuite) TestLengthMismatchInWrite(c *C) {
|
||||
var testData bytes.Buffer
|
||||
err := Write(&testData, bytes.NewBufferString("hello, world"), 5)
|
||||
c.Assert(err, Not(IsNil))
|
||||
}
|
||||
|
||||
func (s *MySuite) TestWriteAndRead(c *C) {
|
||||
testData := "Hello, World"
|
||||
testLength := uint64(len(testData))
|
||||
var testBuffer bytes.Buffer
|
||||
err := Write(&testBuffer, bytes.NewBufferString(testData), testLength)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
testReader, err := Read(&testBuffer)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
var actualData bytes.Buffer
|
||||
length, err := io.Copy(&actualData, testReader)
|
||||
c.Assert(int64(len(testData)), Equals, length)
|
||||
|
||||
c.Assert([]byte(testData), DeepEquals, actualData.Bytes())
|
||||
}
|
||||
|
||||
var buf = make([]byte, 1024*1024*8)
|
||||
|
||||
func benchmarkSize(b *testing.B, size int) {
|
||||
b.SetBytes(int64(size))
|
||||
target := new(bytes.Buffer)
|
||||
for i := 0; i < b.N; i++ {
|
||||
Write(target, bytes.NewReader(buf[:size]), uint64(size))
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDonut1M(b *testing.B) {
|
||||
benchmarkSize(b, 1024*1024)
|
||||
}
|
||||
|
||||
func BenchmarkDonut2M(b *testing.B) {
|
||||
benchmarkSize(b, 1024*1024*2)
|
||||
}
|
||||
|
||||
func BenchmarkDonut4M(b *testing.B) {
|
||||
benchmarkSize(b, 1024*1024*4)
|
||||
}
|
||||
|
||||
func BenchmarkDonut8M(b *testing.B) {
|
||||
benchmarkSize(b, 1024*1024*8)
|
||||
}
|
||||
@ -1,66 +0,0 @@
|
||||
/*
|
||||
* Mini Object Storage, (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package objectv1
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"encoding/gob"
|
||||
"errors"
|
||||
"io"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// Package Version
|
||||
const Version = uint32(1)
|
||||
|
||||
// ObjectMetadata contains information necessary to reconstruct the object and basic object metadata.
|
||||
type ObjectMetadata struct {
|
||||
Bucket string
|
||||
Key string
|
||||
ErasurePart uint16
|
||||
EncodedPart uint8
|
||||
}
|
||||
|
||||
// Write an encoded part to a writer
|
||||
func Write(target io.Writer, metadata ObjectMetadata, reader io.Reader) error {
|
||||
buffer := new(bytes.Buffer)
|
||||
binary.Write(buffer, binary.LittleEndian, uint32(Version))
|
||||
encoder := gob.NewEncoder(buffer)
|
||||
if err := encoder.Encode(metadata); err != nil {
|
||||
return err
|
||||
}
|
||||
reader = io.MultiReader(buffer, reader)
|
||||
_, err := io.Copy(target, reader)
|
||||
return err
|
||||
}
|
||||
|
||||
// ReadMetadata reads the first elements from the stream and returns the object metadata
|
||||
func ReadMetadata(reader io.Reader) (metadata ObjectMetadata, err error) {
|
||||
versionBytes := make([]byte, 4)
|
||||
if err := binary.Read(reader, binary.LittleEndian, versionBytes); err != nil {
|
||||
return metadata, err
|
||||
}
|
||||
var version uint32
|
||||
version = binary.LittleEndian.Uint32(versionBytes)
|
||||
if version != 1 {
|
||||
return metadata, errors.New("Unknown Version: " + strconv.FormatUint(uint64(version), 10))
|
||||
}
|
||||
decoder := gob.NewDecoder(reader)
|
||||
err = decoder.Decode(&metadata)
|
||||
return metadata, err
|
||||
}
|
||||
@ -1,76 +0,0 @@
|
||||
/*
|
||||
* Mini Object Storage, (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package objectv1
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"encoding/binary"
|
||||
"encoding/gob"
|
||||
. "gopkg.in/check.v1"
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
func Test(t *testing.T) { TestingT(t) }
|
||||
|
||||
type MySuite struct{}
|
||||
|
||||
var _ = Suite(&MySuite{})
|
||||
|
||||
func (s *MySuite) TestObjectV1ReadWrite(c *C) {
|
||||
var buffer bytes.Buffer
|
||||
|
||||
data := []byte("Hello, World")
|
||||
|
||||
hash := md5.New()
|
||||
sum := hash.Sum(data)
|
||||
|
||||
objectMetadata := ObjectMetadata{
|
||||
Bucket: "bucket",
|
||||
Key: "key",
|
||||
ErasurePart: 1,
|
||||
EncodedPart: 2,
|
||||
|
||||
ObjectType: Object,
|
||||
Created: time.Now(),
|
||||
ContentType: "application/text",
|
||||
Md5: sum,
|
||||
Length: uint64(len(sum)),
|
||||
}
|
||||
|
||||
err := Write(&buffer, objectMetadata, bytes.NewBuffer(data))
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
versionBuffer := make([]byte, 4)
|
||||
buffer.Read(versionBuffer)
|
||||
c.Assert(binary.LittleEndian.Uint32(versionBuffer), Equals, uint32(1))
|
||||
|
||||
actualMetadata := ObjectMetadata{}
|
||||
decoder := gob.NewDecoder(&buffer)
|
||||
decoder.Decode(&actualMetadata)
|
||||
|
||||
c.Assert(actualMetadata, DeepEquals, objectMetadata)
|
||||
|
||||
var actualData bytes.Buffer
|
||||
|
||||
_, err = io.Copy(&actualData, &buffer)
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(actualData.Bytes(), DeepEquals, data)
|
||||
}
|
||||
@ -227,6 +227,7 @@ func (storage *Storage) ListObjects(bucket string, resources mstorage.BucketReso
|
||||
return results, resources, nil
|
||||
}
|
||||
|
||||
// ByBucketName is a type for sorting bucket metadata by bucket name
|
||||
type ByBucketName []mstorage.BucketMetadata
|
||||
|
||||
// Len of bucket name
|
||||
|
||||
@ -1,111 +0,0 @@
|
||||
/*
|
||||
* Mini Object Storage, (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package singledisk
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"encoding/gob"
|
||||
"errors"
|
||||
"github.com/minio-io/minio/pkg/encoding/erasure"
|
||||
"github.com/minio-io/minio/pkg/storage"
|
||||
"github.com/minio-io/minio/pkg/storage/donut/erasure/erasure1"
|
||||
"github.com/minio-io/minio/pkg/storage/donut/object/objectv1"
|
||||
"github.com/minio-io/minio/pkg/utils/split"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
)
|
||||
|
||||
// StorageDriver creates a new single disk storage driver using donut without encoding.
|
||||
type StorageDriver struct {
|
||||
donutBox DonutBox
|
||||
}
|
||||
|
||||
// DonutBox is an interface specifying how the storage driver should interact with its underlying system.
|
||||
type DonutBox interface {
|
||||
Store(objectv1.ObjectMetadata, erasure1.DataHeader, io.Reader) error
|
||||
Get(bucket, key string, erasurePart uint16, encodedPart uint8) (objectv1.ObjectMetadata, erasure1.DataHeader, io.Reader, error)
|
||||
ListObjects(bucket string) ([]string, error)
|
||||
ListBuckets() ([]storage.BucketMetadata, error)
|
||||
}
|
||||
|
||||
// Start a single disk subsystem
|
||||
func Start(root string, donutBox DonutBox) (chan<- string, <-chan error, storage.Storage) {
|
||||
ctrlChannel := make(chan string)
|
||||
errorChannel := make(chan error)
|
||||
s := new(StorageDriver)
|
||||
s.donutBox = donutBox
|
||||
go start(ctrlChannel, errorChannel, s)
|
||||
return ctrlChannel, errorChannel, s
|
||||
}
|
||||
|
||||
func start(ctrlChannel <-chan string, errorChannel chan<- error, s *StorageDriver) {
|
||||
err := os.MkdirAll(s.root, 0700)
|
||||
errorChannel <- err
|
||||
close(errorChannel)
|
||||
}
|
||||
|
||||
// ListBuckets returns a list of buckets
|
||||
func (diskStorage StorageDriver) ListBuckets() ([]storage.BucketMetadata, error) {
|
||||
return nil, errors.New("Not Implemented")
|
||||
}
|
||||
|
||||
// CreateBucket creates a new bucket
|
||||
func (diskStorage StorageDriver) CreateBucket(bucket string) error {
|
||||
return errors.New("Not Implemented")
|
||||
}
|
||||
|
||||
// CreateBucketPolicy sets a bucket's access policy
|
||||
func (diskStorage StorageDriver) CreateBucketPolicy(bucket string, p storage.BucketPolicy) error {
|
||||
return errors.New("Not Implemented")
|
||||
}
|
||||
|
||||
// GetBucketPolicy returns a bucket's access policy
|
||||
func (diskStorage StorageDriver) GetBucketPolicy(bucket string) (storage.BucketPolicy, error) {
|
||||
return storage.BucketPolicy{}, errors.New("Not Implemented")
|
||||
}
|
||||
|
||||
// GetObject retrieves an object and writes it to a writer
|
||||
func (diskStorage StorageDriver) GetObject(target io.Writer, bucket, key string) (int64, error) {
|
||||
return 0, errors.New("Not Implemented")
|
||||
}
|
||||
|
||||
// GetPartialObject retrieves an object and writes it to a writer
|
||||
func (diskStorage StorageDriver) GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error) {
|
||||
return 0, errors.New("Not Implemented")
|
||||
}
|
||||
|
||||
// GetObjectMetadata retrieves an object's metadata
|
||||
func (diskStorage StorageDriver) GetObjectMetadata(bucket, key string, prefix string) (metadata storage.ObjectMetadata, err error) {
|
||||
return metadata, errors.New("Not Implemented")
|
||||
}
|
||||
|
||||
func readHeaderGob(reader io.Reader) (header ObjectHeader, err error) {
|
||||
return header, errors.New("Not Implemented")
|
||||
}
|
||||
|
||||
// ListObjects lists objects
|
||||
func (diskStorage StorageDriver) ListObjects(bucket string, resources storage.BucketResourcesMetadata) ([]storage.ObjectMetadata, storage.BucketResourcesMetadata, error) {
|
||||
return nil, storage.BucketResourcesMetadata{}, errors.New("Not Implemented")
|
||||
}
|
||||
|
||||
// CreateObject creates a new object
|
||||
func (diskStorage StorageDriver) CreateObject(bucket string, key string, contentType string, data io.Reader) error {
|
||||
return errors.New("Not Implemented")
|
||||
}
|
||||
@ -1,54 +0,0 @@
|
||||
/*
|
||||
* Mini Object Storage, (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package singledisk
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
mstorage "github.com/minio-io/minio/pkg/storage"
|
||||
|
||||
. "gopkg.in/check.v1"
|
||||
)
|
||||
|
||||
func Test(t *testing.T) { TestingT(t) }
|
||||
|
||||
type MySuite struct{}
|
||||
|
||||
var _ = Suite(&MySuite{})
|
||||
|
||||
func (s *MySuite) TestAPISuite(c *C) {
|
||||
c.Skip("Not implemented yet.")
|
||||
var storageList []string
|
||||
create := func() mstorage.Storage {
|
||||
path, err := ioutil.TempDir(os.TempDir(), "minio-fs-")
|
||||
c.Check(err, IsNil)
|
||||
storageList = append(storageList, path)
|
||||
_, _, store := Start(path, StartDonutBox()) // TODO Make InMemory driver
|
||||
return store
|
||||
}
|
||||
mstorage.APITestSuite(c, create)
|
||||
removeRoots(c, storageList)
|
||||
}
|
||||
|
||||
func removeRoots(c *C, roots []string) {
|
||||
for _, root := range roots {
|
||||
err := os.RemoveAll(root)
|
||||
c.Check(err, IsNil)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user