mirror of
https://github.com/minio/minio.git
synced 2025-11-09 21:49:46 -05:00
Move pkg/storage/erasure to pkg/encoding/erasure - and other cleanups
This commit is contained in:
Harshavardhana
161
pkg/storage/donut/fragment/fragment1/fragment.go
Normal file
161
pkg/storage/donut/fragment/fragment1/fragment.go
Normal file
@@ -0,0 +1,161 @@
|
||||
/*
|
||||
* Mini Object Storage, (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package fragment
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"github.com/minio-io/minio/pkg/utils/checksum/crc32c"
|
||||
"github.com/minio-io/minio/pkg/utils/crypto/sha512"
|
||||
)
|
||||
|
||||
/*
|
||||
|
||||
DONUT v1 Spec
|
||||
**********************
|
||||
BlockStart uint32 // Magic="MINI"=1229867341
|
||||
VersionMajor uint32
|
||||
Reserved uint64
|
||||
DataLen uint64
|
||||
HeaderCrc32c uint32
|
||||
BlockData uint32 // Magic="DATA"=1096040772
|
||||
Data io.Reader // matches length
|
||||
HeaderCrc32c uint32
|
||||
DataSha512 [64]byte
|
||||
BlockLen uint64 // length of entire frame, inclusive of MINI and INIM
|
||||
BlockEnd uint32 // Magic="INIM"=1296649801
|
||||
|
||||
*/
|
||||
|
||||
// Magic list
|
||||
var (
|
||||
MagicMINI = binary.LittleEndian.Uint32([]byte{'M', 'I', 'N', 'I'})
|
||||
MagicDATA = binary.LittleEndian.Uint32([]byte{'D', 'A', 'T', 'A'})
|
||||
MagicINIM = binary.LittleEndian.Uint32([]byte{'I', 'N', 'I', 'M'})
|
||||
)
|
||||
|
||||
// DonutFrameHeader -
|
||||
// --------------
|
||||
// BlockStart uint32
|
||||
// VersionMajor uint32
|
||||
// Reserved uint64
|
||||
// DataLen uint64
|
||||
// --------------
|
||||
type DonutFrameHeader struct {
|
||||
MagicMINI uint32
|
||||
Version uint32
|
||||
Reserved uint64
|
||||
DataLength uint64
|
||||
}
|
||||
|
||||
// Crc32c checksum
|
||||
type Crc32c uint32
|
||||
|
||||
// Sha512 checksum
|
||||
type Sha512 [sha512.Size]byte
|
||||
|
||||
// DonutFrameFooter -
|
||||
// --------------
|
||||
// DataSha512 [64]byte
|
||||
// BlockLen uint64
|
||||
// BlockEnd uint32
|
||||
// --------------
|
||||
type DonutFrameFooter struct {
|
||||
DataSha512 Sha512
|
||||
OffsetToMINI uint64
|
||||
MagicINIM uint32
|
||||
}
|
||||
|
||||
// Data buffer
|
||||
type Data bytes.Buffer
|
||||
|
||||
// Write - write donut format to input io.Writer, returns error upon any failure
|
||||
func Write(target io.Writer, reader io.Reader, length uint64) error {
|
||||
// write header
|
||||
header := DonutFrameHeader{
|
||||
MagicMINI: MagicMINI,
|
||||
Version: 1,
|
||||
Reserved: 0,
|
||||
DataLength: length,
|
||||
}
|
||||
var headerBytes bytes.Buffer
|
||||
binary.Write(&headerBytes, binary.LittleEndian, header)
|
||||
headerCrc := crc32c.Sum32(headerBytes.Bytes())
|
||||
|
||||
binary.Write(&headerBytes, binary.LittleEndian, headerCrc)
|
||||
binary.Write(&headerBytes, binary.LittleEndian, MagicDATA)
|
||||
// write header
|
||||
headerLen, err := io.Copy(target, &headerBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// write DATA
|
||||
// create sha512 tee
|
||||
sumReader, sumWriter := io.Pipe()
|
||||
defer sumWriter.Close()
|
||||
checksumChannel := make(chan checksumValue)
|
||||
go generateChecksum(sumReader, checksumChannel)
|
||||
teeReader := io.TeeReader(reader, sumWriter)
|
||||
dataLength, err := io.Copy(target, teeReader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if uint64(dataLength) != length {
|
||||
return errors.New("Specified data length and amount written mismatched")
|
||||
}
|
||||
sumWriter.Close()
|
||||
dataChecksum := <-checksumChannel
|
||||
if dataChecksum.err != nil {
|
||||
return dataChecksum.err
|
||||
}
|
||||
// generate footer
|
||||
frameFooter := DonutFrameFooter{
|
||||
DataSha512: dataChecksum.checksum,
|
||||
OffsetToMINI: length + uint64(headerLen) + uint64(80), /*footer size*/
|
||||
MagicINIM: MagicINIM,
|
||||
}
|
||||
var frameFooterBytes bytes.Buffer
|
||||
binary.Write(&frameFooterBytes, binary.LittleEndian, frameFooter)
|
||||
// write footer crc
|
||||
footerChecksum := crc32c.Sum32(frameFooterBytes.Bytes())
|
||||
if err := binary.Write(target, binary.LittleEndian, footerChecksum); err != nil {
|
||||
return err
|
||||
}
|
||||
// write write footer
|
||||
_, err = io.Copy(target, &frameFooterBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type checksumValue struct {
|
||||
checksum Sha512
|
||||
err error
|
||||
}
|
||||
|
||||
func generateChecksum(reader io.Reader, c chan<- checksumValue) {
|
||||
checksum, err := sha512.SumStream(reader)
|
||||
result := checksumValue{
|
||||
checksum: checksum,
|
||||
err: err,
|
||||
}
|
||||
c <- result
|
||||
}
|
||||
142
pkg/storage/donut/fragment/fragment1/fragment_test.go
Normal file
142
pkg/storage/donut/fragment/fragment1/fragment_test.go
Normal file
@@ -0,0 +1,142 @@
|
||||
/*
|
||||
* Mini Object Storage, (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package fragment
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha512"
|
||||
"encoding/binary"
|
||||
"testing"
|
||||
|
||||
"github.com/minio-io/minio/pkg/utils/checksum/crc32c"
|
||||
. "gopkg.in/check.v1"
|
||||
)
|
||||
|
||||
func Test(t *testing.T) { TestingT(t) }
|
||||
|
||||
type MySuite struct{}
|
||||
|
||||
var _ = Suite(&MySuite{})
|
||||
|
||||
func (s *MySuite) TestSingleWrite(c *C) {
|
||||
//var b io.ReadWriteSeeker
|
||||
var testBuffer bytes.Buffer
|
||||
|
||||
testData := "Hello, World"
|
||||
testLength := uint64(len(testData))
|
||||
err := Write(&testBuffer, bytes.NewBufferString(testData), testLength)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
testBufferLength := uint64(testBuffer.Len())
|
||||
|
||||
// we test our crc here too
|
||||
headerBytes := testBuffer.Bytes()[:24]
|
||||
expectedCrc := crc32c.Sum32(headerBytes)
|
||||
|
||||
// magic mini
|
||||
magicMini := make([]byte, 4)
|
||||
testBuffer.Read(magicMini)
|
||||
c.Assert(magicMini, DeepEquals, []byte{'M', 'I', 'N', 'I'})
|
||||
|
||||
// major version
|
||||
version := make([]byte, 4)
|
||||
testBuffer.Read(version)
|
||||
c.Assert(binary.LittleEndian.Uint32(version), DeepEquals, uint32(1))
|
||||
|
||||
// reserved
|
||||
reserved := make([]byte, 8)
|
||||
testBuffer.Read(reserved)
|
||||
c.Assert(binary.LittleEndian.Uint64(reserved), DeepEquals, uint64(0))
|
||||
|
||||
// data length
|
||||
length := make([]byte, 8)
|
||||
testBuffer.Read(length)
|
||||
c.Assert(binary.LittleEndian.Uint64(length), DeepEquals, testLength)
|
||||
|
||||
// test crc
|
||||
bufCrc := make([]byte, 4)
|
||||
testBuffer.Read(bufCrc)
|
||||
c.Assert(binary.LittleEndian.Uint32(bufCrc), DeepEquals, expectedCrc)
|
||||
|
||||
// magic DATA
|
||||
magicData := make([]byte, 4)
|
||||
testBuffer.Read(magicData)
|
||||
c.Assert(magicData, DeepEquals, []byte{'D', 'A', 'T', 'A'})
|
||||
|
||||
// data
|
||||
actualData := make([]byte, int32(testLength))
|
||||
testBuffer.Read(actualData)
|
||||
c.Assert(string(actualData), DeepEquals, testData)
|
||||
|
||||
// extract footer crc32c
|
||||
actualFooterCrc := make([]byte, 4)
|
||||
testBuffer.Read(actualFooterCrc)
|
||||
remainingBytes := testBuffer.Bytes()
|
||||
remainingSum := crc32c.Sum32(remainingBytes)
|
||||
c.Assert(binary.LittleEndian.Uint32(actualFooterCrc), DeepEquals, remainingSum)
|
||||
|
||||
// sha512
|
||||
expectedSha512 := sha512.Sum512([]byte(testData))
|
||||
actualSha512 := make([]byte, 64)
|
||||
testBuffer.Read(actualSha512)
|
||||
c.Assert(actualSha512, DeepEquals, expectedSha512[:])
|
||||
|
||||
// length
|
||||
actualLength := make([]byte, 8)
|
||||
testBuffer.Read(actualLength)
|
||||
c.Assert(testBufferLength, DeepEquals, binary.LittleEndian.Uint64(actualLength))
|
||||
|
||||
// magic INIM
|
||||
magicInim := make([]byte, 4)
|
||||
testBuffer.Read(magicInim)
|
||||
c.Assert(magicInim, DeepEquals, []byte{'I', 'N', 'I', 'M'})
|
||||
|
||||
// ensure no extra data is in the file
|
||||
c.Assert(testBuffer.Len(), Equals, 0)
|
||||
}
|
||||
|
||||
func (s *MySuite) TestLengthMismatchInWrite(c *C) {
|
||||
var testData bytes.Buffer
|
||||
err := Write(&testData, bytes.NewBufferString("hello, world"), 5)
|
||||
c.Assert(err, Not(IsNil))
|
||||
}
|
||||
|
||||
var buf = make([]byte, 1024*1024*8)
|
||||
|
||||
func benchmarkSize(b *testing.B, size int) {
|
||||
b.SetBytes(int64(size))
|
||||
target := new(bytes.Buffer)
|
||||
for i := 0; i < b.N; i++ {
|
||||
Write(target, bytes.NewReader(buf[:size]), uint64(size))
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDonut1M(b *testing.B) {
|
||||
benchmarkSize(b, 1024*1024)
|
||||
}
|
||||
|
||||
func BenchmarkDonut2M(b *testing.B) {
|
||||
benchmarkSize(b, 1024*1024*2)
|
||||
}
|
||||
|
||||
func BenchmarkDonut4M(b *testing.B) {
|
||||
benchmarkSize(b, 1024*1024*4)
|
||||
}
|
||||
|
||||
func BenchmarkDonut8M(b *testing.B) {
|
||||
benchmarkSize(b, 1024*1024*8)
|
||||
}
|
||||
Reference in New Issue
Block a user