Donut frame is now its own package

pkg/storage/donut/frame/frame_v1/.gitignore

@@ -0,0 +1 @@
+newfile

pkg/storage/donut/frame/frame_v1/donut.go

@@ -0,0 +1,150 @@
+/*
+ * Mini Object Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package frame_v1
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"io"
+
+	"github.com/minio-io/minio/pkg/utils/checksum/crc32c"
+	"github.com/minio-io/minio/pkg/utils/crypto/sha512"
+)
+
+/*
+
+       DONUT v1 Spec
+   **********************
+   BlockStart      [4]byte             // Magic="MINI"=1229867341
+   VersionMajor    uint16
+   VersionMinor    uint16
+   VersionPatch    uint16
+   VersionReserved uint16
+   Reserved        uint64
+   DataLen         uint64
+   HeaderCrc32c    uint32
+   BlockData       [4]byte             // Magic="DATA"=1096040772
+   Data            io.Reader           // matches length
+   HeaderCrc32c    uint32
+   DataSha512      [64]byte
+   BlockLen        uint64              // length of entire frame, inclusive of MINI and INIM
+   BlockEnd        [4]byte             // Magic="INIM"=1296649801
+
+*/
+
+var (
+	MagicMINI = binary.LittleEndian.Uint32([]byte{'M', 'I', 'N', 'I'})
+	MagicDATA = binary.LittleEndian.Uint32([]byte{'D', 'A', 'T', 'A'})
+	MagicINIM = binary.LittleEndian.Uint32([]byte{'I', 'N', 'I', 'M'})
+)
+
+type DonutFrameHeader struct {
+	MagicMINI       uint32
+	VersionMajor    uint16
+	VersionMinor    uint16
+	VersionPatch    uint16
+	VersionReserved uint16
+	Reserved        uint64
+	DataLength      uint64
+}
+type Crc32c uint32
+type Sha512 [sha512.Size]byte
+
+type DonutFrameFooter struct {
+	DataSha512   Sha512
+	OffsetToMINI uint64
+	MagicINIM    uint32
+}
+
+type Data bytes.Buffer
+
+func WriteFrame(target io.Writer, reader io.Reader, length uint64) error {
+	// write header
+	header := DonutFrameHeader{
+		MagicMINI:       MagicMINI,
+		VersionMajor:    1,
+		VersionMinor:    0,
+		VersionPatch:    0,
+		VersionReserved: 0,
+		Reserved:        0,
+		DataLength:      length,
+	}
+	var headerBytes bytes.Buffer
+	binary.Write(&headerBytes, binary.LittleEndian, header)
+	headerCrc := crc32c.Sum32(headerBytes.Bytes())
+
+	binary.Write(&headerBytes, binary.LittleEndian, headerCrc)
+	binary.Write(&headerBytes, binary.LittleEndian, MagicDATA)
+	// write header
+	headerLen, err := io.Copy(target, &headerBytes)
+	if err != nil {
+		return err
+	}
+	// write DATA
+	// create sha512 tee
+	sumReader, sumWriter := io.Pipe()
+	defer sumWriter.Close()
+	checksumChannel := make(chan checksumValue)
+	go generateChecksum(sumReader, checksumChannel)
+	teeReader := io.TeeReader(reader, sumWriter)
+	dataLength, err := io.Copy(target, teeReader)
+	if err != nil {
+		return err
+	}
+	if uint64(dataLength) != length {
+		return errors.New("Specified data length and amount written mismatched")
+	}
+	sumWriter.Close()
+	dataChecksum := <-checksumChannel
+	if dataChecksum.err != nil {
+		return dataChecksum.err
+	}
+	// generate footer
+	frameFooter := DonutFrameFooter{
+		DataSha512:   dataChecksum.checksum,
+		OffsetToMINI: length + uint64(headerLen) + uint64(80), /*footer size*/
+		MagicINIM:    MagicINIM,
+	}
+	var frameFooterBytes bytes.Buffer
+	binary.Write(&frameFooterBytes, binary.LittleEndian, frameFooter)
+	// write footer crc
+	footerChecksum := crc32c.Sum32(frameFooterBytes.Bytes())
+	if err := binary.Write(target, binary.LittleEndian, footerChecksum); err != nil {
+		return err
+	}
+	// write write footer
+	_, err = io.Copy(target, &frameFooterBytes)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+type checksumValue struct {
+	checksum Sha512
+	err      error
+}
+
+func generateChecksum(reader io.Reader, c chan<- checksumValue) {
+	checksum, err := sha512.SumStream(reader)
+	result := checksumValue{
+		checksum: checksum,
+		err:      err,
+	}
+	c <- result
+}

pkg/storage/donut/frame/frame_v1/donut_gen_v1/.gitignore

@@ -0,0 +1,2 @@
+donut_gen
+hello

pkg/storage/donut/frame/frame_v1/donut_gen_v1/donut_gen.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"reflect"
+
+	"github.com/minio-io/minio/pkg/storage/donut/frame/frame_v1"
+)
+
+func main() {
+	fmt.Println("--start")
+
+	file, err := os.OpenFile("newfile", os.O_WRONLY|os.O_CREATE, 0666)
+	if err != nil {
+		panic(err)
+	}
+	data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
+
+	dataBuffer := bytes.NewBuffer(data)
+
+	err = frame_v1.WriteFrame(file, dataBuffer, uint64(dataBuffer.Len()))
+	if err != nil {
+		panic(err)
+	}
+	file.Close()
+	fmt.Println("--closed")
+
+	fmt.Println("--verify")
+	stat, _ := os.Stat("newfile")
+	fileSize := stat.Size()
+
+	rfile, _ := os.OpenFile("newfile", os.O_RDONLY, 0666)
+	blockStart := make([]byte, 4)
+	blockStartCheck := []byte{'M', 'I', 'N', 'I'}
+
+	_, err = rfile.Read(blockStart)
+	if err != nil {
+		panic(err)
+	}
+
+	blockEnd := make([]byte, 4)
+	start := fileSize - 4
+	blockEndCheck := []byte{'I', 'N', 'I', 'M'}
+	rfile.ReadAt(blockEnd, start)
+	rfile.Close()
+
+	if !reflect.DeepEqual(blockStart, blockStartCheck) {
+		panic("Corrupted donut file")
+	}
+
+	if !reflect.DeepEqual(blockEnd, blockEndCheck) {
+		panic("Corrupted donut file")
+	}
+
+	fmt.Println("--verified")
+	fmt.Println("--end")
+}

pkg/storage/donut/frame/frame_v1/donut_test.go

@@ -0,0 +1,158 @@
+/*
+ * Mini Object Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package frame_v1
+
+import (
+	"bytes"
+	"crypto/sha512"
+	"encoding/binary"
+	"testing"
+
+	"github.com/minio-io/minio/pkg/utils/checksum/crc32c"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type MySuite struct{}
+
+var _ = Suite(&MySuite{})
+
+func (s *MySuite) TestSingleWrite(c *C) {
+	//var b io.ReadWriteSeeker
+	var testBuffer bytes.Buffer
+
+	testData := "Hello, World"
+	testLength := uint64(len(testData))
+	err := WriteFrame(&testBuffer, bytes.NewBufferString(testData), testLength)
+	c.Assert(err, IsNil)
+
+	testBufferLength := uint64(testBuffer.Len())
+
+	// we test our crc here too
+	headerBytes := testBuffer.Bytes()[0:28]
+	expectedCrc := crc32c.Sum32(headerBytes)
+
+	// magic mini
+	magicMini := make([]byte, 4)
+	testBuffer.Read(magicMini)
+	c.Assert(magicMini, DeepEquals, []byte{'M', 'I', 'N', 'I'})
+
+	// major version
+	majorVersion := make([]byte, 2)
+	testBuffer.Read(majorVersion)
+	c.Assert(binary.LittleEndian.Uint16(majorVersion), DeepEquals, uint16(1))
+
+	// minor version
+	minorVersion := make([]byte, 2)
+	testBuffer.Read(minorVersion)
+	c.Assert(binary.LittleEndian.Uint16(minorVersion), DeepEquals, uint16(0))
+
+	// patch version
+	patchVersion := make([]byte, 2)
+	testBuffer.Read(patchVersion)
+	c.Assert(binary.LittleEndian.Uint16(patchVersion), DeepEquals, uint16(0))
+
+	// reserved version
+	reservedVersion := make([]byte, 2)
+	testBuffer.Read(reservedVersion)
+	c.Assert(binary.LittleEndian.Uint16(reservedVersion), DeepEquals, uint16(0))
+
+	// reserved
+	reserved := make([]byte, 8)
+	testBuffer.Read(reserved)
+	c.Assert(binary.LittleEndian.Uint64(reserved), DeepEquals, uint64(0))
+
+	// data length
+	length := make([]byte, 8)
+	testBuffer.Read(length)
+	c.Assert(binary.LittleEndian.Uint64(length), DeepEquals, testLength)
+
+	// test crc
+	bufCrc := make([]byte, 4)
+	testBuffer.Read(bufCrc)
+	c.Assert(binary.LittleEndian.Uint32(bufCrc), DeepEquals, expectedCrc)
+
+	// magic DATA
+	magicData := make([]byte, 4)
+	testBuffer.Read(magicData)
+	c.Assert(magicData, DeepEquals, []byte{'D', 'A', 'T', 'A'})
+
+	// data
+	actualData := make([]byte, int32(testLength))
+	testBuffer.Read(actualData)
+	c.Assert(string(actualData), DeepEquals, testData)
+
+	// extract footer crc32c
+	actualFooterCrc := make([]byte, 4)
+	testBuffer.Read(actualFooterCrc)
+	remainingBytes := testBuffer.Bytes()
+	remainingSum := crc32c.Sum32(remainingBytes)
+	c.Assert(binary.LittleEndian.Uint32(actualFooterCrc), DeepEquals, remainingSum)
+
+	// sha512
+	expectedSha512 := sha512.Sum512([]byte(testData))
+	actualSha512 := make([]byte, 64)
+	testBuffer.Read(actualSha512)
+	c.Assert(actualSha512, DeepEquals, expectedSha512[:])
+
+	// length
+	actualLength := make([]byte, 8)
+	testBuffer.Read(actualLength)
+	c.Assert(testBufferLength, DeepEquals, binary.LittleEndian.Uint64(actualLength))
+
+	// magic INIM
+	magicInim := make([]byte, 4)
+	testBuffer.Read(magicInim)
+	c.Assert(magicInim, DeepEquals, []byte{'I', 'N', 'I', 'M'})
+
+	// ensure no extra data is in the file
+	c.Assert(testBuffer.Len(), Equals, 0)
+}
+
+func (s *MySuite) TestLengthMismatchInWrite(c *C) {
+	var testData bytes.Buffer
+	err := WriteFrame(&testData, bytes.NewBufferString("hello, world"), 5)
+	c.Assert(err, Not(IsNil))
+}
+
+var buf = make([]byte, 1024*1024*1024)
+
+func benchmarkSize(b *testing.B, size int) {
+	b.SetBytes(int64(size))
+	target := new(bytes.Buffer)
+	for i := 0; i < b.N; i++ {
+		WriteFrame(target, bytes.NewReader(buf[:size]), uint64(size))
+	}
+}
+
+func BenchmarkDonut64M(b *testing.B) {
+	benchmarkSize(b, 1024*1024*64)
+}
+
+func BenchmarkDonut128M(b *testing.B) {
+	benchmarkSize(b, 1024*1024*128)
+}
+
+func BenchmarkDonut256M(b *testing.B) {
+	benchmarkSize(b, 1024*1024*256)
+}
+
+func BenchmarkDonut512M(b *testing.B) {
+	benchmarkSize(b, 1024*1024*512)
+}