Merge pull request #191 from harshavardhana/pr_out_add_proper_entries_for_gobheader_data_copy

pkg/storage/donut/v1/.gitignore

@@ -0,0 +1 @@
+newfile

pkg/storage/donut/v1/donut.go

@@ -17,9 +17,14 @@
 package v1
 
 import (
+	"bytes"
 	"encoding/binary"
+	"encoding/gob"
 	"errors"
 	"io"
+	"sync"
+
+	"github.com/minio-io/minio/pkg/storage/erasure"
 )
 
 /*
@@ -55,7 +60,7 @@ type DonutFormat struct {
 	VersionReserved uint16
 	Reserved        uint64
 	GobHeaderLen    uint32
-	GobHeader       GobHeader
+	GobHeader       []byte
 	BlockData       uint32 // Magic="DATA"=1096040772
 	Data            io.Reader
 	BlockLen        uint64
@@ -68,36 +73,61 @@ type DonutFooter struct {
 }
 
 type Donut struct {
-	file io.Writer
-	// mutex
+	file  io.Writer
+	mutex *sync.RWMutex
 }
 
-type GobHeader struct{}
+type GobHeader struct {
+	Blocks        []EncodedChunk
+	Md5sum        []byte
+	EncoderParams erasure.EncoderParams
+}
 
-func (donut *Donut) Write(gobHeader GobHeader, object io.Reader) error {
-	// TODO mutex
-	// Create bytes buffer representing the new object
-	donutFormat := DonutFormat{
-		BlockStart:      MagicMINI,
-		VersionMajor:    1,
-		VersionMinor:    0,
-		VersionPatch:    0,
-		VersionReserved: 0,
-		Reserved:        0,
-		GobHeaderLen:    0,
-		GobHeader:       gobHeader,
-		BlockData:       MagicDATA,
-		Data:            object,
-		BlockLen:        0,
-		BlockEnd:        MagicINIM,
+type EncodedChunk struct {
+	Crc    uint32
+	Length int
+	Offset int
+}
+
+func New(file io.Writer) *Donut {
+	donut := Donut{}
+	donut.mutex = new(sync.RWMutex)
+	donut.file = file
+	return &donut
+}
+
+func (donut *Donut) WriteGob(gobHeader GobHeader) (bytes.Buffer, error) {
+	var gobBuffer bytes.Buffer
+	encoder := gob.NewEncoder(&gobBuffer)
+	err := encoder.Encode(gobHeader)
+	if err != nil {
+		return bytes.Buffer{}, err
 	}
-	if err := donut.WriteFormat(donut.file, donutFormat); err != nil {
+	return gobBuffer, nil
+}
+
+func (donut *Donut) WriteEnd(target io.Writer, donutFormat DonutFormat) error {
+	if err := binary.Write(target, binary.LittleEndian, donutFormat.BlockLen); err != nil {
+		return err
+	}
+	if err := binary.Write(target, binary.LittleEndian, donutFormat.BlockEnd); err != nil {
 		return err
 	}
 	return nil
 }
 
-func (donut *Donut) WriteFormat(target io.Writer, donutFormat DonutFormat) error {
+func (donut *Donut) WriteData(target io.Writer, donutFormat DonutFormat) error {
+	var b bytes.Buffer
+	if count, err := io.Copy(&b, donutFormat.Data); uint64(count) != donutFormat.BlockLen || err != nil {
+		if err == nil {
+			return binary.Write(target, binary.LittleEndian, b.Bytes())
+		}
+		return errors.New("Copy failed, count incorrect.")
+	}
+	return nil
+}
+
+func (donut *Donut) WriteBegin(target io.Writer, donutFormat DonutFormat) error {
 	if err := binary.Write(target, binary.LittleEndian, donutFormat.BlockStart); err != nil {
 		return err
 	}
@@ -125,16 +155,43 @@ func (donut *Donut) WriteFormat(target io.Writer, donutFormat DonutFormat) error
 	if err := binary.Write(target, binary.LittleEndian, donutFormat.BlockData); err != nil {
 		return err
 	}
-	if count, err := io.Copy(target, donutFormat.Data); uint64(count) != donutFormat.BlockLen || err != nil {
-		if err == nil {
-			return err
-		}
-		return errors.New("Copy failed, count incorrect.")
-	}
-	if err := binary.Write(target, binary.LittleEndian, donutFormat.BlockLen); err != nil {
+	return nil
+}
+
+func (donut *Donut) Write(gobHeader GobHeader, object io.Reader) error {
+	donut.mutex.Lock()
+	defer donut.mutex.Unlock()
+
+	gobBytes, err := donut.WriteGob(gobHeader)
+	if err != nil {
 		return err
 	}
-	if err := binary.Write(target, binary.LittleEndian, donutFormat.BlockEnd); err != nil {
+
+	// Create bytes buffer representing the new object
+	donutFormat := DonutFormat{
+		BlockStart:      MagicMINI,
+		VersionMajor:    1,
+		VersionMinor:    0,
+		VersionPatch:    0,
+		VersionReserved: 0,
+		Reserved:        0,
+		GobHeaderLen:    uint32(gobBytes.Len()),
+		GobHeader:       gobBytes.Bytes(),
+		BlockData:       MagicDATA,
+		Data:            object,
+		BlockLen:        0,
+		BlockEnd:        MagicINIM,
+	}
+
+	if err := donut.WriteBegin(donut.file, donutFormat); err != nil {
+		return err
+	}
+
+	if err := donut.WriteData(donut.file, donutFormat); err != nil {
+		return err
+	}
+
+	if err := donut.WriteEnd(donut.file, donutFormat); err != nil {
 		return err
 	}
 

pkg/storage/donut/v1/donut_gen.go

@@ -0,0 +1,63 @@
+// +build ignore
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"reflect"
+
+	"github.com/minio-io/minio/pkg/storage/donut/v1"
+)
+
+func main() {
+	fmt.Println("--start")
+
+	file, err := os.OpenFile("newfile", os.O_WRONLY|os.O_CREATE, 0666)
+	if err != nil {
+		panic(err)
+	}
+	donut := v1.New(file)
+
+	gobHeader := v1.GobHeader{}
+	data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
+
+	dataBuffer := bytes.NewBuffer(data)
+	err = donut.Write(gobHeader, dataBuffer)
+	if err != nil {
+		panic(err)
+	}
+	file.Close()
+	fmt.Println("--closed")
+
+	fmt.Println("--verify")
+	stat, _ := os.Stat("newfile")
+	fileSize := stat.Size()
+
+	rfile, _ := os.OpenFile("newfile", os.O_RDONLY, 0666)
+	blockStart := make([]byte, 4)
+	blockStartCheck := []byte{'M', 'I', 'N', 'I'}
+
+	_, err = rfile.Read(blockStart)
+	if err != nil {
+		panic(err)
+	}
+
+	blockEnd := make([]byte, 4)
+	start := fileSize - 4
+	blockEndCheck := []byte{'I', 'N', 'I', 'M'}
+	rfile.ReadAt(blockEnd, start)
+	rfile.Close()
+
+	if !reflect.DeepEqual(blockStart, blockStartCheck) {
+		panic("Corrupted donut file")
+	}
+
+	if !reflect.DeepEqual(blockEnd, blockEndCheck) {
+		panic("Corrupted donut file")
+	}
+
+	fmt.Println("--verified")
+	fmt.Println("--end")
+}

pkg/storage/donut/v1/donut_test.go

@@ -32,11 +32,10 @@ func (s *MySuite) TestAPISuite(c *C) {
 	var b bytes.Buffer
 	var o bytes.Buffer
 
-	donut := Donut{&b}
+	donut := New(&b)
 	gobheader := GobHeader{}
 	err := donut.Write(gobheader, &o)
 	c.Assert(err, IsNil)
-	c.Log(b.Bytes())
 	blockStart := make([]byte, 4)
 	blockEnd := make([]byte, 4)