From 3151f9b6c3cd8e2a006fcf393fa3baf92c8f443a Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sat, 21 Feb 2015 00:27:29 -0800 Subject: [PATCH] Add proper entries for GobHeader, Data copy. This commit also adds 'donut_gen.go' for generating sample donut file --- pkg/storage/donut/v1/.gitignore | 1 + pkg/storage/donut/v1/donut.go | 117 +++++++++++++++++++++-------- pkg/storage/donut/v1/donut_gen.go | 63 ++++++++++++++++ pkg/storage/donut/v1/donut_test.go | 3 +- 4 files changed, 152 insertions(+), 32 deletions(-) create mode 100644 pkg/storage/donut/v1/.gitignore create mode 100644 pkg/storage/donut/v1/donut_gen.go diff --git a/pkg/storage/donut/v1/.gitignore b/pkg/storage/donut/v1/.gitignore new file mode 100644 index 000000000..07be1efd3 --- /dev/null +++ b/pkg/storage/donut/v1/.gitignore @@ -0,0 +1 @@ +newfile \ No newline at end of file diff --git a/pkg/storage/donut/v1/donut.go b/pkg/storage/donut/v1/donut.go index d0eaaa669..eb5de53c5 100644 --- a/pkg/storage/donut/v1/donut.go +++ b/pkg/storage/donut/v1/donut.go @@ -17,9 +17,14 @@ package v1 import ( + "bytes" "encoding/binary" + "encoding/gob" "errors" "io" + "sync" + + "github.com/minio-io/minio/pkg/storage/erasure" ) /* @@ -55,7 +60,7 @@ type DonutFormat struct { VersionReserved uint16 Reserved uint64 GobHeaderLen uint32 - GobHeader GobHeader + GobHeader []byte BlockData uint32 // Magic="DATA"=1096040772 Data io.Reader BlockLen uint64 @@ -68,36 +73,61 @@ type DonutFooter struct { } type Donut struct { - file io.Writer - // mutex + file io.Writer + mutex *sync.RWMutex } -type GobHeader struct{} +type GobHeader struct { + Blocks []EncodedChunk + Md5sum []byte + EncoderParams erasure.EncoderParams +} -func (donut *Donut) Write(gobHeader GobHeader, object io.Reader) error { - // TODO mutex - // Create bytes buffer representing the new object - donutFormat := DonutFormat{ - BlockStart: MagicMINI, - VersionMajor: 1, - VersionMinor: 0, - VersionPatch: 0, - VersionReserved: 0, - Reserved: 0, - GobHeaderLen: 0, - GobHeader: gobHeader, - BlockData: MagicDATA, - Data: object, - BlockLen: 0, - BlockEnd: MagicINIM, +type EncodedChunk struct { + Crc uint32 + Length int + Offset int +} + +func New(file io.Writer) *Donut { + donut := Donut{} + donut.mutex = new(sync.RWMutex) + donut.file = file + return &donut +} + +func (donut *Donut) WriteGob(gobHeader GobHeader) (bytes.Buffer, error) { + var gobBuffer bytes.Buffer + encoder := gob.NewEncoder(&gobBuffer) + err := encoder.Encode(gobHeader) + if err != nil { + return bytes.Buffer{}, err } - if err := donut.WriteFormat(donut.file, donutFormat); err != nil { + return gobBuffer, nil +} + +func (donut *Donut) WriteEnd(target io.Writer, donutFormat DonutFormat) error { + if err := binary.Write(target, binary.LittleEndian, donutFormat.BlockLen); err != nil { + return err + } + if err := binary.Write(target, binary.LittleEndian, donutFormat.BlockEnd); err != nil { return err } return nil } -func (donut *Donut) WriteFormat(target io.Writer, donutFormat DonutFormat) error { +func (donut *Donut) WriteData(target io.Writer, donutFormat DonutFormat) error { + var b bytes.Buffer + if count, err := io.Copy(&b, donutFormat.Data); uint64(count) != donutFormat.BlockLen || err != nil { + if err == nil { + return binary.Write(target, binary.LittleEndian, b.Bytes()) + } + return errors.New("Copy failed, count incorrect.") + } + return nil +} + +func (donut *Donut) WriteBegin(target io.Writer, donutFormat DonutFormat) error { if err := binary.Write(target, binary.LittleEndian, donutFormat.BlockStart); err != nil { return err } @@ -125,16 +155,43 @@ func (donut *Donut) WriteFormat(target io.Writer, donutFormat DonutFormat) error if err := binary.Write(target, binary.LittleEndian, donutFormat.BlockData); err != nil { return err } - if count, err := io.Copy(target, donutFormat.Data); uint64(count) != donutFormat.BlockLen || err != nil { - if err == nil { - return err - } - return errors.New("Copy failed, count incorrect.") - } - if err := binary.Write(target, binary.LittleEndian, donutFormat.BlockLen); err != nil { + return nil +} + +func (donut *Donut) Write(gobHeader GobHeader, object io.Reader) error { + donut.mutex.Lock() + defer donut.mutex.Unlock() + + gobBytes, err := donut.WriteGob(gobHeader) + if err != nil { return err } - if err := binary.Write(target, binary.LittleEndian, donutFormat.BlockEnd); err != nil { + + // Create bytes buffer representing the new object + donutFormat := DonutFormat{ + BlockStart: MagicMINI, + VersionMajor: 1, + VersionMinor: 0, + VersionPatch: 0, + VersionReserved: 0, + Reserved: 0, + GobHeaderLen: uint32(gobBytes.Len()), + GobHeader: gobBytes.Bytes(), + BlockData: MagicDATA, + Data: object, + BlockLen: 0, + BlockEnd: MagicINIM, + } + + if err := donut.WriteBegin(donut.file, donutFormat); err != nil { + return err + } + + if err := donut.WriteData(donut.file, donutFormat); err != nil { + return err + } + + if err := donut.WriteEnd(donut.file, donutFormat); err != nil { return err } diff --git a/pkg/storage/donut/v1/donut_gen.go b/pkg/storage/donut/v1/donut_gen.go new file mode 100644 index 000000000..b95159f31 --- /dev/null +++ b/pkg/storage/donut/v1/donut_gen.go @@ -0,0 +1,63 @@ +// +build ignore + +package main + +import ( + "bytes" + "fmt" + "os" + "reflect" + + "github.com/minio-io/minio/pkg/storage/donut/v1" +) + +func main() { + fmt.Println("--start") + + file, err := os.OpenFile("newfile", os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + panic(err) + } + donut := v1.New(file) + + gobHeader := v1.GobHeader{} + data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.") + + dataBuffer := bytes.NewBuffer(data) + err = donut.Write(gobHeader, dataBuffer) + if err != nil { + panic(err) + } + file.Close() + fmt.Println("--closed") + + fmt.Println("--verify") + stat, _ := os.Stat("newfile") + fileSize := stat.Size() + + rfile, _ := os.OpenFile("newfile", os.O_RDONLY, 0666) + blockStart := make([]byte, 4) + blockStartCheck := []byte{'M', 'I', 'N', 'I'} + + _, err = rfile.Read(blockStart) + if err != nil { + panic(err) + } + + blockEnd := make([]byte, 4) + start := fileSize - 4 + blockEndCheck := []byte{'I', 'N', 'I', 'M'} + rfile.ReadAt(blockEnd, start) + rfile.Close() + + if !reflect.DeepEqual(blockStart, blockStartCheck) { + panic("Corrupted donut file") + } + + if !reflect.DeepEqual(blockEnd, blockEndCheck) { + panic("Corrupted donut file") + } + + fmt.Println("--verified") + fmt.Println("--end") +} diff --git a/pkg/storage/donut/v1/donut_test.go b/pkg/storage/donut/v1/donut_test.go index 04796fd1f..9963d0b22 100644 --- a/pkg/storage/donut/v1/donut_test.go +++ b/pkg/storage/donut/v1/donut_test.go @@ -32,11 +32,10 @@ func (s *MySuite) TestAPISuite(c *C) { var b bytes.Buffer var o bytes.Buffer - donut := Donut{&b} + donut := New(&b) gobheader := GobHeader{} err := donut.Write(gobheader, &o) c.Assert(err, IsNil) - c.Log(b.Bytes()) blockStart := make([]byte, 4) blockEnd := make([]byte, 4)