MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-03-30 09:13:41 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #131 from fkautz/pr_out_split_s_splitstream_now_creates_and_returns_a_read_only_channel

Browse Source 
Split's SplitStream now creates and returns a read only channel.
This commit is contained in:
Harshavardhana 2014-12-16 16:36:13 -08:00
commit eaf08edc5e
3 changed files with 18 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
pkgs/split/split.go
View File

@ -55,7 +55,13 @@ type JoinMessage struct {
// for chunk := range channel { // for chunk := range channel {
// log.Println(chunk.Data) // log.Println(chunk.Data)
// } // }
func SplitStream(reader io.Reader, chunkSize uint64, ch chan SplitMessage) { func SplitStream(reader io.Reader, chunkSize uint64) <-chan SplitMessage {
ch := make(chan SplitMessage)
go splitStreamGoRoutine(reader, chunkSize, ch)
return ch
}
func splitStreamGoRoutine(reader io.Reader, chunkSize uint64, ch chan SplitMessage) {
// we read until EOF or another error // we read until EOF or another error
var readError error var readError error
@ -95,7 +101,13 @@ func SplitStream(reader io.Reader, chunkSize uint64, ch chan SplitMessage) {
close(ch) close(ch)
} }
func JoinStream(dirname string, inputPrefix string, ch chan JoinMessage) { func JoinStream(dirname string, inputPrefix string) <-chan JoinMessage {
ch := make(chan JoinMessage)
go joinStreamGoRoutine(dirname, inputPrefix, ch)
return ch
}
func joinStreamGoRoutine(dirname string, inputPrefix string, ch chan JoinMessage) {
var readError error var readError error
var bytesBuffer bytes.Buffer var bytesBuffer bytes.Buffer
@ -147,8 +159,7 @@ func JoinFilesWithPrefix(dirname string, inputPrefix string, outputFile string)
return errors.New("Invalid output file") return errors.New("Invalid output file")
} }
ch := make(chan JoinMessage) ch := JoinStream(dirname, inputPrefix)
go JoinStream(dirname, inputPrefix, ch)
var multiReaders []io.Reader var multiReaders []io.Reader
var aggregatedLength int64 var aggregatedLength int64
@ -194,8 +205,7 @@ func SplitFilesWithPrefix(filename string, chunkstr string, outputPrefix string)
} }
// start stream splitting goroutine // start stream splitting goroutine
ch := make(chan SplitMessage) ch := SplitStream(file, chunkSize)
go SplitStream(file, chunkSize, ch)
// used to write each chunk out as a separate file. {{outputPrefix}}.{{i}} // used to write each chunk out as a separate file. {{outputPrefix}}.{{i}}
i := 0 i := 0

3
pkgs/split/split_test.go
View File

@ -38,9 +38,8 @@ func (s *MySuite) TestSplitStream(c *C) {
bytesWriter.Write([]byte(strconv.Itoa(i))) bytesWriter.Write([]byte(strconv.Itoa(i)))
} }
bytesWriter.Flush() bytesWriter.Flush()
ch := make(chan SplitMessage)
reader := bytes.NewReader(bytesBuffer.Bytes()) reader := bytes.NewReader(bytesBuffer.Bytes())
go SplitStream(reader, 25, ch) ch := SplitStream(reader, 25)
var resultsBuffer bytes.Buffer var resultsBuffer bytes.Buffer
resultsWriter := bufio.NewWriter(&resultsBuffer) resultsWriter := bufio.NewWriter(&resultsBuffer)
for chunk := range ch { for chunk := range ch {

3
pkgs/storage/encodedstorage/encoded_storage.go
View File

@ -136,8 +136,7 @@ func (eStorage *encodedStorage) List(objectPath string) ([]storage.ObjectDescrip
func (eStorage *encodedStorage) Put(objectPath string, object io.Reader) error { func (eStorage *encodedStorage) Put(objectPath string, object io.Reader) error {
// split // split
chunks := make(chan split.SplitMessage) chunks := split.SplitStream(object, eStorage.BlockSize)
go split.SplitStream(object, eStorage.BlockSize, chunks)
// for each chunk // for each chunk
encoderParameters, err := erasure.ParseEncoderParams(eStorage.K, eStorage.M, erasure.CAUCHY) encoderParameters, err := erasure.ParseEncoderParams(eStorage.K, eStorage.M, erasure.CAUCHY)