Replace snappy.Writer/io.Pipe with snappyCompressReader. (#7316)

Prevents deferred close functions from being called while still
  attempting to copy reader to snappyWriter.
 Reduces code duplication when compressing objects.
This commit is contained in:
Kale Blankenship
2019-03-05 08:35:37 -08:00
committed by kannappanr
parent c54b0c0ca1
commit ef132c5714
4 changed files with 119 additions and 71 deletions

View File

@@ -675,3 +675,54 @@ func CleanMinioInternalMetadataKeys(metadata map[string]string) map[string]strin
}
return newMeta
}
// snappyCompressReader compresses data as it reads
// from the underlying io.Reader.
type snappyCompressReader struct {
r io.Reader
w *snappy.Writer
closed bool
buf bytes.Buffer
}
func newSnappyCompressReader(r io.Reader) *snappyCompressReader {
cr := &snappyCompressReader{r: r}
cr.w = snappy.NewBufferedWriter(&cr.buf)
return cr
}
func (cr *snappyCompressReader) Read(p []byte) (int, error) {
if cr.closed {
// if snappy writer is closed r has been completely read,
// return any remaining data in buf.
return cr.buf.Read(p)
}
// read from original using p as buffer
nr, readErr := cr.r.Read(p)
// write read bytes to snappy writer
nw, err := cr.w.Write(p[:nr])
if err != nil {
return 0, err
}
if nw != nr {
return 0, io.ErrShortWrite
}
// if last of data from reader, close snappy writer to flush
if readErr == io.EOF {
err := cr.w.Close()
cr.closed = true
if err != nil {
return 0, err
}
}
// read compressed bytes out of buf
n, err := cr.buf.Read(p)
if readErr != io.EOF && (err == nil || err == io.EOF) {
err = readErr
}
return n, err
}