mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
Fix hanging erasure writes (#12253)
However, this slice is also used for closing the writers, so close is never called on these. Furthermore when an error is returned from a write it is now reported to the reader. bonus: remove unused heal param from `newBitrotWriter`. * Remove copy, now that we don't mutate.
This commit is contained in:
parent
55375fa7f6
commit
cde6469b88
@ -57,9 +57,15 @@ func (b *streamingBitrotWriter) Write(p []byte) (int, error) {
|
|||||||
hashBytes := b.h.Sum(nil)
|
hashBytes := b.h.Sum(nil)
|
||||||
_, err := b.iow.Write(hashBytes)
|
_, err := b.iow.Write(hashBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
b.closeWithErr(err)
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
return b.iow.Write(p)
|
n, err := b.iow.Write(p)
|
||||||
|
if err != nil {
|
||||||
|
b.closeWithErr(err)
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
return n, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *streamingBitrotWriter) Close() error {
|
func (b *streamingBitrotWriter) Close() error {
|
||||||
@ -77,13 +83,17 @@ func (b *streamingBitrotWriter) Close() error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Returns streaming bitrot writer implementation.
|
// newStreamingBitrotWriterBuffer returns streaming bitrot writer implementation.
|
||||||
func newStreamingBitrotWriterBuffer(w io.Writer, algo BitrotAlgorithm, shardSize int64) io.WriteCloser {
|
// The output is written to the supplied writer w.
|
||||||
return &streamingBitrotWriter{iow: ioutil.NopCloser(w), h: algo.New(), shardSize: shardSize, canClose: nil}
|
func newStreamingBitrotWriterBuffer(w io.Writer, algo BitrotAlgorithm, shardSize int64) io.Writer {
|
||||||
|
return &streamingBitrotWriter{iow: ioutil.NopCloser(w), h: algo.New(), shardSize: shardSize, canClose: nil, closeWithErr: func(err error) error {
|
||||||
|
// Similar to CloseWithError on pipes we always return nil.
|
||||||
|
return nil
|
||||||
|
}}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Returns streaming bitrot writer implementation.
|
// Returns streaming bitrot writer implementation.
|
||||||
func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64, heal bool) io.Writer {
|
func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.Writer {
|
||||||
r, w := io.Pipe()
|
r, w := io.Pipe()
|
||||||
h := algo.New()
|
h := algo.New()
|
||||||
|
|
||||||
|
@ -100,9 +100,9 @@ func BitrotAlgorithmFromString(s string) (a BitrotAlgorithm) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func newBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64, heal bool) io.Writer {
|
func newBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.Writer {
|
||||||
if algo == HighwayHash256S {
|
if algo == HighwayHash256S {
|
||||||
return newStreamingBitrotWriter(disk, volume, filePath, length, algo, shardSize, heal)
|
return newStreamingBitrotWriter(disk, volume, filePath, length, algo, shardSize)
|
||||||
}
|
}
|
||||||
return newWholeBitrotWriter(disk, volume, filePath, algo, shardSize)
|
return newWholeBitrotWriter(disk, volume, filePath, algo, shardSize)
|
||||||
}
|
}
|
||||||
|
@ -42,7 +42,7 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
|
|||||||
|
|
||||||
disk.MakeVol(context.Background(), volume)
|
disk.MakeVol(context.Background(), volume)
|
||||||
|
|
||||||
writer := newBitrotWriter(disk, volume, filePath, 35, bitrotAlgo, 10, false)
|
writer := newBitrotWriter(disk, volume, filePath, 35, bitrotAlgo, 10)
|
||||||
|
|
||||||
_, err = writer.Write([]byte("aaaaaaaaaa"))
|
_, err = writer.Write([]byte("aaaaaaaaaa"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -20,13 +20,12 @@ package cmd
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
crand "crypto/rand"
|
||||||
"io"
|
"io"
|
||||||
"math/rand"
|
"math/rand"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
crand "crypto/rand"
|
"github.com/dustin/go-humanize"
|
||||||
|
|
||||||
humanize "github.com/dustin/go-humanize"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func (a badDisk) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
|
func (a badDisk) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
|
||||||
@ -109,8 +108,7 @@ func TestErasureDecode(t *testing.T) {
|
|||||||
buffer := make([]byte, test.blocksize, 2*test.blocksize)
|
buffer := make([]byte, test.blocksize, 2*test.blocksize)
|
||||||
writers := make([]io.Writer, len(disks))
|
writers := make([]io.Writer, len(disks))
|
||||||
for i, disk := range disks {
|
for i, disk := range disks {
|
||||||
writers[i] = newBitrotWriter(disk, "testbucket", "object",
|
writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(test.data), writeAlgorithm, erasure.ShardSize())
|
||||||
erasure.ShardFileSize(test.data), writeAlgorithm, erasure.ShardSize(), false)
|
|
||||||
}
|
}
|
||||||
n, err := erasure.Encode(context.Background(), bytes.NewReader(data[:]), writers, buffer, erasure.dataBlocks+1)
|
n, err := erasure.Encode(context.Background(), bytes.NewReader(data[:]), writers, buffer, erasure.dataBlocks+1)
|
||||||
closeBitrotWriters(writers)
|
closeBitrotWriters(writers)
|
||||||
@ -236,8 +234,7 @@ func TestErasureDecodeRandomOffsetLength(t *testing.T) {
|
|||||||
if disk == nil {
|
if disk == nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
writers[i] = newBitrotWriter(disk, "testbucket", "object",
|
writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(length), DefaultBitrotAlgorithm, erasure.ShardSize())
|
||||||
erasure.ShardFileSize(length), DefaultBitrotAlgorithm, erasure.ShardSize(), false)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// 10000 iterations with random offsets and lengths.
|
// 10000 iterations with random offsets and lengths.
|
||||||
@ -307,8 +304,7 @@ func benchmarkErasureDecode(data, parity, dataDown, parityDown int, size int64,
|
|||||||
if disk == nil {
|
if disk == nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
writers[i] = newBitrotWriter(disk, "testbucket", "object",
|
writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize())
|
||||||
erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize(), false)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
content := make([]byte, size)
|
content := make([]byte, size)
|
||||||
|
@ -20,7 +20,6 @@ package cmd
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"io"
|
"io"
|
||||||
|
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"github.com/minio/minio/cmd/logger"
|
"github.com/minio/minio/cmd/logger"
|
||||||
|
@ -109,7 +109,7 @@ func TestErasureEncode(t *testing.T) {
|
|||||||
if disk == OfflineDisk {
|
if disk == OfflineDisk {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(int64(len(data[test.offset:]))), test.algorithm, erasure.ShardSize(), false)
|
writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(int64(len(data[test.offset:]))), test.algorithm, erasure.ShardSize())
|
||||||
}
|
}
|
||||||
n, err := erasure.Encode(context.Background(), bytes.NewReader(data[test.offset:]), writers, buffer, erasure.dataBlocks+1)
|
n, err := erasure.Encode(context.Background(), bytes.NewReader(data[test.offset:]), writers, buffer, erasure.dataBlocks+1)
|
||||||
closeBitrotWriters(writers)
|
closeBitrotWriters(writers)
|
||||||
@ -133,7 +133,7 @@ func TestErasureEncode(t *testing.T) {
|
|||||||
if disk == nil {
|
if disk == nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
writers[i] = newBitrotWriter(disk, "testbucket", "object2", erasure.ShardFileSize(int64(len(data[test.offset:]))), test.algorithm, erasure.ShardSize(), false)
|
writers[i] = newBitrotWriter(disk, "testbucket", "object2", erasure.ShardFileSize(int64(len(data[test.offset:]))), test.algorithm, erasure.ShardSize())
|
||||||
}
|
}
|
||||||
for j := range disks[:test.offDisks] {
|
for j := range disks[:test.offDisks] {
|
||||||
switch w := writers[j].(type) {
|
switch w := writers[j].(type) {
|
||||||
@ -197,8 +197,7 @@ func benchmarkErasureEncode(data, parity, dataDown, parityDown int, size int64,
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
disk.Delete(context.Background(), "testbucket", "object", false)
|
disk.Delete(context.Background(), "testbucket", "object", false)
|
||||||
writers[i] = newBitrotWriter(disk, "testbucket", "object",
|
writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize())
|
||||||
erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize(), false)
|
|
||||||
}
|
}
|
||||||
_, err := erasure.Encode(context.Background(), bytes.NewReader(content), writers, buffer, erasure.dataBlocks+1)
|
_, err := erasure.Encode(context.Background(), bytes.NewReader(content), writers, buffer, erasure.dataBlocks+1)
|
||||||
closeBitrotWriters(writers)
|
closeBitrotWriters(writers)
|
||||||
|
@ -91,8 +91,7 @@ func TestErasureHeal(t *testing.T) {
|
|||||||
buffer := make([]byte, test.blocksize, 2*test.blocksize)
|
buffer := make([]byte, test.blocksize, 2*test.blocksize)
|
||||||
writers := make([]io.Writer, len(disks))
|
writers := make([]io.Writer, len(disks))
|
||||||
for i, disk := range disks {
|
for i, disk := range disks {
|
||||||
writers[i] = newBitrotWriter(disk, "testbucket", "testobject",
|
writers[i] = newBitrotWriter(disk, "testbucket", "testobject", erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize())
|
||||||
erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize(), true)
|
|
||||||
}
|
}
|
||||||
_, err = erasure.Encode(context.Background(), bytes.NewReader(data), writers, buffer, erasure.dataBlocks+1)
|
_, err = erasure.Encode(context.Background(), bytes.NewReader(data), writers, buffer, erasure.dataBlocks+1)
|
||||||
closeBitrotWriters(writers)
|
closeBitrotWriters(writers)
|
||||||
@ -135,8 +134,7 @@ func TestErasureHeal(t *testing.T) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
os.Remove(pathJoin(disk.String(), "testbucket", "testobject"))
|
os.Remove(pathJoin(disk.String(), "testbucket", "testobject"))
|
||||||
staleWriters[i] = newBitrotWriter(disk, "testbucket", "testobject",
|
staleWriters[i] = newBitrotWriter(disk, "testbucket", "testobject", erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize())
|
||||||
erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize(), true)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Number of buffers, max 2GB
|
// Number of buffers, max 2GB
|
||||||
|
@ -466,7 +466,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
|
|||||||
writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
|
writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
|
||||||
} else {
|
} else {
|
||||||
writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, partPath,
|
writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, partPath,
|
||||||
tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize(), true)
|
tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
err = erasure.Heal(ctx, readers, writers, partSize, er.bp)
|
err = erasure.Heal(ctx, readers, writers, partSize, er.bp)
|
||||||
|
@ -482,8 +482,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
|
|||||||
if disk == nil {
|
if disk == nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, tmpPartPath,
|
writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, tmpPartPath, erasure.ShardFileSize(data.Size()), DefaultBitrotAlgorithm, erasure.ShardSize())
|
||||||
erasure.ShardFileSize(data.Size()), DefaultBitrotAlgorithm, erasure.ShardSize(), false)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
n, err := erasure.Encode(rctx, data, writers, buffer, writeQuorum)
|
n, err := erasure.Encode(rctx, data, writers, buffer, writeQuorum)
|
||||||
|
@ -733,8 +733,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
|||||||
writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
|
writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, tempErasureObj,
|
writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, tempErasureObj, shardFileSize, DefaultBitrotAlgorithm, erasure.ShardSize())
|
||||||
shardFileSize, DefaultBitrotAlgorithm, erasure.ShardSize(), false)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
n, erasureErr := erasure.Encode(ctx, data, writers, buffer, writeQuorum)
|
n, erasureErr := erasure.Encode(ctx, data, writers, buffer, writeQuorum)
|
||||||
|
@ -1787,7 +1787,7 @@ func TestXLStorageVerifyFile(t *testing.T) {
|
|||||||
algo = HighwayHash256S
|
algo = HighwayHash256S
|
||||||
shardSize := int64(1024 * 1024)
|
shardSize := int64(1024 * 1024)
|
||||||
shard := make([]byte, shardSize)
|
shard := make([]byte, shardSize)
|
||||||
w := newStreamingBitrotWriter(storage, volName, fileName, size, algo, shardSize, false)
|
w := newStreamingBitrotWriter(storage, volName, fileName, size, algo, shardSize)
|
||||||
reader := bytes.NewReader(data)
|
reader := bytes.NewReader(data)
|
||||||
for {
|
for {
|
||||||
// Using io.Copy instead of this loop will not work for us as io.Copy
|
// Using io.Copy instead of this loop will not work for us as io.Copy
|
||||||
|
Loading…
Reference in New Issue
Block a user