MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-08 21:24:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

[feat] Add configurable deadline for writers (#11822)

Browse Source 
This PR adds deadlines per Write() calls, such
that slow drives are timed-out appropriately and
the overall responsiveness for Writes() is always
up to a predefined threshold providing applications
sustained latency even if one of the drives is slow
to respond.
This commit is contained in:
Harshavardhana
2021-03-18 14:09:55 -07:00
committed by GitHub
parent d46c3c07a8
commit 51a8619a79
16 changed files with 191 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
cmd/xl-storage_test.go
View File

@@ -1730,7 +1730,7 @@ func TestXLStorageVerifyFile(t *testing.T) {
// 4) Streaming bitrot check on corrupted file
// create xlStorage test setup
xlStorage, path, err := newXLStorageTestSetup()
storage, path, err := newXLStorageTestSetup()
if err != nil {
t.Fatalf("Unable to create xlStorage test setup, %s", err)
}
@@ -1738,7 +1738,7 @@ func TestXLStorageVerifyFile(t *testing.T) {
volName := "testvol"
fileName := "testfile"
if err := xlStorage.MakeVol(context.Background(), volName); err != nil {
if err := storage.MakeVol(context.Background(), volName); err != nil {
t.Fatal(err)
}
@@ -1752,29 +1752,29 @@ func TestXLStorageVerifyFile(t *testing.T) {
h := algo.New()
h.Write(data)
hashBytes := h.Sum(nil)
if err := xlStorage.WriteAll(context.Background(), volName, fileName, data); err != nil {
if err := storage.WriteAll(context.Background(), volName, fileName, data); err != nil {
t.Fatal(err)
}
if err := xlStorage.storage.bitrotVerify(pathJoin(path, volName, fileName), size, algo, hashBytes, 0); err != nil {
if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size, algo, hashBytes, 0); err != nil {
t.Fatal(err)
}
// 2) Whole-file bitrot check on corrupted file
if err := xlStorage.AppendFile(context.Background(), volName, fileName, []byte("a")); err != nil {
if err := storage.AppendFile(context.Background(), volName, fileName, []byte("a")); err != nil {
t.Fatal(err)
}
// Check if VerifyFile reports the incorrect file length (the correct length is `size+1`)
if err := xlStorage.storage.bitrotVerify(pathJoin(path, volName, fileName), size, algo, hashBytes, 0); err == nil {
if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size, algo, hashBytes, 0); err == nil {
t.Fatal("expected to fail bitrot check")
}
// Check if bitrot fails
if err := xlStorage.storage.bitrotVerify(pathJoin(path, volName, fileName), size+1, algo, hashBytes, 0); err == nil {
if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size+1, algo, hashBytes, 0); err == nil {
t.Fatal("expected to fail bitrot check")
}
if err := xlStorage.Delete(context.Background(), volName, fileName, false); err != nil {
if err := storage.Delete(context.Background(), volName, fileName, false); err != nil {
t.Fatal(err)
}
@@ -1782,7 +1782,7 @@ func TestXLStorageVerifyFile(t *testing.T) {
algo = HighwayHash256S
shardSize := int64(1024 * 1024)
shard := make([]byte, shardSize)
w := newStreamingBitrotWriter(xlStorage, volName, fileName, size, algo, shardSize)
w := newStreamingBitrotWriter(storage, volName, fileName, size, algo, shardSize, false)
reader := bytes.NewReader(data)
for {
// Using io.Copy instead of this loop will not work for us as io.Copy
@@ -1798,13 +1798,13 @@ func TestXLStorageVerifyFile(t *testing.T) {
}
t.Fatal(err)
}
w.Close()
if err := xlStorage.storage.bitrotVerify(pathJoin(path, volName, fileName), size, algo, nil, shardSize); err != nil {
w.(io.Closer).Close()
if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size, algo, nil, shardSize); err != nil {
t.Fatal(err)
}
// 4) Streaming bitrot check on corrupted file
filePath := pathJoin(xlStorage.String(), volName, fileName)
filePath := pathJoin(storage.String(), volName, fileName)
f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_SYNC, 0644)
if err != nil {
t.Fatal(err)
@@ -1814,10 +1814,10 @@ func TestXLStorageVerifyFile(t *testing.T) {
t.Fatal(err)
}
f.Close()
if err := xlStorage.storage.bitrotVerify(pathJoin(path, volName, fileName), size, algo, nil, shardSize); err == nil {
if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size, algo, nil, shardSize); err == nil {
t.Fatal("expected to fail bitrot check")
}
if err := xlStorage.storage.bitrotVerify(pathJoin(path, volName, fileName), size+1, algo, nil, shardSize); err == nil {
if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size+1, algo, nil, shardSize); err == nil {
t.Fatal("expected to fail bitrot check")
}
}