MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-04-30 14:47:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Add storage layer contexts (#10321)

Browse Source 
Add context to all (non-trivial) calls to the storage layer. 

Contexts are propagated through the REST client.

- `context.TODO()` is left in place for the places where it needs to be added to the caller.
- `endWalkCh` could probably be removed from the walkers, but no changes so far.

The "dangerous" part is that now a caller disconnecting *will* propagate down,  so a 
"delete" operation will now be interrupted. In some cases we might want to disconnect 
this functionality so the operation completes if it has started, leaving the system in a cleaner state.
This commit is contained in:
Klaus Post 2020-09-04 09:45:06 -07:00 committed by GitHub
parent 0037951b6e
commit 2d58a8d861
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
36 changed files with 466 additions and 467 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cmd/bitrot-streaming.go
View File

@ -18,6 +18,7 @@ package cmd
import ( import (
"bytes" "bytes"
"context"
"encoding/hex" "encoding/hex"
"fmt" "fmt"
"hash" "hash"
@ -80,7 +81,7 @@ func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length i
bitrotSumsTotalSize := ceilFrac(length, shardSize) * int64(h.Size()) // Size used for storing bitrot checksums. bitrotSumsTotalSize := ceilFrac(length, shardSize) * int64(h.Size()) // Size used for storing bitrot checksums.
totalFileSize = bitrotSumsTotalSize + length totalFileSize = bitrotSumsTotalSize + length
} }
err := disk.CreateFile(volume, filePath, totalFileSize, r) err := disk.CreateFile(context.TODO(), volume, filePath, totalFileSize, r)
r.CloseWithError(err) r.CloseWithError(err)
close(bw.canClose) close(bw.canClose)
}() }()
@ -118,7 +119,7 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
// For the first ReadAt() call we need to open the stream for reading. // For the first ReadAt() call we need to open the stream for reading.
b.currOffset = offset b.currOffset = offset
streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
b.rc, err = b.disk.ReadFileStream(b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset) b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
if err != nil { if err != nil {
return 0, err return 0, err
} }

5
cmd/bitrot-whole.go
View File

@ -17,6 +17,7 @@
package cmd package cmd
import ( import (
"context"
"fmt" "fmt"
"hash" "hash"
"io" "io"
@ -34,7 +35,7 @@ type wholeBitrotWriter struct {
} }
func (b *wholeBitrotWriter) Write(p []byte) (int, error) { func (b *wholeBitrotWriter) Write(p []byte) (int, error) {
err := b.disk.AppendFile(b.volume, b.filePath, p) err := b.disk.AppendFile(context.TODO(), b.volume, b.filePath, p)
if err != nil { if err != nil {
logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err)) logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err))
return 0, err return 0, err
@ -69,7 +70,7 @@ type wholeBitrotReader struct {
func (b *wholeBitrotReader) ReadAt(buf []byte, offset int64) (n int, err error) { func (b *wholeBitrotReader) ReadAt(buf []byte, offset int64) (n int, err error) {
if b.buf == nil { if b.buf == nil {
b.buf = make([]byte, b.tillOffset-offset) b.buf = make([]byte, b.tillOffset-offset)
if _, err := b.disk.ReadFile(b.volume, b.filePath, offset, b.buf, b.verifier); err != nil { if _, err := b.disk.ReadFile(context.TODO(), b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err)) logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err))
return 0, err return 0, err
} }

3
cmd/bitrot_test.go
View File

@ -17,6 +17,7 @@
package cmd package cmd
import ( import (
"context"
"io" "io"
"io/ioutil" "io/ioutil"
"log" "log"
@ -39,7 +40,7 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
t.Fatal(err) t.Fatal(err)
} }
disk.MakeVol(volume) disk.MakeVol(context.Background(), volume)
writer := newBitrotWriter(disk, volume, filePath, 35, bitrotAlgo, 10) writer := newBitrotWriter(disk, volume, filePath, 35, bitrotAlgo, 10)

4
cmd/bootstrap-peer-server.go
View File

@ -131,7 +131,7 @@ func (client *bootstrapRESTClient) callWithContext(ctx context.Context, method s
values = make(url.Values) values = make(url.Values)
} }
respBody, err = client.restClient.CallWithContext(ctx, method, values, body, length) respBody, err = client.restClient.Call(ctx, method, values, body, length)
if err == nil { if err == nil {
return respBody, nil return respBody, nil
} }
@ -234,7 +234,7 @@ func newBootstrapRESTClient(endpoint Endpoint) *bootstrapRESTClient {
ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout) ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
// Instantiate a new rest client for healthcheck // Instantiate a new rest client for healthcheck
// to avoid recursive healthCheckFn() // to avoid recursive healthCheckFn()
respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).CallWithContext(ctx, bootstrapRESTMethodHealth, nil, nil, -1) respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).Call(ctx, bootstrapRESTMethodHealth, nil, nil, -1)
xhttp.DrainBody(respBody) xhttp.DrainBody(respBody)
cancel() cancel()
var ne *rest.NetworkError var ne *rest.NetworkError

14
cmd/erasure-bucket.go
View File

@ -49,7 +49,7 @@ func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket stri
index := index index := index
g.Go(func() error { g.Go(func() error {
if storageDisks[index] != nil { if storageDisks[index] != nil {
if err := storageDisks[index].MakeVol(bucket); err != nil { if err := storageDisks[index].MakeVol(ctx, bucket); err != nil {
if err != errVolumeExists { if err != errVolumeExists {
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
} }
@ -75,7 +75,7 @@ func undoDeleteBucket(storageDisks []StorageAPI, bucket string) {
} }
index := index index := index
g.Go(func() error { g.Go(func() error {
_ = storageDisks[index].MakeVol(bucket) _ = storageDisks[index].MakeVol(context.Background(), bucket)
return nil return nil
}, index) }, index)
} }
@ -92,7 +92,7 @@ func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string) (
bucketErrs = append(bucketErrs, errDiskNotFound) bucketErrs = append(bucketErrs, errDiskNotFound)
continue continue
} }
volInfo, serr := disk.StatVol(bucketName) volInfo, serr := disk.StatVol(ctx, bucketName)
if serr == nil { if serr == nil {
return BucketInfo(volInfo), nil return BucketInfo(volInfo), nil
} }
@ -129,7 +129,7 @@ func (er erasureObjects) listBuckets(ctx context.Context) (bucketsInfo []BucketI
continue continue
} }
var volsInfo []VolInfo var volsInfo []VolInfo
volsInfo, err = disk.ListVols() volsInfo, err = disk.ListVols(ctx)
if err == nil { if err == nil {
// NOTE: The assumption here is that volumes across all disks in // NOTE: The assumption here is that volumes across all disks in
// readQuorum have consistent view i.e they all have same number // readQuorum have consistent view i.e they all have same number
@ -181,10 +181,10 @@ func deleteDanglingBucket(ctx context.Context, storageDisks []StorageAPI, dErrs
for index, err := range dErrs { for index, err := range dErrs {
if err == errVolumeNotEmpty { if err == errVolumeNotEmpty {
// Attempt to delete bucket again. // Attempt to delete bucket again.
if derr := storageDisks[index].DeleteVol(bucket, false); derr == errVolumeNotEmpty { if derr := storageDisks[index].DeleteVol(ctx, bucket, false); derr == errVolumeNotEmpty {
_ = cleanupDir(ctx, storageDisks[index], bucket, "") _ = cleanupDir(ctx, storageDisks[index], bucket, "")
_ = storageDisks[index].DeleteVol(bucket, false) _ = storageDisks[index].DeleteVol(ctx, bucket, false)
// Cleanup all the previously incomplete multiparts. // Cleanup all the previously incomplete multiparts.
_ = cleanupDir(ctx, storageDisks[index], minioMetaMultipartBucket, bucket) _ = cleanupDir(ctx, storageDisks[index], minioMetaMultipartBucket, bucket)
@ -204,7 +204,7 @@ func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, forceD
index := index index := index
g.Go(func() error { g.Go(func() error {
if storageDisks[index] != nil { if storageDisks[index] != nil {
if err := storageDisks[index].DeleteVol(bucket, forceDelete); err != nil { if err := storageDisks[index].DeleteVol(ctx, bucket, forceDelete); err != nil {
return err return err
} }
err := cleanupDir(ctx, storageDisks[index], minioMetaMultipartBucket, bucket) err := cleanupDir(ctx, storageDisks[index], minioMetaMultipartBucket, bucket)

2
cmd/erasure-common.go
View File

@ -77,7 +77,7 @@ func (er erasureObjects) isObject(ctx context.Context, bucket, prefix string) (o
return errDiskNotFound return errDiskNotFound
} }
// Check if 'prefix' is an object on this 'disk', else continue the check the next disk // Check if 'prefix' is an object on this 'disk', else continue the check the next disk
return storageDisks[index].CheckFile(bucket, prefix) return storageDisks[index].CheckFile(ctx, bucket, prefix)
}, index) }, index)
} }

2
cmd/erasure-decode_test.go
View File

@ -28,7 +28,7 @@ import (
humanize "github.com/dustin/go-humanize" humanize "github.com/dustin/go-humanize"
) )
func (a badDisk) ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) { func (a badDisk) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
return 0, errFaultyDisk return 0, errFaultyDisk
} }

8
cmd/erasure-encode_test.go
View File

@ -32,11 +32,11 @@ func (a badDisk) String() string {
return "bad-disk" return "bad-disk"
} }
func (a badDisk) AppendFile(volume string, path string, buf []byte) error { func (a badDisk) AppendFile(ctx context.Context, volume string, path string, buf []byte) error {
return errFaultyDisk return errFaultyDisk
} }
func (a badDisk) ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error) { func (a badDisk) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) {
return nil, errFaultyDisk return nil, errFaultyDisk
} }
@ -44,7 +44,7 @@ func (a badDisk) UpdateBloomFilter(ctx context.Context, oldest, current uint64)
return nil, errFaultyDisk return nil, errFaultyDisk
} }
func (a badDisk) CreateFile(volume, path string, size int64, reader io.Reader) error { func (a badDisk) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error {
return errFaultyDisk return errFaultyDisk
} }
@ -195,7 +195,7 @@ func benchmarkErasureEncode(data, parity, dataDown, parityDown int, size int64,
if disk == OfflineDisk { if disk == OfflineDisk {
continue continue
} }
disk.DeleteFile("testbucket", "object") disk.DeleteFile(context.Background(), "testbucket", "object")
writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize()) writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize())
} }
_, err := erasure.Encode(context.Background(), bytes.NewReader(content), writers, buffer, erasure.dataBlocks+1) _, err := erasure.Encode(context.Background(), bytes.NewReader(content), writers, buffer, erasure.dataBlocks+1)

4
cmd/erasure-healing-common.go
View File

@ -175,9 +175,9 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
// disk has a valid xl.meta but may not have all the // disk has a valid xl.meta but may not have all the
// parts. This is considered an outdated disk, since // parts. This is considered an outdated disk, since
// it needs healing too. // it needs healing too.
dataErrs[i] = onlineDisk.VerifyFile(bucket, object, partsMetadata[i]) dataErrs[i] = onlineDisk.VerifyFile(ctx, bucket, object, partsMetadata[i])
case madmin.HealNormalScan: case madmin.HealNormalScan:
dataErrs[i] = onlineDisk.CheckParts(bucket, object, partsMetadata[i]) dataErrs[i] = onlineDisk.CheckParts(ctx, bucket, object, partsMetadata[i])
} }
if dataErrs[i] == nil { if dataErrs[i] == nil {

2
cmd/erasure-healing-common_test.go
View File

@ -210,7 +210,7 @@ func TestListOnlineDisks(t *testing.T) {
// and check if that disk // and check if that disk
// appears in outDatedDisks. // appears in outDatedDisks.
tamperedIndex = index tamperedIndex = index
dErr := erasureDisks[index].DeleteFile(bucket, pathJoin(object, fi.DataDir, "part.1")) dErr := erasureDisks[index].DeleteFile(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
if dErr != nil { if dErr != nil {
t.Fatalf("Test %d: Failed to delete %s - %v", i+1, t.Fatalf("Test %d: Failed to delete %s - %v", i+1,
filepath.Join(object, "part.1"), dErr) filepath.Join(object, "part.1"), dErr)

14
cmd/erasure-healing.go
View File

@ -78,7 +78,7 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints
afterState[index] = madmin.DriveStateOffline afterState[index] = madmin.DriveStateOffline
return errDiskNotFound return errDiskNotFound
} }
if _, serr := storageDisks[index].StatVol(bucket); serr != nil { if _, serr := storageDisks[index].StatVol(ctx, bucket); serr != nil {
if serr == errDiskNotFound { if serr == errDiskNotFound {
beforeState[index] = madmin.DriveStateOffline beforeState[index] = madmin.DriveStateOffline
afterState[index] = madmin.DriveStateOffline afterState[index] = madmin.DriveStateOffline
@ -136,7 +136,7 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints
index := index index := index
g.Go(func() error { g.Go(func() error {
if beforeState[index] == madmin.DriveStateMissing { if beforeState[index] == madmin.DriveStateMissing {
makeErr := storageDisks[index].MakeVol(bucket) makeErr := storageDisks[index].MakeVol(ctx, bucket)
if makeErr == nil { if makeErr == nil {
afterState[index] = madmin.DriveStateOk afterState[index] = madmin.DriveStateOk
} }
@ -171,7 +171,7 @@ func listAllBuckets(storageDisks []StorageAPI, healBuckets map[string]VolInfo) (
continue continue
} }
var volsInfo []VolInfo var volsInfo []VolInfo
volsInfo, err = disk.ListVols() volsInfo, err = disk.ListVols(context.TODO())
if err != nil { if err != nil {
if IsErrIgnored(err, bucketMetadataOpIgnoredErrs...) { if IsErrIgnored(err, bucketMetadataOpIgnoredErrs...) {
continue continue
@ -472,7 +472,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
} }
// Attempt a rename now from healed data to final location. // Attempt a rename now from healed data to final location.
if err = disk.RenameData(minioMetaTmpBucket, tmpID, partsMetadata[i].DataDir, bucket, object); err != nil { if err = disk.RenameData(ctx, minioMetaTmpBucket, tmpID, partsMetadata[i].DataDir, bucket, object); err != nil {
if err != errIsNotRegular && err != errFileNotFound { if err != errIsNotRegular && err != errFileNotFound {
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
} }
@ -525,7 +525,7 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
wg.Add(1) wg.Add(1)
go func(index int, disk StorageAPI) { go func(index int, disk StorageAPI) {
defer wg.Done() defer wg.Done()
_ = disk.DeleteFile(bucket, object) _ = disk.DeleteFile(ctx, bucket, object)
}(index, disk) }(index, disk)
} }
wg.Wait() wg.Wait()
@ -557,7 +557,7 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
for i, err := range errs { for i, err := range errs {
if err == errVolumeNotFound || err == errFileNotFound { if err == errVolumeNotFound || err == errFileNotFound {
// Bucket or prefix/directory not found // Bucket or prefix/directory not found
merr := storageDisks[i].MakeVol(pathJoin(bucket, object)) merr := storageDisks[i].MakeVol(ctx, pathJoin(bucket, object))
switch merr { switch merr {
case nil, errVolumeExists: case nil, errVolumeExists:
hr.After.Drives[i].State = madmin.DriveStateOk hr.After.Drives[i].State = madmin.DriveStateOk
@ -642,7 +642,7 @@ func statAllDirs(ctx context.Context, storageDisks []StorageAPI, bucket, prefix
} }
index := index index := index
g.Go(func() error { g.Go(func() error {
entries, err := storageDisks[index].ListDir(bucket, prefix, 1) entries, err := storageDisks[index].ListDir(ctx, bucket, prefix, 1)
if err != nil { if err != nil {
return err return err
} }

32
cmd/erasure-healing_test.go
View File

@ -66,7 +66,7 @@ func TestHealing(t *testing.T) {
} }
disk := er.getDisks()[0] disk := er.getDisks()[0]
fileInfoPreHeal, err := disk.ReadVersion(bucket, object, "") fileInfoPreHeal, err := disk.ReadVersion(context.Background(), bucket, object, "")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -83,7 +83,7 @@ func TestHealing(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
fileInfoPostHeal, err := disk.ReadVersion(bucket, object, "") fileInfoPostHeal, err := disk.ReadVersion(context.Background(), bucket, object, "")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -102,7 +102,7 @@ func TestHealing(t *testing.T) {
// gone down when an object was replaced by a new object. // gone down when an object was replaced by a new object.
fileInfoOutDated := fileInfoPreHeal fileInfoOutDated := fileInfoPreHeal
fileInfoOutDated.ModTime = time.Now() fileInfoOutDated.ModTime = time.Now()
err = disk.WriteMetadata(bucket, object, fileInfoOutDated) err = disk.WriteMetadata(context.Background(), bucket, object, fileInfoOutDated)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -112,7 +112,7 @@ func TestHealing(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
fileInfoPostHeal, err = disk.ReadVersion(bucket, object, "") fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -134,7 +134,7 @@ func TestHealing(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
// Stat the bucket to make sure that it was created. // Stat the bucket to make sure that it was created.
_, err = er.getDisks()[0].StatVol(bucket) _, err = er.getDisks()[0].StatVol(context.Background(), bucket)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -200,7 +200,7 @@ func TestHealObjectCorrupted(t *testing.T) {
er := z.zones[0].sets[0] er := z.zones[0].sets[0]
erasureDisks := er.getDisks() erasureDisks := er.getDisks()
firstDisk := erasureDisks[0] firstDisk := erasureDisks[0]
err = firstDisk.DeleteFile(bucket, pathJoin(object, xlStorageFormatFile)) err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile))
if err != nil { if err != nil {
t.Fatalf("Failed to delete a file - %v", err) t.Fatalf("Failed to delete a file - %v", err)
} }
@ -216,16 +216,16 @@ func TestHealObjectCorrupted(t *testing.T) {
t.Fatalf("Failed to getLatestFileInfo - %v", err) t.Fatalf("Failed to getLatestFileInfo - %v", err)
} }
if err = firstDisk.CheckFile(bucket, object); err != nil { if err = firstDisk.CheckFile(context.Background(), bucket, object); err != nil {
t.Errorf("Expected er.meta file to be present but stat failed - %v", err) t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
} }
err = firstDisk.DeleteFile(bucket, pathJoin(object, fi.DataDir, "part.1")) err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
if err != nil { if err != nil {
t.Errorf("Failure during deleting part.1 - %v", err) t.Errorf("Failure during deleting part.1 - %v", err)
} }
err = firstDisk.WriteAll(bucket, pathJoin(object, fi.DataDir, "part.1"), bytes.NewReader([]byte{})) err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bytes.NewReader([]byte{}))
if err != nil { if err != nil {
t.Errorf("Failure during creating part.1 - %v", err) t.Errorf("Failure during creating part.1 - %v", err)
} }
@ -245,13 +245,13 @@ func TestHealObjectCorrupted(t *testing.T) {
t.Fatalf("FileInfo not equal after healing") t.Fatalf("FileInfo not equal after healing")
} }
err = firstDisk.DeleteFile(bucket, pathJoin(object, fi.DataDir, "part.1")) err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
if err != nil { if err != nil {
t.Errorf("Failure during deleting part.1 - %v", err) t.Errorf("Failure during deleting part.1 - %v", err)
} }
bdata := bytes.Repeat([]byte("b"), int(nfi.Size)) bdata := bytes.Repeat([]byte("b"), int(nfi.Size))
err = firstDisk.WriteAll(bucket, pathJoin(object, fi.DataDir, "part.1"), bytes.NewReader(bdata)) err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bytes.NewReader(bdata))
if err != nil { if err != nil {
t.Errorf("Failure during creating part.1 - %v", err) t.Errorf("Failure during creating part.1 - %v", err)
} }
@ -274,7 +274,7 @@ func TestHealObjectCorrupted(t *testing.T) {
// Test 4: checks if HealObject returns an error when xl.meta is not found // Test 4: checks if HealObject returns an error when xl.meta is not found
// in more than read quorum number of disks, to create a corrupted situation. // in more than read quorum number of disks, to create a corrupted situation.
for i := 0; i <= len(er.getDisks())/2; i++ { for i := 0; i <= len(er.getDisks())/2; i++ {
er.getDisks()[i].DeleteFile(bucket, pathJoin(object, xlStorageFormatFile)) er.getDisks()[i].DeleteFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile))
} }
// Try healing now, expect to receive errFileNotFound. // Try healing now, expect to receive errFileNotFound.
@ -350,7 +350,7 @@ func TestHealObjectErasure(t *testing.T) {
t.Fatalf("Failed to complete multipart upload - %v", err) t.Fatalf("Failed to complete multipart upload - %v", err)
} }
err = firstDisk.DeleteFile(bucket, pathJoin(object, xlStorageFormatFile)) err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile))
if err != nil { if err != nil {
t.Fatalf("Failed to delete a file - %v", err) t.Fatalf("Failed to delete a file - %v", err)
} }
@ -360,7 +360,7 @@ func TestHealObjectErasure(t *testing.T) {
t.Fatalf("Failed to heal object - %v", err) t.Fatalf("Failed to heal object - %v", err)
} }
if err = firstDisk.CheckFile(bucket, object); err != nil { if err = firstDisk.CheckFile(context.Background(), bucket, object); err != nil {
t.Errorf("Expected er.meta file to be present but stat failed - %v", err) t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
} }
@ -421,7 +421,7 @@ func TestHealEmptyDirectoryErasure(t *testing.T) {
z := obj.(*erasureZones) z := obj.(*erasureZones)
er := z.zones[0].sets[0] er := z.zones[0].sets[0]
firstDisk := er.getDisks()[0] firstDisk := er.getDisks()[0]
err = firstDisk.DeleteFile(bucket, object) err = firstDisk.DeleteFile(context.Background(), bucket, object)
if err != nil { if err != nil {
t.Fatalf("Failed to delete a file - %v", err) t.Fatalf("Failed to delete a file - %v", err)
} }
@ -433,7 +433,7 @@ func TestHealEmptyDirectoryErasure(t *testing.T) {
} }
// Check if the empty directory is restored in the first disk // Check if the empty directory is restored in the first disk
_, err = firstDisk.StatVol(pathJoin(bucket, object)) _, err = firstDisk.StatVol(context.Background(), pathJoin(bucket, object))
if err != nil { if err != nil {
t.Fatalf("Expected object to be present but stat failed - %v", err) t.Fatalf("Expected object to be present but stat failed - %v", err)
} }

2
cmd/erasure-metadata-utils.go
View File

@ -126,7 +126,7 @@ func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, ve
if disks[index] == nil { if disks[index] == nil {
return errDiskNotFound return errDiskNotFound
} }
metadataArray[index], err = disks[index].ReadVersion(bucket, object, versionID) metadataArray[index], err = disks[index].ReadVersion(ctx, bucket, object, versionID)
if err != nil { if err != nil {
if err != errFileNotFound && err != errVolumeNotFound && err != errFileVersionNotFound { if err != errFileNotFound && err != errVolumeNotFound && err != errFileVersionNotFound {
logger.GetReqInfo(ctx).AppendTags("disk", disks[index].String()) logger.GetReqInfo(ctx).AppendTags("disk", disks[index].String())

4
cmd/erasure-metadata.go
View File

@ -273,7 +273,7 @@ func renameFileInfo(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry
if disks[index] == nil { if disks[index] == nil {
return errDiskNotFound return errDiskNotFound
} }
if err := disks[index].RenameData(srcBucket, srcEntry, "", dstBucket, dstEntry); err != nil { if err := disks[index].RenameData(ctx, srcBucket, srcEntry, "", dstBucket, dstEntry); err != nil {
if !IsErrIgnored(err, ignoredErr...) { if !IsErrIgnored(err, ignoredErr...) {
return err return err
} }
@ -304,7 +304,7 @@ func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, bucket, prefix
} }
// Pick one FileInfo for a disk at index. // Pick one FileInfo for a disk at index.
files[index].Erasure.Index = index + 1 files[index].Erasure.Index = index + 1
return disks[index].WriteMetadata(bucket, prefix, files[index]) return disks[index].WriteMetadata(ctx, bucket, prefix, files[index])
}, index) }, index)
} }

12
cmd/erasure-multipart.go
View File

@ -63,7 +63,7 @@ func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir stri
// Ignoring failure to remove parts that weren't present in CompleteMultipartUpload // Ignoring failure to remove parts that weren't present in CompleteMultipartUpload
// requests. xl.meta is the authoritative source of truth on which parts constitute // requests. xl.meta is the authoritative source of truth on which parts constitute
// the object. The presence of parts that don't belong in the object doesn't affect correctness. // the object. The presence of parts that don't belong in the object doesn't affect correctness.
_ = storageDisks[index].DeleteFile(minioMetaMultipartBucket, curpartPath) _ = storageDisks[index].DeleteFile(context.TODO(), minioMetaMultipartBucket, curpartPath)
return nil return nil
}, index) }, index)
} }
@ -99,18 +99,18 @@ func (er erasureObjects) cleanupStaleMultipartUploads(ctx context.Context, clean
// Remove the old multipart uploads on the given disk. // Remove the old multipart uploads on the given disk.
func (er erasureObjects) cleanupStaleMultipartUploadsOnDisk(ctx context.Context, disk StorageAPI, expiry time.Duration) { func (er erasureObjects) cleanupStaleMultipartUploadsOnDisk(ctx context.Context, disk StorageAPI, expiry time.Duration) {
now := time.Now() now := time.Now()
shaDirs, err := disk.ListDir(minioMetaMultipartBucket, "", -1) shaDirs, err := disk.ListDir(ctx, minioMetaMultipartBucket, "", -1)
if err != nil { if err != nil {
return return
} }
for _, shaDir := range shaDirs { for _, shaDir := range shaDirs {
uploadIDDirs, err := disk.ListDir(minioMetaMultipartBucket, shaDir, -1) uploadIDDirs, err := disk.ListDir(ctx, minioMetaMultipartBucket, shaDir, -1)
if err != nil { if err != nil {
continue continue
} }
for _, uploadIDDir := range uploadIDDirs { for _, uploadIDDir := range uploadIDDirs {
uploadIDPath := pathJoin(shaDir, uploadIDDir) uploadIDPath := pathJoin(shaDir, uploadIDDir)
fi, err := disk.ReadVersion(minioMetaMultipartBucket, uploadIDPath, "") fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, uploadIDPath, "")
if err != nil { if err != nil {
continue continue
} }
@ -139,7 +139,7 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
if disk == nil { if disk == nil {
continue continue
} }
uploadIDs, err = disk.ListDir(minioMetaMultipartBucket, er.getMultipartSHADir(bucket, object), -1) uploadIDs, err = disk.ListDir(ctx, minioMetaMultipartBucket, er.getMultipartSHADir(bucket, object), -1)
if err != nil { if err != nil {
if err == errDiskNotFound { if err == errDiskNotFound {
continue continue
@ -172,7 +172,7 @@ retry:
if populatedUploadIds.Contains(uploadID) { if populatedUploadIds.Contains(uploadID) {
continue continue
} }
fi, err := disk.ReadVersion(minioMetaMultipartBucket, pathJoin(er.getUploadIDDir(bucket, object, uploadID)), "") fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, pathJoin(er.getUploadIDDir(bucket, object, uploadID)), "")
if err != nil { if err != nil {
if err == errDiskNotFound || err == errFileNotFound { if err == errDiskNotFound || err == errFileNotFound {
goto retry goto retry

16
cmd/erasure-object.go
View File

@ -49,7 +49,7 @@ func (er erasureObjects) putObjectDir(ctx context.Context, bucket, object string
} }
index := index index := index
g.Go(func() error { g.Go(func() error {
err := storageDisks[index].MakeVol(pathJoin(bucket, object)) err := storageDisks[index].MakeVol(ctx, pathJoin(bucket, object))
if err != nil && err != errVolumeExists { if err != nil && err != errVolumeExists {
return err return err
} }
@ -362,7 +362,7 @@ func (er erasureObjects) getObjectInfoDir(ctx context.Context, bucket, object st
index := index index := index
g.Go(func() error { g.Go(func() error {
// Check if 'prefix' is an object on this 'disk'. // Check if 'prefix' is an object on this 'disk'.
entries, err := storageDisks[index].ListDir(bucket, object, 1) entries, err := storageDisks[index].ListDir(ctx, bucket, object, 1)
if err != nil { if err != nil {
return err return err
} }
@ -459,7 +459,7 @@ func undoRename(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry str
index := index index := index
g.Go(func() error { g.Go(func() error {
if errs[index] == nil { if errs[index] == nil {
_ = disks[index].RenameFile(dstBucket, dstEntry, srcBucket, srcEntry) _ = disks[index].RenameFile(context.TODO(), dstBucket, dstEntry, srcBucket, srcEntry)
} }
return nil return nil
}, index) }, index)
@ -479,7 +479,7 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, da
if disks[index] == nil { if disks[index] == nil {
return errDiskNotFound return errDiskNotFound
} }
if err := disks[index].RenameData(srcBucket, srcEntry, dataDir, dstBucket, dstEntry); err != nil { if err := disks[index].RenameData(ctx, srcBucket, srcEntry, dataDir, dstBucket, dstEntry); err != nil {
if !IsErrIgnored(err, ignoredErr...) { if !IsErrIgnored(err, ignoredErr...) {
return err return err
} }
@ -504,7 +504,7 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, da
ug.Go(func() error { ug.Go(func() error {
// Undo all the partial rename operations. // Undo all the partial rename operations.
if errs[index] == nil { if errs[index] == nil {
_ = disks[index].RenameData(dstBucket, dstEntry, dataDir, srcBucket, srcEntry) _ = disks[index].RenameData(context.Background(), dstBucket, dstEntry, dataDir, srcBucket, srcEntry)
} }
return nil return nil
}, index) }, index)
@ -532,7 +532,7 @@ func rename(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBuc
if disks[index] == nil { if disks[index] == nil {
return errDiskNotFound return errDiskNotFound
} }
if err := disks[index].RenameFile(srcBucket, srcEntry, dstBucket, dstEntry); err != nil { if err := disks[index].RenameFile(ctx, srcBucket, srcEntry, dstBucket, dstEntry); err != nil {
if !IsErrIgnored(err, ignoredErr...) { if !IsErrIgnored(err, ignoredErr...) {
return err return err
} }
@ -776,7 +776,7 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object
if disks[index] == nil { if disks[index] == nil {
return errDiskNotFound return errDiskNotFound
} }
err := disks[index].DeleteVersion(bucket, object, fi) err := disks[index].DeleteVersion(ctx, bucket, object, fi)
if err != nil && err != errVolumeNotFound { if err != nil && err != errVolumeNotFound {
return err return err
} }
@ -899,7 +899,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
wg.Add(1) wg.Add(1)
go func(index int, disk StorageAPI) { go func(index int, disk StorageAPI) {
defer wg.Done() defer wg.Done()
delObjErrs[index] = disk.DeleteVersions(bucket, versions) delObjErrs[index] = disk.DeleteVersions(ctx, bucket, versions)
}(index, disk) }(index, disk)
} }

10
cmd/erasure-sets.go
View File

@ -140,7 +140,7 @@ func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
// Close the internal connection to avoid connection leaks. // Close the internal connection to avoid connection leaks.
disk.Close() disk.Close()
if errors.Is(err, errUnformattedDisk) { if errors.Is(err, errUnformattedDisk) {
info, derr := disk.DiskInfo() info, derr := disk.DiskInfo(context.TODO())
if derr != nil && info.RootDisk { if derr != nil && info.RootDisk {
return nil, nil, fmt.Errorf("Disk: %s returned %w but its a root disk refusing to use it", return nil, nil, fmt.Errorf("Disk: %s returned %w but its a root disk refusing to use it",
disk, derr) // make sure to '%w' to wrap the error disk, derr) // make sure to '%w' to wrap the error
@ -981,7 +981,7 @@ func (s *erasureSets) startMergeWalksVersionsN(ctx context.Context, bucket, pref
// Disk can be offline // Disk can be offline
continue continue
} }
entryCh, err := disk.WalkVersions(bucket, prefix, marker, recursive, endWalkCh) entryCh, err := disk.WalkVersions(GlobalContext, bucket, prefix, marker, recursive, endWalkCh)
if err != nil { if err != nil {
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
// Disk walk returned error, ignore it. // Disk walk returned error, ignore it.
@ -1012,7 +1012,7 @@ func (s *erasureSets) startMergeWalksN(ctx context.Context, bucket, prefix, mark
// Disk can be offline // Disk can be offline
continue continue
} }
entryCh, err := disk.Walk(bucket, prefix, marker, recursive, endWalkCh) entryCh, err := disk.Walk(GlobalContext, bucket, prefix, marker, recursive, endWalkCh)
if err != nil { if err != nil {
// Disk walk returned error, ignore it. // Disk walk returned error, ignore it.
continue continue
@ -1042,7 +1042,7 @@ func (s *erasureSets) startSplunkMergeWalksN(ctx context.Context, bucket, prefix
// Disk can be offline // Disk can be offline
continue continue
} }
entryCh, err := disk.WalkSplunk(bucket, prefix, marker, endWalkCh) entryCh, err := disk.WalkSplunk(GlobalContext, bucket, prefix, marker, endWalkCh)
if err != nil { if err != nil {
// Disk walk returned error, ignore it. // Disk walk returned error, ignore it.
continue continue
@ -1300,7 +1300,7 @@ func getHealDiskInfos(storageDisks []StorageAPI, errs []error) ([]DiskInfo, []er
return errDiskNotFound return errDiskNotFound
} }
var err error var err error
infos[index], err = storageDisks[index].DiskInfo() infos[index], err = storageDisks[index].DiskInfo(context.TODO())
return err return err
}, index) }, index)
} }

4
cmd/erasure.go
View File

@ -139,7 +139,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di
// Storage disk is empty, perhaps ignored disk or not available. // Storage disk is empty, perhaps ignored disk or not available.
return errDiskNotFound return errDiskNotFound
} }
info, err := disks[index].DiskInfo() info, err := disks[index].DiskInfo(context.TODO())
if err != nil { if err != nil {
if !IsErr(err, baseErrs...) { if !IsErr(err, baseErrs...) {
reqInfo := (&logger.ReqInfo{}).AppendTags("disk", disks[index].String()) reqInfo := (&logger.ReqInfo{}).AppendTags("disk", disks[index].String())
@ -282,7 +282,7 @@ func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []Buc
if d == nil || !d.IsOnline() { if d == nil || !d.IsOnline() {
continue continue
} }
di, err := d.DiskInfo() di, err := d.DiskInfo(ctx)
if err != nil { if err != nil {
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
continue continue

2
cmd/erasure_test.go
View File

@ -134,7 +134,7 @@ func newErasureTestSetup(dataBlocks int, parityBlocks int, blockSize int64) (*er
if err != nil { if err != nil {
return nil, err return nil, err
} }
err = disks[i].MakeVol("testbucket") err = disks[i].MakeVol(context.Background(), "testbucket")
if err != nil { if err != nil {
return nil, err return nil, err
} }

12
cmd/format-erasure.go
View File

@ -355,15 +355,15 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3) error {
tmpFormat := mustGetUUID() tmpFormat := mustGetUUID()
// Purge any existing temporary file, okay to ignore errors here. // Purge any existing temporary file, okay to ignore errors here.
defer disk.DeleteFile(minioMetaBucket, tmpFormat) defer disk.DeleteFile(context.TODO(), minioMetaBucket, tmpFormat)
// write to unique file. // write to unique file.
if err = disk.WriteAll(minioMetaBucket, tmpFormat, bytes.NewReader(formatBytes)); err != nil { if err = disk.WriteAll(context.TODO(), minioMetaBucket, tmpFormat, bytes.NewReader(formatBytes)); err != nil {
return err return err
} }
// Rename file `uuid.json` --> `format.json`. // Rename file `uuid.json` --> `format.json`.
if err = disk.RenameFile(minioMetaBucket, tmpFormat, minioMetaBucket, formatConfigFile); err != nil { if err = disk.RenameFile(context.TODO(), minioMetaBucket, tmpFormat, minioMetaBucket, formatConfigFile); err != nil {
return err return err
} }
@ -392,13 +392,13 @@ func isHiddenDirectories(vols ...VolInfo) bool {
// loadFormatErasure - loads format.json from disk. // loadFormatErasure - loads format.json from disk.
func loadFormatErasure(disk StorageAPI) (format *formatErasureV3, err error) { func loadFormatErasure(disk StorageAPI) (format *formatErasureV3, err error) {
buf, err := disk.ReadAll(minioMetaBucket, formatConfigFile) buf, err := disk.ReadAll(context.TODO(), minioMetaBucket, formatConfigFile)
if err != nil { if err != nil {
// 'file not found' and 'volume not found' as // 'file not found' and 'volume not found' as
// same. 'volume not found' usually means its a fresh disk. // same. 'volume not found' usually means its a fresh disk.
if err == errFileNotFound || err == errVolumeNotFound { if err == errFileNotFound || err == errVolumeNotFound {
var vols []VolInfo var vols []VolInfo
vols, err = disk.ListVols() vols, err = disk.ListVols(context.TODO())
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -862,7 +862,7 @@ func makeFormatErasureMetaVolumes(disk StorageAPI) error {
return errDiskNotFound return errDiskNotFound
} }
// Attempt to create MinIO internal buckets. // Attempt to create MinIO internal buckets.
return disk.MakeVolBulk(minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, dataUsageBucket) return disk.MakeVolBulk(context.TODO(), minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, dataUsageBucket)
} }
// Get all UUIDs which are present in reference format should // Get all UUIDs which are present in reference format should

2
cmd/global-heal.go
View File

@ -131,7 +131,7 @@ func healErasureSet(ctx context.Context, setIndex int, xlObj *erasureObjects, se
continue continue
} }
entryCh, err := disk.WalkVersions(bucket.Name, "", "", true, ctx.Done()) entryCh, err := disk.WalkVersions(ctx, bucket.Name, "", "", true, ctx.Done())
if err != nil { if err != nil {
// Disk walk returned error, ignore it. // Disk walk returned error, ignore it.
continue continue

4
cmd/lock-rest-client.go
View File

@ -63,7 +63,7 @@ func (client *lockRESTClient) callWithContext(ctx context.Context, method string
values = make(url.Values) values = make(url.Values)
} }
respBody, err = client.restClient.Call(method, values, body, length) respBody, err = client.restClient.Call(ctx, method, values, body, length)
if err == nil { if err == nil {
return respBody, nil return respBody, nil
} }
@ -158,7 +158,7 @@ func newlockRESTClient(endpoint Endpoint) *lockRESTClient {
ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout) ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
// Instantiate a new rest client for healthcheck // Instantiate a new rest client for healthcheck
// to avoid recursive healthCheckFn() // to avoid recursive healthCheckFn()
respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).CallWithContext(ctx, lockRESTMethodHealth, nil, nil, -1) respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).Call(ctx, lockRESTMethodHealth, nil, nil, -1)
xhttp.DrainBody(respBody) xhttp.DrainBody(respBody)
cancel() cancel()
var ne *rest.NetworkError var ne *rest.NetworkError

104
cmd/naughty-disk_test.go
View File

@ -94,139 +94,139 @@ func (d *naughtyDisk) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageC
return d.disk.CrawlAndGetDataUsage(ctx, cache) return d.disk.CrawlAndGetDataUsage(ctx, cache)
} }
func (d *naughtyDisk) DiskInfo() (info DiskInfo, err error) { func (d *naughtyDisk) DiskInfo(ctx context.Context) (info DiskInfo, err error) {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return info, err return info, err
} }
return d.disk.DiskInfo() return d.disk.DiskInfo(ctx)
} }
func (d *naughtyDisk) MakeVolBulk(volumes ...string) (err error) { func (d *naughtyDisk) MakeVolBulk(ctx context.Context, volumes ...string) (err error) {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return err return err
} }
return d.disk.MakeVolBulk(volumes...) return d.disk.MakeVolBulk(ctx, volumes...)
} }
func (d *naughtyDisk) MakeVol(volume string) (err error) { func (d *naughtyDisk) MakeVol(ctx context.Context, volume string) (err error) {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return err return err
} }
return d.disk.MakeVol(volume) return d.disk.MakeVol(ctx, volume)
} }
func (d *naughtyDisk) ListVols() (vols []VolInfo, err error) { func (d *naughtyDisk) ListVols(ctx context.Context) (vols []VolInfo, err error) {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return nil, err return nil, err
} }
return d.disk.ListVols() return d.disk.ListVols(ctx)
} }
func (d *naughtyDisk) StatVol(volume string) (volInfo VolInfo, err error) { func (d *naughtyDisk) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return VolInfo{}, err return VolInfo{}, err
} }
return d.disk.StatVol(volume) return d.disk.StatVol(ctx, volume)
} }
func (d *naughtyDisk) DeleteVol(volume string, forceDelete bool) (err error) { func (d *naughtyDisk) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return err return err
} }
return d.disk.DeleteVol(volume, forceDelete) return d.disk.DeleteVol(ctx, volume, forceDelete)
} }
func (d *naughtyDisk) WalkSplunk(volume, path, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error) { func (d *naughtyDisk) WalkSplunk(ctx context.Context, volume, dirPath, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error) {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return nil, err return nil, err
} }
return d.disk.WalkSplunk(volume, path, marker, endWalkCh) return d.disk.WalkSplunk(ctx, volume, dirPath, marker, endWalkCh)
} }
func (d *naughtyDisk) WalkVersions(volume, path, marker string, recursive bool, endWalkVersionsCh <-chan struct{}) (chan FileInfoVersions, error) { func (d *naughtyDisk) WalkVersions(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error) {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return nil, err return nil, err
} }
return d.disk.WalkVersions(volume, path, marker, recursive, endWalkVersionsCh) return d.disk.WalkVersions(ctx, volume, dirPath, marker, recursive, endWalkCh)
} }
func (d *naughtyDisk) Walk(volume, path, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) { func (d *naughtyDisk) Walk(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return nil, err return nil, err
} }
return d.disk.Walk(volume, path, marker, recursive, endWalkCh) return d.disk.Walk(ctx, volume, dirPath, marker, recursive, endWalkCh)
} }
func (d *naughtyDisk) ListDir(volume, path string, count int) (entries []string, err error) { func (d *naughtyDisk) ListDir(ctx context.Context, volume, dirPath string, count int) (entries []string, err error) {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return []string{}, err return []string{}, err
} }
return d.disk.ListDir(volume, path, count) return d.disk.ListDir(ctx, volume, dirPath, count)
} }
func (d *naughtyDisk) ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) { func (d *naughtyDisk) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return 0, err return 0, err
} }
return d.disk.ReadFile(volume, path, offset, buf, verifier) return d.disk.ReadFile(ctx, volume, path, offset, buf, verifier)
} }
func (d *naughtyDisk) ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error) { func (d *naughtyDisk) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return nil, err return nil, err
} }
return d.disk.ReadFileStream(volume, path, offset, length) return d.disk.ReadFileStream(ctx, volume, path, offset, length)
} }
func (d *naughtyDisk) CreateFile(volume, path string, size int64, reader io.Reader) error { func (d *naughtyDisk) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return err return err
} }
return d.disk.CreateFile(volume, path, size, reader) return d.disk.CreateFile(ctx, volume, path, size, reader)
} }
func (d *naughtyDisk) AppendFile(volume, path string, buf []byte) error { func (d *naughtyDisk) AppendFile(ctx context.Context, volume string, path string, buf []byte) error {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return err return err
} }
return d.disk.AppendFile(volume, path, buf) return d.disk.AppendFile(ctx, volume, path, buf)
} }
func (d *naughtyDisk) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath string) error { func (d *naughtyDisk) RenameData(ctx context.Context, srcVolume, srcPath, dataDir, dstVolume, dstPath string) error {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return err return err
} }
return d.disk.RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath) return d.disk.RenameData(ctx, srcVolume, srcPath, dataDir, dstVolume, dstPath)
} }
func (d *naughtyDisk) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error { func (d *naughtyDisk) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return err return err
} }
return d.disk.RenameFile(srcVolume, srcPath, dstVolume, dstPath) return d.disk.RenameFile(ctx, srcVolume, srcPath, dstVolume, dstPath)
} }
func (d *naughtyDisk) CheckParts(volume string, path string, fi FileInfo) (err error) { func (d *naughtyDisk) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (err error) {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return err return err
} }
return d.disk.CheckParts(volume, path, fi) return d.disk.CheckParts(ctx, volume, path, fi)
} }
func (d *naughtyDisk) CheckFile(volume string, path string) (err error) { func (d *naughtyDisk) CheckFile(ctx context.Context, volume string, path string) (err error) {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return err return err
} }
return d.disk.CheckFile(volume, path) return d.disk.CheckFile(ctx, volume, path)
} }
func (d *naughtyDisk) DeleteFile(volume string, path string) (err error) { func (d *naughtyDisk) DeleteFile(ctx context.Context, volume string, path string) (err error) {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return err return err
} }
return d.disk.DeleteFile(volume, path) return d.disk.DeleteFile(ctx, volume, path)
} }
func (d *naughtyDisk) DeleteVersions(volume string, versions []FileInfo) []error { func (d *naughtyDisk) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) []error {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
errs := make([]error, len(versions)) errs := make([]error, len(versions))
for i := range errs { for i := range errs {
@ -234,47 +234,47 @@ func (d *naughtyDisk) DeleteVersions(volume string, versions []FileInfo) []error
} }
return errs return errs
} }
return d.disk.DeleteVersions(volume, versions) return d.disk.DeleteVersions(ctx, volume, versions)
} }
func (d *naughtyDisk) WriteMetadata(volume, path string, fi FileInfo) (err error) { func (d *naughtyDisk) WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) (err error) {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return err return err
} }
return d.disk.WriteMetadata(volume, path, fi) return d.disk.WriteMetadata(ctx, volume, path, fi)
} }
func (d *naughtyDisk) DeleteVersion(volume string, path string, fi FileInfo) (err error) { func (d *naughtyDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo) (err error) {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return err return err
} }
return d.disk.DeleteVersion(volume, path, fi) return d.disk.DeleteVersion(ctx, volume, path, fi)
} }
func (d *naughtyDisk) ReadVersion(volume string, path string, versionID string) (fi FileInfo, err error) { func (d *naughtyDisk) ReadVersion(ctx context.Context, volume, path, versionID string) (fi FileInfo, err error) {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return FileInfo{}, err return FileInfo{}, err
} }
return d.disk.ReadVersion(volume, path, versionID) return d.disk.ReadVersion(ctx, volume, path, versionID)
} }
func (d *naughtyDisk) WriteAll(volume string, path string, reader io.Reader) (err error) { func (d *naughtyDisk) WriteAll(ctx context.Context, volume string, path string, reader io.Reader) (err error) {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return err return err
} }
return d.disk.WriteAll(volume, path, reader) return d.disk.WriteAll(ctx, volume, path, reader)
} }
func (d *naughtyDisk) ReadAll(volume string, path string) (buf []byte, err error) { func (d *naughtyDisk) ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return nil, err return nil, err
} }
return d.disk.ReadAll(volume, path) return d.disk.ReadAll(ctx, volume, path)
} }
func (d *naughtyDisk) VerifyFile(volume, path string, fi FileInfo) error { func (d *naughtyDisk) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error {
if err := d.calcError(); err != nil { if err := d.calcError(); err != nil {
return err return err
} }
return d.disk.VerifyFile(volume, path, fi) return d.disk.VerifyFile(ctx, volume, path, fi)
} }

6
cmd/object-api-common.go
View File

@ -99,7 +99,7 @@ func cleanupDir(ctx context.Context, storage StorageAPI, volume, dirPath string)
delFunc = func(entryPath string) error { delFunc = func(entryPath string) error {
if !HasSuffix(entryPath, SlashSeparator) { if !HasSuffix(entryPath, SlashSeparator) {
// Delete the file entry. // Delete the file entry.
err := storage.DeleteFile(volume, entryPath) err := storage.DeleteFile(ctx, volume, entryPath)
if err != errDiskNotFound && err != errUnformattedDisk { if err != errDiskNotFound && err != errUnformattedDisk {
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
} }
@ -107,7 +107,7 @@ func cleanupDir(ctx context.Context, storage StorageAPI, volume, dirPath string)
} }
// If it's a directory, list and call delFunc() for each entry. // If it's a directory, list and call delFunc() for each entry.
entries, err := storage.ListDir(volume, entryPath, -1) entries, err := storage.ListDir(ctx, volume, entryPath, -1)
// If entryPath prefix never existed, safe to ignore. // If entryPath prefix never existed, safe to ignore.
if err == errFileNotFound { if err == errFileNotFound {
return nil return nil
@ -120,7 +120,7 @@ func cleanupDir(ctx context.Context, storage StorageAPI, volume, dirPath string)
// Entry path is empty, just delete it. // Entry path is empty, just delete it.
if len(entries) == 0 { if len(entries) == 0 {
err = storage.DeleteFile(volume, entryPath) err = storage.DeleteFile(ctx, volume, entryPath)
if err != errDiskNotFound && err != errUnformattedDisk { if err != errDiskNotFound && err != errUnformattedDisk {
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
} }

4
cmd/peer-rest-client.go
View File

@ -62,7 +62,7 @@ func (client *peerRESTClient) callWithContext(ctx context.Context, method string
values = make(url.Values) values = make(url.Values)
} }
respBody, err = client.restClient.CallWithContext(ctx, method, values, body, length) respBody, err = client.restClient.Call(ctx, method, values, body, length)
if err == nil { if err == nil {
return respBody, nil return respBody, nil
} }
@ -882,7 +882,7 @@ func newPeerRESTClient(peer *xnet.Host) *peerRESTClient {
ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout) ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
// Instantiate a new rest client for healthcheck // Instantiate a new rest client for healthcheck
// to avoid recursive healthCheckFn() // to avoid recursive healthCheckFn()
respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).CallWithContext(ctx, peerRESTMethodHealth, nil, nil, -1) respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).Call(ctx, peerRESTMethodHealth, nil, nil, -1)
xhttp.DrainBody(respBody) xhttp.DrainBody(respBody)
cancel() cancel()
var ne *rest.NetworkError var ne *rest.NetworkError

10
cmd/rest/client.go
View File

@ -95,8 +95,8 @@ func (e restError) Timeout() bool {
return true return true
} }
// CallWithContext - make a REST call with context. // Call - make a REST call with context.
func (c *Client) CallWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (reply io.ReadCloser, err error) { func (c *Client) Call(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (reply io.ReadCloser, err error) {
if !c.IsOnline() { if !c.IsOnline() {
return nil, &NetworkError{Err: &url.Error{Op: method, URL: c.url.String(), Err: restError("remote server offline")}} return nil, &NetworkError{Err: &url.Error{Op: method, URL: c.url.String(), Err: restError("remote server offline")}}
} }
@ -153,12 +153,6 @@ func (c *Client) CallWithContext(ctx context.Context, method string, values url.
return resp.Body, nil return resp.Body, nil
} }
// Call - make a REST call.
func (c *Client) Call(method string, values url.Values, body io.Reader, length int64) (reply io.ReadCloser, err error) {
ctx := context.Background()
return c.CallWithContext(ctx, method, values, body, length)
}
// Close closes all idle connections of the underlying http client // Close closes all idle connections of the underlying http client
func (c *Client) Close() { func (c *Client) Close() {
atomic.StoreInt32(&c.connected, closed) atomic.StoreInt32(&c.connected, closed)

56
cmd/storage-interface.go
View File

@ -35,47 +35,47 @@ type StorageAPI interface {
GetDiskID() (string, error) GetDiskID() (string, error)
SetDiskID(id string) SetDiskID(id string)
DiskInfo() (info DiskInfo, err error) DiskInfo(ctx context.Context) (info DiskInfo, err error)
CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache) (dataUsageCache, error) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache) (dataUsageCache, error)
// Volume operations. // Volume operations.
MakeVol(volume string) (err error) MakeVol(ctx context.Context, volume string) (err error)
MakeVolBulk(volumes ...string) (err error) MakeVolBulk(ctx context.Context, volumes ...string) (err error)
ListVols() (vols []VolInfo, err error) ListVols(ctx context.Context) (vols []VolInfo, err error)
StatVol(volume string) (vol VolInfo, err error) StatVol(ctx context.Context, volume string) (vol VolInfo, err error)
DeleteVol(volume string, forceDelete bool) (err error) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error)
// WalkVersions in sorted order directly on disk. // WalkVersions in sorted order directly on disk.
WalkVersions(volume, dirPath string, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error) WalkVersions(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error)
// Walk in sorted order directly on disk. // Walk in sorted order directly on disk.
Walk(volume, dirPath string, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) Walk(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error)
// Walk in sorted order directly on disk. // Walk in sorted order directly on disk.
WalkSplunk(volume, dirPath string, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error) WalkSplunk(ctx context.Context, volume, dirPath, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error)
// Metadata operations // Metadata operations
DeleteVersion(volume, path string, fi FileInfo) error DeleteVersion(ctx context.Context, volume, path string, fi FileInfo) error
DeleteVersions(volume string, versions []FileInfo) []error DeleteVersions(ctx context.Context, volume string, versions []FileInfo) []error
WriteMetadata(volume, path string, fi FileInfo) error WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) error
ReadVersion(volume, path, versionID string) (FileInfo, error) ReadVersion(ctx context.Context, volume, path, versionID string) (FileInfo, error)
RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath string) error RenameData(ctx context.Context, srcVolume, srcPath, dataDir, dstVolume, dstPath string) error
// File operations. // File operations.
ListDir(volume, dirPath string, count int) ([]string, error) ListDir(ctx context.Context, volume, dirPath string, count int) ([]string, error)
ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error)
AppendFile(volume string, path string, buf []byte) (err error) AppendFile(ctx context.Context, volume string, path string, buf []byte) (err error)
CreateFile(volume, path string, size int64, reader io.Reader) error CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error
ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error)
RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error
CheckParts(volume string, path string, fi FileInfo) error CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error
CheckFile(volume string, path string) (err error) CheckFile(ctx context.Context, volume string, path string) (err error)
DeleteFile(volume string, path string) (err error) DeleteFile(ctx context.Context, volume string, path string) (err error)
VerifyFile(volume, path string, fi FileInfo) error VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error
// Write all data, syncs the data to disk. // Write all data, syncs the data to disk.
WriteAll(volume string, path string, reader io.Reader) (err error) WriteAll(ctx context.Context, volume string, path string, reader io.Reader) (err error)
// Read all. // Read all.
ReadAll(volume string, path string) (buf []byte, err error) ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error)
} }
// storageReader is an io.Reader view of a disk // storageReader is an io.Reader view of a disk
@ -86,7 +86,7 @@ type storageReader struct {
} }
func (r *storageReader) Read(p []byte) (n int, err error) { func (r *storageReader) Read(p []byte) (n int, err error) {
nn, err := r.storage.ReadFile(r.volume, r.path, r.offset, p, nil) nn, err := r.storage.ReadFile(context.TODO(), r.volume, r.path, r.offset, p, nil)
r.offset += nn r.offset += nn
n = int(nn) n = int(nn)
@ -103,7 +103,7 @@ type storageWriter struct {
} }
func (w *storageWriter) Write(p []byte) (n int, err error) { func (w *storageWriter) Write(p []byte) (n int, err error) {
err = w.storage.AppendFile(w.volume, w.path, p) err = w.storage.AppendFile(context.TODO(), w.volume, w.path, p)
if err == nil { if err == nil {
n = len(p) n = len(p)
} }

130
cmd/storage-rest-client.go
View File

@ -119,7 +119,7 @@ type storageRESTClient struct {
// Wrapper to restClient.Call to handle network errors, in case of network error the connection is makred disconnected // Wrapper to restClient.Call to handle network errors, in case of network error the connection is makred disconnected
// permanently. The only way to restore the storage connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints() // permanently. The only way to restore the storage connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
// after verifying format.json // after verifying format.json
func (client *storageRESTClient) call(method string, values url.Values, body io.Reader, length int64) (io.ReadCloser, error) { func (client *storageRESTClient) call(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (io.ReadCloser, error) {
if !client.IsOnline() { if !client.IsOnline() {
return nil, errDiskNotFound return nil, errDiskNotFound
} }
@ -127,7 +127,7 @@ func (client *storageRESTClient) call(method string, values url.Values, body io.
values = make(url.Values) values = make(url.Values)
} }
values.Set(storageRESTDiskID, client.diskID) values.Set(storageRESTDiskID, client.diskID)
respBody, err := client.restClient.Call(method, values, body, length) respBody, err := client.restClient.Call(ctx, method, values, body, length)
if err == nil { if err == nil {
return respBody, nil return respBody, nil
} }
@ -157,9 +157,7 @@ func (client *storageRESTClient) Hostname() string {
func (client *storageRESTClient) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache) (dataUsageCache, error) { func (client *storageRESTClient) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache) (dataUsageCache, error) {
b := cache.serialize() b := cache.serialize()
respBody, err := client.call(storageRESTMethodCrawlAndGetDataUsage, respBody, err := client.call(ctx, storageRESTMethodCrawlAndGetDataUsage, url.Values{}, bytes.NewBuffer(b), int64(len(b)))
url.Values{},
bytes.NewBuffer(b), int64(len(b)))
defer http.DrainBody(respBody) defer http.DrainBody(respBody)
if err != nil { if err != nil {
return cache, err return cache, err
@ -185,8 +183,8 @@ func (client *storageRESTClient) SetDiskID(id string) {
} }
// DiskInfo - fetch disk information for a remote disk. // DiskInfo - fetch disk information for a remote disk.
func (client *storageRESTClient) DiskInfo() (info DiskInfo, err error) { func (client *storageRESTClient) DiskInfo(ctx context.Context) (info DiskInfo, err error) {
respBody, err := client.call(storageRESTMethodDiskInfo, nil, nil, -1) respBody, err := client.call(ctx, storageRESTMethodDiskInfo, nil, nil, -1)
if err != nil { if err != nil {
return return
} }
@ -202,81 +200,81 @@ func (client *storageRESTClient) DiskInfo() (info DiskInfo, err error) {
} }
// MakeVolBulk - create multiple volumes in a bulk operation. // MakeVolBulk - create multiple volumes in a bulk operation.
func (client *storageRESTClient) MakeVolBulk(volumes ...string) (err error) { func (client *storageRESTClient) MakeVolBulk(ctx context.Context, volumes ...string) (err error) {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTVolumes, strings.Join(volumes, ",")) values.Set(storageRESTVolumes, strings.Join(volumes, ","))
respBody, err := client.call(storageRESTMethodMakeVolBulk, values, nil, -1) respBody, err := client.call(ctx, storageRESTMethodMakeVolBulk, values, nil, -1)
defer http.DrainBody(respBody) defer http.DrainBody(respBody)
return err return err
} }
// MakeVol - create a volume on a remote disk. // MakeVol - create a volume on a remote disk.
func (client *storageRESTClient) MakeVol(volume string) (err error) { func (client *storageRESTClient) MakeVol(ctx context.Context, volume string) (err error) {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTVolume, volume) values.Set(storageRESTVolume, volume)
respBody, err := client.call(storageRESTMethodMakeVol, values, nil, -1) respBody, err := client.call(ctx, storageRESTMethodMakeVol, values, nil, -1)
defer http.DrainBody(respBody) defer http.DrainBody(respBody)
return err return err
} }
// ListVols - List all volumes on a remote disk. // ListVols - List all volumes on a remote disk.
func (client *storageRESTClient) ListVols() (volinfo []VolInfo, err error) { func (client *storageRESTClient) ListVols(ctx context.Context) (vols []VolInfo, err error) {
respBody, err := client.call(storageRESTMethodListVols, nil, nil, -1) respBody, err := client.call(ctx, storageRESTMethodListVols, nil, nil, -1)
if err != nil { if err != nil {
return return
} }
defer http.DrainBody(respBody) defer http.DrainBody(respBody)
err = gob.NewDecoder(respBody).Decode(&volinfo) err = gob.NewDecoder(respBody).Decode(&vols)
return volinfo, err return vols, err
} }
// StatVol - get volume info over the network. // StatVol - get volume info over the network.
func (client *storageRESTClient) StatVol(volume string) (volInfo VolInfo, err error) { func (client *storageRESTClient) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTVolume, volume) values.Set(storageRESTVolume, volume)
respBody, err := client.call(storageRESTMethodStatVol, values, nil, -1) respBody, err := client.call(ctx, storageRESTMethodStatVol, values, nil, -1)
if err != nil { if err != nil {
return return
} }
defer http.DrainBody(respBody) defer http.DrainBody(respBody)
err = gob.NewDecoder(respBody).Decode(&volInfo) err = gob.NewDecoder(respBody).Decode(&vol)
return volInfo, err return vol, err
} }
// DeleteVol - Deletes a volume over the network. // DeleteVol - Deletes a volume over the network.
func (client *storageRESTClient) DeleteVol(volume string, forceDelete bool) (err error) { func (client *storageRESTClient) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTVolume, volume) values.Set(storageRESTVolume, volume)
if forceDelete { if forceDelete {
values.Set(storageRESTForceDelete, "true") values.Set(storageRESTForceDelete, "true")
} }
respBody, err := client.call(storageRESTMethodDeleteVol, values, nil, -1) respBody, err := client.call(ctx, storageRESTMethodDeleteVol, values, nil, -1)
defer http.DrainBody(respBody) defer http.DrainBody(respBody)
return err return err
} }
// AppendFile - append to a file. // AppendFile - append to a file.
func (client *storageRESTClient) AppendFile(volume, path string, buffer []byte) error { func (client *storageRESTClient) AppendFile(ctx context.Context, volume string, path string, buf []byte) error {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTVolume, volume) values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path) values.Set(storageRESTFilePath, path)
reader := bytes.NewReader(buffer) reader := bytes.NewReader(buf)
respBody, err := client.call(storageRESTMethodAppendFile, values, reader, -1) respBody, err := client.call(ctx, storageRESTMethodAppendFile, values, reader, -1)
defer http.DrainBody(respBody) defer http.DrainBody(respBody)
return err return err
} }
func (client *storageRESTClient) CreateFile(volume, path string, length int64, r io.Reader) error { func (client *storageRESTClient) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTVolume, volume) values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path) values.Set(storageRESTFilePath, path)
values.Set(storageRESTLength, strconv.Itoa(int(length))) values.Set(storageRESTLength, strconv.Itoa(int(size)))
respBody, err := client.call(storageRESTMethodCreateFile, values, ioutil.NopCloser(r), length) respBody, err := client.call(ctx, storageRESTMethodCreateFile, values, ioutil.NopCloser(reader), size)
defer http.DrainBody(respBody) defer http.DrainBody(respBody)
return err return err
} }
func (client *storageRESTClient) WriteMetadata(volume, path string, fi FileInfo) error { func (client *storageRESTClient) WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) error {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTVolume, volume) values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path) values.Set(storageRESTFilePath, path)
@ -286,12 +284,12 @@ func (client *storageRESTClient) WriteMetadata(volume, path string, fi FileInfo)
return err return err
} }
respBody, err := client.call(storageRESTMethodWriteMetadata, values, &reader, -1) respBody, err := client.call(ctx, storageRESTMethodWriteMetadata, values, &reader, -1)
defer http.DrainBody(respBody) defer http.DrainBody(respBody)
return err return err
} }
func (client *storageRESTClient) DeleteVersion(volume, path string, fi FileInfo) error { func (client *storageRESTClient) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo) error {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTVolume, volume) values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path) values.Set(storageRESTFilePath, path)
@ -301,33 +299,33 @@ func (client *storageRESTClient) DeleteVersion(volume, path string, fi FileInfo)
return err return err
} }
respBody, err := client.call(storageRESTMethodDeleteVersion, values, &buffer, -1) respBody, err := client.call(ctx, storageRESTMethodDeleteVersion, values, &buffer, -1)
defer http.DrainBody(respBody) defer http.DrainBody(respBody)
return err return err
} }
// WriteAll - write all data to a file. // WriteAll - write all data to a file.
func (client *storageRESTClient) WriteAll(volume, path string, reader io.Reader) error { func (client *storageRESTClient) WriteAll(ctx context.Context, volume string, path string, reader io.Reader) error {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTVolume, volume) values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path) values.Set(storageRESTFilePath, path)
respBody, err := client.call(storageRESTMethodWriteAll, values, reader, -1) respBody, err := client.call(ctx, storageRESTMethodWriteAll, values, reader, -1)
defer http.DrainBody(respBody) defer http.DrainBody(respBody)
return err return err
} }
// CheckFile - stat a file metadata. // CheckFile - stat a file metadata.
func (client *storageRESTClient) CheckFile(volume, path string) error { func (client *storageRESTClient) CheckFile(ctx context.Context, volume string, path string) error {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTVolume, volume) values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path) values.Set(storageRESTFilePath, path)
respBody, err := client.call(storageRESTMethodCheckFile, values, nil, -1) respBody, err := client.call(ctx, storageRESTMethodCheckFile, values, nil, -1)
defer http.DrainBody(respBody) defer http.DrainBody(respBody)
return err return err
} }
// CheckParts - stat all file parts. // CheckParts - stat all file parts.
func (client *storageRESTClient) CheckParts(volume, path string, fi FileInfo) error { func (client *storageRESTClient) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTVolume, volume) values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path) values.Set(storageRESTFilePath, path)
@ -338,32 +336,32 @@ func (client *storageRESTClient) CheckParts(volume, path string, fi FileInfo) er
return err return err
} }
respBody, err := client.call(storageRESTMethodCheckParts, values, &reader, -1) respBody, err := client.call(ctx, storageRESTMethodCheckParts, values, &reader, -1)
defer http.DrainBody(respBody) defer http.DrainBody(respBody)
return err return err
} }
// RenameData - rename source path to destination path atomically, metadata and data file. // RenameData - rename source path to destination path atomically, metadata and data file.
func (client *storageRESTClient) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath string) (err error) { func (client *storageRESTClient) RenameData(ctx context.Context, srcVolume, srcPath, dataDir, dstVolume, dstPath string) (err error) {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTSrcVolume, srcVolume) values.Set(storageRESTSrcVolume, srcVolume)
values.Set(storageRESTSrcPath, srcPath) values.Set(storageRESTSrcPath, srcPath)
values.Set(storageRESTDataDir, dataDir) values.Set(storageRESTDataDir, dataDir)
values.Set(storageRESTDstVolume, dstVolume) values.Set(storageRESTDstVolume, dstVolume)
values.Set(storageRESTDstPath, dstPath) values.Set(storageRESTDstPath, dstPath)
respBody, err := client.call(storageRESTMethodRenameData, values, nil, -1) respBody, err := client.call(ctx, storageRESTMethodRenameData, values, nil, -1)
defer http.DrainBody(respBody) defer http.DrainBody(respBody)
return err return err
} }
func (client *storageRESTClient) ReadVersion(volume, path, versionID string) (fi FileInfo, err error) { func (client *storageRESTClient) ReadVersion(ctx context.Context, volume, path, versionID string) (fi FileInfo, err error) {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTVolume, volume) values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path) values.Set(storageRESTFilePath, path)
values.Set(storageRESTVersionID, versionID) values.Set(storageRESTVersionID, versionID)
respBody, err := client.call(storageRESTMethodReadVersion, values, nil, -1) respBody, err := client.call(ctx, storageRESTMethodReadVersion, values, nil, -1)
if err != nil { if err != nil {
return fi, err return fi, err
} }
@ -374,11 +372,11 @@ func (client *storageRESTClient) ReadVersion(volume, path, versionID string) (fi
} }
// ReadAll - reads all contents of a file. // ReadAll - reads all contents of a file.
func (client *storageRESTClient) ReadAll(volume, path string) ([]byte, error) { func (client *storageRESTClient) ReadAll(ctx context.Context, volume string, path string) ([]byte, error) {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTVolume, volume) values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path) values.Set(storageRESTFilePath, path)
respBody, err := client.call(storageRESTMethodReadAll, values, nil, -1) respBody, err := client.call(ctx, storageRESTMethodReadAll, values, nil, -1)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -387,13 +385,13 @@ func (client *storageRESTClient) ReadAll(volume, path string) ([]byte, error) {
} }
// ReadFileStream - returns a reader for the requested file. // ReadFileStream - returns a reader for the requested file.
func (client *storageRESTClient) ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error) { func (client *storageRESTClient) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTVolume, volume) values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path) values.Set(storageRESTFilePath, path)
values.Set(storageRESTOffset, strconv.Itoa(int(offset))) values.Set(storageRESTOffset, strconv.Itoa(int(offset)))
values.Set(storageRESTLength, strconv.Itoa(int(length))) values.Set(storageRESTLength, strconv.Itoa(int(length)))
respBody, err := client.call(storageRESTMethodReadFileStream, values, nil, -1) respBody, err := client.call(ctx, storageRESTMethodReadFileStream, values, nil, -1)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -401,12 +399,12 @@ func (client *storageRESTClient) ReadFileStream(volume, path string, offset, len
} }
// ReadFile - reads section of a file. // ReadFile - reads section of a file.
func (client *storageRESTClient) ReadFile(volume, path string, offset int64, buffer []byte, verifier *BitrotVerifier) (int64, error) { func (client *storageRESTClient) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (int64, error) {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTVolume, volume) values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path) values.Set(storageRESTFilePath, path)
values.Set(storageRESTOffset, strconv.Itoa(int(offset))) values.Set(storageRESTOffset, strconv.Itoa(int(offset)))
values.Set(storageRESTLength, strconv.Itoa(len(buffer))) values.Set(storageRESTLength, strconv.Itoa(len(buf)))
if verifier != nil { if verifier != nil {
values.Set(storageRESTBitrotAlgo, verifier.algorithm.String()) values.Set(storageRESTBitrotAlgo, verifier.algorithm.String())
values.Set(storageRESTBitrotHash, hex.EncodeToString(verifier.sum)) values.Set(storageRESTBitrotHash, hex.EncodeToString(verifier.sum))
@ -414,21 +412,21 @@ func (client *storageRESTClient) ReadFile(volume, path string, offset int64, buf
values.Set(storageRESTBitrotAlgo, "") values.Set(storageRESTBitrotAlgo, "")
values.Set(storageRESTBitrotHash, "") values.Set(storageRESTBitrotHash, "")
} }
respBody, err := client.call(storageRESTMethodReadFile, values, nil, -1) respBody, err := client.call(ctx, storageRESTMethodReadFile, values, nil, -1)
if err != nil { if err != nil {
return 0, err return 0, err
} }
defer http.DrainBody(respBody) defer http.DrainBody(respBody)
n, err := io.ReadFull(respBody, buffer) n, err := io.ReadFull(respBody, buf)
return int64(n), err return int64(n), err
} }
func (client *storageRESTClient) WalkSplunk(volume, dirPath, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error) { func (client *storageRESTClient) WalkSplunk(ctx context.Context, volume, dirPath, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error) {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTVolume, volume) values.Set(storageRESTVolume, volume)
values.Set(storageRESTDirPath, dirPath) values.Set(storageRESTDirPath, dirPath)
values.Set(storageRESTMarkerPath, marker) values.Set(storageRESTMarkerPath, marker)
respBody, err := client.call(storageRESTMethodWalkSplunk, values, nil, -1) respBody, err := client.call(ctx, storageRESTMethodWalkSplunk, values, nil, -1)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -457,13 +455,13 @@ func (client *storageRESTClient) WalkSplunk(volume, dirPath, marker string, endW
return ch, nil return ch, nil
} }
func (client *storageRESTClient) WalkVersions(volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error) { func (client *storageRESTClient) WalkVersions(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error) {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTVolume, volume) values.Set(storageRESTVolume, volume)
values.Set(storageRESTDirPath, dirPath) values.Set(storageRESTDirPath, dirPath)
values.Set(storageRESTMarkerPath, marker) values.Set(storageRESTMarkerPath, marker)
values.Set(storageRESTRecursive, strconv.FormatBool(recursive)) values.Set(storageRESTRecursive, strconv.FormatBool(recursive))
respBody, err := client.call(storageRESTMethodWalkVersions, values, nil, -1) respBody, err := client.call(ctx, storageRESTMethodWalkVersions, values, nil, -1)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -494,13 +492,13 @@ func (client *storageRESTClient) WalkVersions(volume, dirPath, marker string, re
return ch, nil return ch, nil
} }
func (client *storageRESTClient) Walk(volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) { func (client *storageRESTClient) Walk(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTVolume, volume) values.Set(storageRESTVolume, volume)
values.Set(storageRESTDirPath, dirPath) values.Set(storageRESTDirPath, dirPath)
values.Set(storageRESTMarkerPath, marker) values.Set(storageRESTMarkerPath, marker)
values.Set(storageRESTRecursive, strconv.FormatBool(recursive)) values.Set(storageRESTRecursive, strconv.FormatBool(recursive))
respBody, err := client.call(storageRESTMethodWalk, values, nil, -1) respBody, err := client.call(ctx, storageRESTMethodWalk, values, nil, -1)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -530,12 +528,12 @@ func (client *storageRESTClient) Walk(volume, dirPath, marker string, recursive
} }
// ListDir - lists a directory. // ListDir - lists a directory.
func (client *storageRESTClient) ListDir(volume, dirPath string, count int) (entries []string, err error) { func (client *storageRESTClient) ListDir(ctx context.Context, volume, dirPath string, count int) (entries []string, err error) {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTVolume, volume) values.Set(storageRESTVolume, volume)
values.Set(storageRESTDirPath, dirPath) values.Set(storageRESTDirPath, dirPath)
values.Set(storageRESTCount, strconv.Itoa(count)) values.Set(storageRESTCount, strconv.Itoa(count))
respBody, err := client.call(storageRESTMethodListDir, values, nil, -1) respBody, err := client.call(ctx, storageRESTMethodListDir, values, nil, -1)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -545,17 +543,17 @@ func (client *storageRESTClient) ListDir(volume, dirPath string, count int) (ent
} }
// DeleteFile - deletes a file. // DeleteFile - deletes a file.
func (client *storageRESTClient) DeleteFile(volume, path string) error { func (client *storageRESTClient) DeleteFile(ctx context.Context, volume string, path string) error {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTVolume, volume) values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path) values.Set(storageRESTFilePath, path)
respBody, err := client.call(storageRESTMethodDeleteFile, values, nil, -1) respBody, err := client.call(ctx, storageRESTMethodDeleteFile, values, nil, -1)
defer http.DrainBody(respBody) defer http.DrainBody(respBody)
return err return err
} }
// DeleteVersions - deletes list of specified versions if present // DeleteVersions - deletes list of specified versions if present
func (client *storageRESTClient) DeleteVersions(volume string, versions []FileInfo) (errs []error) { func (client *storageRESTClient) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) (errs []error) {
if len(versions) == 0 { if len(versions) == 0 {
return errs return errs
} }
@ -572,7 +570,7 @@ func (client *storageRESTClient) DeleteVersions(volume string, versions []FileIn
errs = make([]error, len(versions)) errs = make([]error, len(versions))
respBody, err := client.call(storageRESTMethodDeleteVersions, values, &buffer, -1) respBody, err := client.call(ctx, storageRESTMethodDeleteVersions, values, &buffer, -1)
defer http.DrainBody(respBody) defer http.DrainBody(respBody)
if err != nil { if err != nil {
for i := range errs { for i := range errs {
@ -605,18 +603,18 @@ func (client *storageRESTClient) DeleteVersions(volume string, versions []FileIn
} }
// RenameFile - renames a file. // RenameFile - renames a file.
func (client *storageRESTClient) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err error) { func (client *storageRESTClient) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) (err error) {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTSrcVolume, srcVolume) values.Set(storageRESTSrcVolume, srcVolume)
values.Set(storageRESTSrcPath, srcPath) values.Set(storageRESTSrcPath, srcPath)
values.Set(storageRESTDstVolume, dstVolume) values.Set(storageRESTDstVolume, dstVolume)
values.Set(storageRESTDstPath, dstPath) values.Set(storageRESTDstPath, dstPath)
respBody, err := client.call(storageRESTMethodRenameFile, values, nil, -1) respBody, err := client.call(ctx, storageRESTMethodRenameFile, values, nil, -1)
defer http.DrainBody(respBody) defer http.DrainBody(respBody)
return err return err
} }
func (client *storageRESTClient) VerifyFile(volume, path string, fi FileInfo) error { func (client *storageRESTClient) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error {
values := make(url.Values) values := make(url.Values)
values.Set(storageRESTVolume, volume) values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path) values.Set(storageRESTFilePath, path)
@ -626,7 +624,7 @@ func (client *storageRESTClient) VerifyFile(volume, path string, fi FileInfo) er
return err return err
} }
respBody, err := client.call(storageRESTMethodVerifyFile, values, &reader, -1) respBody, err := client.call(ctx, storageRESTMethodVerifyFile, values, &reader, -1)
defer http.DrainBody(respBody) defer http.DrainBody(respBody)
if err != nil { if err != nil {
return err return err
@ -674,7 +672,7 @@ func newStorageRESTClient(endpoint Endpoint) *storageRESTClient {
ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout) ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
// Instantiate a new rest client for healthcheck // Instantiate a new rest client for healthcheck
// to avoid recursive healthCheckFn() // to avoid recursive healthCheckFn()
respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).CallWithContext(ctx, storageRESTMethodHealth, nil, nil, -1) respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).Call(ctx, storageRESTMethodHealth, nil, nil, -1)
xhttp.DrainBody(respBody) xhttp.DrainBody(respBody)
cancel() cancel()
return !errors.Is(err, context.DeadlineExceeded) && toStorageErr(err) != errDiskNotFound return !errors.Is(err, context.DeadlineExceeded) && toStorageErr(err) != errDiskNotFound

52
cmd/storage-rest-server.go
View File

@ -138,7 +138,7 @@ func (s *storageRESTServer) DiskInfoHandler(w http.ResponseWriter, r *http.Reque
if !s.IsValid(w, r) { if !s.IsValid(w, r) {
return return
} }
info, err := s.storage.DiskInfo() info, err := s.storage.DiskInfo(r.Context())
if err != nil { if err != nil {
info.Error = err.Error() info.Error = err.Error()
} }
@ -179,7 +179,7 @@ func (s *storageRESTServer) MakeVolHandler(w http.ResponseWriter, r *http.Reques
} }
vars := mux.Vars(r) vars := mux.Vars(r)
volume := vars[storageRESTVolume] volume := vars[storageRESTVolume]
err := s.storage.MakeVol(volume) err := s.storage.MakeVol(r.Context(), volume)
if err != nil { if err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
} }
@ -192,7 +192,7 @@ func (s *storageRESTServer) MakeVolBulkHandler(w http.ResponseWriter, r *http.Re
} }
vars := mux.Vars(r) vars := mux.Vars(r)
volumes := strings.Split(vars[storageRESTVolumes], ",") volumes := strings.Split(vars[storageRESTVolumes], ",")
err := s.storage.MakeVolBulk(volumes...) err := s.storage.MakeVolBulk(r.Context(), volumes...)
if err != nil { if err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
} }
@ -203,7 +203,7 @@ func (s *storageRESTServer) ListVolsHandler(w http.ResponseWriter, r *http.Reque
if !s.IsValid(w, r) { if !s.IsValid(w, r) {
return return
} }
infos, err := s.storage.ListVols() infos, err := s.storage.ListVols(r.Context())
if err != nil { if err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
return return
@ -219,7 +219,7 @@ func (s *storageRESTServer) StatVolHandler(w http.ResponseWriter, r *http.Reques
} }
vars := mux.Vars(r) vars := mux.Vars(r)
volume := vars[storageRESTVolume] volume := vars[storageRESTVolume]
info, err := s.storage.StatVol(volume) info, err := s.storage.StatVol(r.Context(), volume)
if err != nil { if err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
return return
@ -236,7 +236,7 @@ func (s *storageRESTServer) DeleteVolHandler(w http.ResponseWriter, r *http.Requ
vars := mux.Vars(r) vars := mux.Vars(r)
volume := vars[storageRESTVolume] volume := vars[storageRESTVolume]
forceDelete := vars[storageRESTForceDelete] == "true" forceDelete := vars[storageRESTForceDelete] == "true"
err := s.storage.DeleteVol(volume, forceDelete) err := s.storage.DeleteVol(r.Context(), volume, forceDelete)
if err != nil { if err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
} }
@ -257,7 +257,7 @@ func (s *storageRESTServer) AppendFileHandler(w http.ResponseWriter, r *http.Req
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
return return
} }
err = s.storage.AppendFile(volume, filePath, buf) err = s.storage.AppendFile(r.Context(), volume, filePath, buf)
if err != nil { if err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
} }
@ -278,7 +278,7 @@ func (s *storageRESTServer) CreateFileHandler(w http.ResponseWriter, r *http.Req
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
return return
} }
err = s.storage.CreateFile(volume, filePath, int64(fileSize), r.Body) err = s.storage.CreateFile(r.Context(), volume, filePath, int64(fileSize), r.Body)
if err != nil { if err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
} }
@ -304,7 +304,7 @@ func (s *storageRESTServer) DeleteVersionHandler(w http.ResponseWriter, r *http.
return return
} }
err := s.storage.DeleteVersion(volume, filePath, fi) err := s.storage.DeleteVersion(r.Context(), volume, filePath, fi)
if err != nil { if err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
} }
@ -320,7 +320,7 @@ func (s *storageRESTServer) ReadVersionHandler(w http.ResponseWriter, r *http.Re
filePath := vars[storageRESTFilePath] filePath := vars[storageRESTFilePath]
versionID := vars[storageRESTVersionID] versionID := vars[storageRESTVersionID]
fi, err := s.storage.ReadVersion(volume, filePath, versionID) fi, err := s.storage.ReadVersion(r.Context(), volume, filePath, versionID)
if err != nil { if err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
return return
@ -351,7 +351,7 @@ func (s *storageRESTServer) WriteMetadataHandler(w http.ResponseWriter, r *http.
return return
} }
err = s.storage.WriteMetadata(volume, filePath, fi) err = s.storage.WriteMetadata(r.Context(), volume, filePath, fi)
if err != nil { if err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
} }
@ -371,7 +371,7 @@ func (s *storageRESTServer) WriteAllHandler(w http.ResponseWriter, r *http.Reque
return return
} }
err := s.storage.WriteAll(volume, filePath, io.LimitReader(r.Body, r.ContentLength)) err := s.storage.WriteAll(r.Context(), volume, filePath, io.LimitReader(r.Body, r.ContentLength))
if err != nil { if err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
} }
@ -397,7 +397,7 @@ func (s *storageRESTServer) CheckPartsHandler(w http.ResponseWriter, r *http.Req
return return
} }
if err := s.storage.CheckParts(volume, filePath, fi); err != nil { if err := s.storage.CheckParts(r.Context(), volume, filePath, fi); err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
} }
} }
@ -411,7 +411,7 @@ func (s *storageRESTServer) CheckFileHandler(w http.ResponseWriter, r *http.Requ
volume := vars[storageRESTVolume] volume := vars[storageRESTVolume]
filePath := vars[storageRESTFilePath] filePath := vars[storageRESTFilePath]
if err := s.storage.CheckFile(volume, filePath); err != nil { if err := s.storage.CheckFile(r.Context(), volume, filePath); err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
} }
} }
@ -425,7 +425,7 @@ func (s *storageRESTServer) ReadAllHandler(w http.ResponseWriter, r *http.Reques
volume := vars[storageRESTVolume] volume := vars[storageRESTVolume]
filePath := vars[storageRESTFilePath] filePath := vars[storageRESTFilePath]
buf, err := s.storage.ReadAll(volume, filePath) buf, err := s.storage.ReadAll(r.Context(), volume, filePath)
if err != nil { if err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
return return
@ -469,7 +469,7 @@ func (s *storageRESTServer) ReadFileHandler(w http.ResponseWriter, r *http.Reque
verifier = NewBitrotVerifier(BitrotAlgorithmFromString(vars[storageRESTBitrotAlgo]), hash) verifier = NewBitrotVerifier(BitrotAlgorithmFromString(vars[storageRESTBitrotAlgo]), hash)
} }
buf := make([]byte, length) buf := make([]byte, length)
_, err = s.storage.ReadFile(volume, filePath, int64(offset), buf, verifier) _, err = s.storage.ReadFile(r.Context(), volume, filePath, int64(offset), buf, verifier)
if err != nil { if err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
return return
@ -498,7 +498,7 @@ func (s *storageRESTServer) ReadFileStreamHandler(w http.ResponseWriter, r *http
return return
} }
rc, err := s.storage.ReadFileStream(volume, filePath, int64(offset), int64(length)) rc, err := s.storage.ReadFileStream(r.Context(), volume, filePath, int64(offset), int64(length))
if err != nil { if err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
return return
@ -524,7 +524,7 @@ func (s *storageRESTServer) WalkSplunkHandler(w http.ResponseWriter, r *http.Req
setEventStreamHeaders(w) setEventStreamHeaders(w)
encoder := gob.NewEncoder(w) encoder := gob.NewEncoder(w)
fch, err := s.storage.WalkSplunk(volume, dirPath, markerPath, r.Context().Done()) fch, err := s.storage.WalkSplunk(r.Context(), volume, dirPath, markerPath, r.Context().Done())
if err != nil { if err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
return return
@ -552,7 +552,7 @@ func (s *storageRESTServer) WalkVersionsHandler(w http.ResponseWriter, r *http.R
setEventStreamHeaders(w) setEventStreamHeaders(w)
encoder := gob.NewEncoder(w) encoder := gob.NewEncoder(w)
fch, err := s.storage.WalkVersions(volume, dirPath, markerPath, recursive, r.Context().Done()) fch, err := s.storage.WalkVersions(r.Context(), volume, dirPath, markerPath, recursive, r.Context().Done())
if err != nil { if err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
return return
@ -580,7 +580,7 @@ func (s *storageRESTServer) WalkHandler(w http.ResponseWriter, r *http.Request)
setEventStreamHeaders(w) setEventStreamHeaders(w)
encoder := gob.NewEncoder(w) encoder := gob.NewEncoder(w)
fch, err := s.storage.Walk(volume, dirPath, markerPath, recursive, r.Context().Done()) fch, err := s.storage.Walk(r.Context(), volume, dirPath, markerPath, recursive, r.Context().Done())
if err != nil { if err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
return return
@ -604,7 +604,7 @@ func (s *storageRESTServer) ListDirHandler(w http.ResponseWriter, r *http.Reques
return return
} }
entries, err := s.storage.ListDir(volume, dirPath, count) entries, err := s.storage.ListDir(r.Context(), volume, dirPath, count)
if err != nil { if err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
return return
@ -622,7 +622,7 @@ func (s *storageRESTServer) DeleteFileHandler(w http.ResponseWriter, r *http.Req
volume := vars[storageRESTVolume] volume := vars[storageRESTVolume]
filePath := vars[storageRESTFilePath] filePath := vars[storageRESTFilePath]
err := s.storage.DeleteFile(volume, filePath) err := s.storage.DeleteFile(r.Context(), volume, filePath)
if err != nil { if err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
} }
@ -663,7 +663,7 @@ func (s *storageRESTServer) DeleteVersionsHandler(w http.ResponseWriter, r *http
setEventStreamHeaders(w) setEventStreamHeaders(w)
encoder := gob.NewEncoder(w) encoder := gob.NewEncoder(w)
done := keepHTTPResponseAlive(w) done := keepHTTPResponseAlive(w)
errs := s.storage.DeleteVersions(volume, versions) errs := s.storage.DeleteVersions(r.Context(), volume, versions)
done(nil) done(nil)
for idx := range versions { for idx := range versions {
if errs[idx] != nil { if errs[idx] != nil {
@ -685,7 +685,7 @@ func (s *storageRESTServer) RenameDataHandler(w http.ResponseWriter, r *http.Req
dataDir := vars[storageRESTDataDir] dataDir := vars[storageRESTDataDir]
dstVolume := vars[storageRESTDstVolume] dstVolume := vars[storageRESTDstVolume]
dstFilePath := vars[storageRESTDstPath] dstFilePath := vars[storageRESTDstPath]
err := s.storage.RenameData(srcVolume, srcFilePath, dataDir, dstVolume, dstFilePath) err := s.storage.RenameData(r.Context(), srcVolume, srcFilePath, dataDir, dstVolume, dstFilePath)
if err != nil { if err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
} }
@ -701,7 +701,7 @@ func (s *storageRESTServer) RenameFileHandler(w http.ResponseWriter, r *http.Req
srcFilePath := vars[storageRESTSrcPath] srcFilePath := vars[storageRESTSrcPath]
dstVolume := vars[storageRESTDstVolume] dstVolume := vars[storageRESTDstVolume]
dstFilePath := vars[storageRESTDstPath] dstFilePath := vars[storageRESTDstPath]
err := s.storage.RenameFile(srcVolume, srcFilePath, dstVolume, dstFilePath) err := s.storage.RenameFile(r.Context(), srcVolume, srcFilePath, dstVolume, dstFilePath)
if err != nil { if err != nil {
s.writeErrorResponse(w, err) s.writeErrorResponse(w, err)
} }
@ -809,7 +809,7 @@ func (s *storageRESTServer) VerifyFileHandler(w http.ResponseWriter, r *http.Req
setEventStreamHeaders(w) setEventStreamHeaders(w)
encoder := gob.NewEncoder(w) encoder := gob.NewEncoder(w)
done := keepHTTPResponseAlive(w) done := keepHTTPResponseAlive(w)
err = s.storage.VerifyFile(volume, filePath, fi) err = s.storage.VerifyFile(r.Context(), volume, filePath, fi)
done(nil) done(nil)
vresp := &VerifyFileResp{} vresp := &VerifyFileResp{}
if err != nil { if err != nil {

61
cmd/storage-rest_test.go
View File

@ -17,6 +17,7 @@
package cmd package cmd
import ( import (
"context"
"io/ioutil" "io/ioutil"
"net/http/httptest" "net/http/httptest"
"os" "os"
@ -43,7 +44,7 @@ func testStorageAPIDiskInfo(t *testing.T, storage StorageAPI) {
} }
for i, testCase := range testCases { for i, testCase := range testCases {
_, err := storage.DiskInfo() _, err := storage.DiskInfo(context.Background())
expectErr := (err != nil) expectErr := (err != nil)
if expectErr != testCase.expectErr { if expectErr != testCase.expectErr {
@ -66,7 +67,7 @@ func testStorageAPIMakeVol(t *testing.T, storage StorageAPI) {
} }
for i, testCase := range testCases { for i, testCase := range testCases {
err := storage.MakeVol(testCase.volumeName) err := storage.MakeVol(context.Background(), testCase.volumeName)
expectErr := (err != nil) expectErr := (err != nil)
if expectErr != testCase.expectErr { if expectErr != testCase.expectErr {
@ -87,13 +88,13 @@ func testStorageAPIListVols(t *testing.T, storage StorageAPI) {
for i, testCase := range testCases { for i, testCase := range testCases {
for _, volumeName := range testCase.volumeNames { for _, volumeName := range testCase.volumeNames {
err := storage.MakeVol(volumeName) err := storage.MakeVol(context.Background(), volumeName)
if err != nil { if err != nil {
t.Fatalf("unexpected error %v", err) t.Fatalf("unexpected error %v", err)
} }
} }
result, err := storage.ListVols() result, err := storage.ListVols(context.Background())
expectErr := (err != nil) expectErr := (err != nil)
if expectErr != testCase.expectErr { if expectErr != testCase.expectErr {
@ -109,7 +110,7 @@ func testStorageAPIListVols(t *testing.T, storage StorageAPI) {
} }
func testStorageAPIStatVol(t *testing.T, storage StorageAPI) { func testStorageAPIStatVol(t *testing.T, storage StorageAPI) {
err := storage.MakeVol("foo") err := storage.MakeVol(context.Background(), "foo")
if err != nil { if err != nil {
t.Fatalf("unexpected error %v", err) t.Fatalf("unexpected error %v", err)
} }
@ -124,7 +125,7 @@ func testStorageAPIStatVol(t *testing.T, storage StorageAPI) {
} }
for i, testCase := range testCases { for i, testCase := range testCases {
result, err := storage.StatVol(testCase.volumeName) result, err := storage.StatVol(context.Background(), testCase.volumeName)
expectErr := (err != nil) expectErr := (err != nil)
if expectErr != testCase.expectErr { if expectErr != testCase.expectErr {
@ -140,7 +141,7 @@ func testStorageAPIStatVol(t *testing.T, storage StorageAPI) {
} }
func testStorageAPIDeleteVol(t *testing.T, storage StorageAPI) { func testStorageAPIDeleteVol(t *testing.T, storage StorageAPI) {
err := storage.MakeVol("foo") err := storage.MakeVol(context.Background(), "foo")
if err != nil { if err != nil {
t.Fatalf("unexpected error %v", err) t.Fatalf("unexpected error %v", err)
} }
@ -155,7 +156,7 @@ func testStorageAPIDeleteVol(t *testing.T, storage StorageAPI) {
} }
for i, testCase := range testCases { for i, testCase := range testCases {
err := storage.DeleteVol(testCase.volumeName, false) err := storage.DeleteVol(context.Background(), testCase.volumeName, false)
expectErr := (err != nil) expectErr := (err != nil)
if expectErr != testCase.expectErr { if expectErr != testCase.expectErr {
@ -165,11 +166,11 @@ func testStorageAPIDeleteVol(t *testing.T, storage StorageAPI) {
} }
func testStorageAPICheckFile(t *testing.T, storage StorageAPI) { func testStorageAPICheckFile(t *testing.T, storage StorageAPI) {
err := storage.MakeVol("foo") err := storage.MakeVol(context.Background(), "foo")
if err != nil { if err != nil {
t.Fatalf("unexpected error %v", err) t.Fatalf("unexpected error %v", err)
} }
err = storage.AppendFile("foo", pathJoin("myobject", xlStorageFormatFile), []byte("foo")) err = storage.AppendFile(context.Background(), "foo", pathJoin("myobject", xlStorageFormatFile), []byte("foo"))
if err != nil { if err != nil {
t.Fatalf("unexpected error %v", err) t.Fatalf("unexpected error %v", err)
} }
@ -185,7 +186,7 @@ func testStorageAPICheckFile(t *testing.T, storage StorageAPI) {
} }
for i, testCase := range testCases { for i, testCase := range testCases {
err := storage.CheckFile(testCase.volumeName, testCase.objectName) err := storage.CheckFile(context.Background(), testCase.volumeName, testCase.objectName)
expectErr := (err != nil) expectErr := (err != nil)
if expectErr != testCase.expectErr { if expectErr != testCase.expectErr {
@ -195,11 +196,11 @@ func testStorageAPICheckFile(t *testing.T, storage StorageAPI) {
} }
func testStorageAPIListDir(t *testing.T, storage StorageAPI) { func testStorageAPIListDir(t *testing.T, storage StorageAPI) {
err := storage.MakeVol("foo") err := storage.MakeVol(context.Background(), "foo")
if err != nil { if err != nil {
t.Fatalf("unexpected error %v", err) t.Fatalf("unexpected error %v", err)
} }
err = storage.AppendFile("foo", "path/to/myobject", []byte("foo")) err = storage.AppendFile(context.Background(), "foo", "path/to/myobject", []byte("foo"))
if err != nil { if err != nil {
t.Fatalf("unexpected error %v", err) t.Fatalf("unexpected error %v", err)
} }
@ -216,7 +217,7 @@ func testStorageAPIListDir(t *testing.T, storage StorageAPI) {
} }
for i, testCase := range testCases { for i, testCase := range testCases {
result, err := storage.ListDir(testCase.volumeName, testCase.prefix, -1) result, err := storage.ListDir(context.Background(), testCase.volumeName, testCase.prefix, -1)
expectErr := (err != nil) expectErr := (err != nil)
if expectErr != testCase.expectErr { if expectErr != testCase.expectErr {
@ -232,11 +233,11 @@ func testStorageAPIListDir(t *testing.T, storage StorageAPI) {
} }
func testStorageAPIReadAll(t *testing.T, storage StorageAPI) { func testStorageAPIReadAll(t *testing.T, storage StorageAPI) {
err := storage.MakeVol("foo") err := storage.MakeVol(context.Background(), "foo")
if err != nil { if err != nil {
t.Fatalf("unexpected error %v", err) t.Fatalf("unexpected error %v", err)
} }
err = storage.AppendFile("foo", "myobject", []byte("foo")) err = storage.AppendFile(context.Background(), "foo", "myobject", []byte("foo"))
if err != nil { if err != nil {
t.Fatalf("unexpected error %v", err) t.Fatalf("unexpected error %v", err)
} }
@ -253,7 +254,7 @@ func testStorageAPIReadAll(t *testing.T, storage StorageAPI) {
} }
for i, testCase := range testCases { for i, testCase := range testCases {
result, err := storage.ReadAll(testCase.volumeName, testCase.objectName) result, err := storage.ReadAll(context.Background(), testCase.volumeName, testCase.objectName)
expectErr := (err != nil) expectErr := (err != nil)
if expectErr != testCase.expectErr { if expectErr != testCase.expectErr {
@ -269,11 +270,11 @@ func testStorageAPIReadAll(t *testing.T, storage StorageAPI) {
} }
func testStorageAPIReadFile(t *testing.T, storage StorageAPI) { func testStorageAPIReadFile(t *testing.T, storage StorageAPI) {
err := storage.MakeVol("foo") err := storage.MakeVol(context.Background(), "foo")
if err != nil { if err != nil {
t.Fatalf("unexpected error %v", err) t.Fatalf("unexpected error %v", err)
} }
err = storage.AppendFile("foo", "myobject", []byte("foo")) err = storage.AppendFile(context.Background(), "foo", "myobject", []byte("foo"))
if err != nil { if err != nil {
t.Fatalf("unexpected error %v", err) t.Fatalf("unexpected error %v", err)
} }
@ -294,7 +295,7 @@ func testStorageAPIReadFile(t *testing.T, storage StorageAPI) {
result := make([]byte, 100) result := make([]byte, 100)
for i, testCase := range testCases { for i, testCase := range testCases {
result = result[testCase.offset:3] result = result[testCase.offset:3]
_, err := storage.ReadFile(testCase.volumeName, testCase.objectName, testCase.offset, result, nil) _, err := storage.ReadFile(context.Background(), testCase.volumeName, testCase.objectName, testCase.offset, result, nil)
expectErr := (err != nil) expectErr := (err != nil)
if expectErr != testCase.expectErr { if expectErr != testCase.expectErr {
@ -310,7 +311,7 @@ func testStorageAPIReadFile(t *testing.T, storage StorageAPI) {
} }
func testStorageAPIAppendFile(t *testing.T, storage StorageAPI) { func testStorageAPIAppendFile(t *testing.T, storage StorageAPI) {
err := storage.MakeVol("foo") err := storage.MakeVol(context.Background(), "foo")
if err != nil { if err != nil {
t.Fatalf("unexpected error %v", err) t.Fatalf("unexpected error %v", err)
} }
@ -328,7 +329,7 @@ func testStorageAPIAppendFile(t *testing.T, storage StorageAPI) {
} }
for i, testCase := range testCases { for i, testCase := range testCases {
err := storage.AppendFile(testCase.volumeName, testCase.objectName, testCase.data) err := storage.AppendFile(context.Background(), testCase.volumeName, testCase.objectName, testCase.data)
expectErr := (err != nil) expectErr := (err != nil)
if expectErr != testCase.expectErr { if expectErr != testCase.expectErr {
@ -338,12 +339,12 @@ func testStorageAPIAppendFile(t *testing.T, storage StorageAPI) {
} }
func testStorageAPIDeleteFile(t *testing.T, storage StorageAPI) { func testStorageAPIDeleteFile(t *testing.T, storage StorageAPI) {
err := storage.MakeVol("foo") err := storage.MakeVol(context.Background(), "foo")
if err != nil { if err != nil {
t.Fatalf("unexpected error %v", err) t.Fatalf("unexpected error %v", err)
} }
err = storage.AppendFile("foo", "myobject", []byte("foo")) err = storage.AppendFile(context.Background(), "foo", "myobject", []byte("foo"))
if err != nil { if err != nil {
t.Fatalf("unexpected error %v", err) t.Fatalf("unexpected error %v", err)
} }
@ -361,7 +362,7 @@ func testStorageAPIDeleteFile(t *testing.T, storage StorageAPI) {
} }
for i, testCase := range testCases { for i, testCase := range testCases {
err := storage.DeleteFile(testCase.volumeName, testCase.objectName) err := storage.DeleteFile(context.Background(), testCase.volumeName, testCase.objectName)
expectErr := (err != nil) expectErr := (err != nil)
if expectErr != testCase.expectErr { if expectErr != testCase.expectErr {
@ -371,22 +372,22 @@ func testStorageAPIDeleteFile(t *testing.T, storage StorageAPI) {
} }
func testStorageAPIRenameFile(t *testing.T, storage StorageAPI) { func testStorageAPIRenameFile(t *testing.T, storage StorageAPI) {
err := storage.MakeVol("foo") err := storage.MakeVol(context.Background(), "foo")
if err != nil { if err != nil {
t.Fatalf("unexpected error %v", err) t.Fatalf("unexpected error %v", err)
} }
err = storage.MakeVol("bar") err = storage.MakeVol(context.Background(), "bar")
if err != nil { if err != nil {
t.Fatalf("unexpected error %v", err) t.Fatalf("unexpected error %v", err)
} }
err = storage.AppendFile("foo", "myobject", []byte("foo")) err = storage.AppendFile(context.Background(), "foo", "myobject", []byte("foo"))
if err != nil { if err != nil {
t.Fatalf("unexpected error %v", err) t.Fatalf("unexpected error %v", err)
} }
err = storage.AppendFile("foo", "otherobject", []byte("foo")) err = storage.AppendFile(context.Background(), "foo", "otherobject", []byte("foo"))
if err != nil { if err != nil {
t.Fatalf("unexpected error %v", err) t.Fatalf("unexpected error %v", err)
} }
@ -405,7 +406,7 @@ func testStorageAPIRenameFile(t *testing.T, storage StorageAPI) {
} }
for i, testCase := range testCases { for i, testCase := range testCases {
err := storage.RenameFile(testCase.volumeName, testCase.objectName, testCase.destVolumeName, testCase.destObjectName) err := storage.RenameFile(context.Background(), testCase.volumeName, testCase.objectName, testCase.destVolumeName, testCase.destObjectName)
expectErr := (err != nil) expectErr := (err != nil)
if expectErr != testCase.expectErr { if expectErr != testCase.expectErr {

16
cmd/tree-walk_test.go
View File

@ -69,14 +69,14 @@ func TestFilterMatchingPrefix(t *testing.T) {
// Helper function that creates a volume and files in it. // Helper function that creates a volume and files in it.
func createNamespace(disk StorageAPI, volume string, files []string) error { func createNamespace(disk StorageAPI, volume string, files []string) error {
// Make a volume. // Make a volume.
err := disk.MakeVol(volume) err := disk.MakeVol(context.Background(), volume)
if err != nil { if err != nil {
return err return err
} }
// Create files. // Create files.
for _, file := range files { for _, file := range files {
err = disk.AppendFile(volume, file, []byte{}) err = disk.AppendFile(context.Background(), volume, file, []byte{})
if err != nil { if err != nil {
return err return err
} }
@ -88,7 +88,7 @@ func createNamespace(disk StorageAPI, volume string, files []string) error {
// disks - used for doing disk.ListDir() // disks - used for doing disk.ListDir()
func listDirFactory(ctx context.Context, disk StorageAPI, isLeaf IsLeafFunc) ListDirFunc { func listDirFactory(ctx context.Context, disk StorageAPI, isLeaf IsLeafFunc) ListDirFunc {
return func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string, delayIsLeaf bool) { return func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string, delayIsLeaf bool) {
entries, err := disk.ListDir(volume, dirPath, -1) entries, err := disk.ListDir(ctx, volume, dirPath, -1)
if err != nil { if err != nil {
return false, nil, false return false, nil, false
} }
@ -163,7 +163,7 @@ func TestTreeWalk(t *testing.T) {
} }
isLeafDir := func(bucket, leafPath string) bool { isLeafDir := func(bucket, leafPath string) bool {
entries, _ := disk.ListDir(bucket, leafPath, 1) entries, _ := disk.ListDir(context.Background(), bucket, leafPath, 1)
return len(entries) == 0 return len(entries) == 0
} }
@ -207,7 +207,7 @@ func TestTreeWalkTimeout(t *testing.T) {
} }
isLeafDir := func(bucket, leafPath string) bool { isLeafDir := func(bucket, leafPath string) bool {
entries, _ := disk.ListDir(bucket, leafPath, 1) entries, _ := disk.ListDir(context.Background(), bucket, leafPath, 1)
return len(entries) == 0 return len(entries) == 0
} }
@ -275,7 +275,7 @@ func TestRecursiveTreeWalk(t *testing.T) {
} }
isLeafDir := func(bucket, leafPath string) bool { isLeafDir := func(bucket, leafPath string) bool {
entries, _ := disk1.ListDir(bucket, leafPath, 1) entries, _ := disk1.ListDir(context.Background(), bucket, leafPath, 1)
return len(entries) == 0 return len(entries) == 0
} }
@ -389,7 +389,7 @@ func TestSortedness(t *testing.T) {
} }
isLeafDir := func(bucket, leafPath string) bool { isLeafDir := func(bucket, leafPath string) bool {
entries, _ := disk1.ListDir(bucket, leafPath, 1) entries, _ := disk1.ListDir(context.Background(), bucket, leafPath, 1)
return len(entries) == 0 return len(entries) == 0
} }
@ -469,7 +469,7 @@ func TestTreeWalkIsEnd(t *testing.T) {
} }
isLeafDir := func(bucket, leafPath string) bool { isLeafDir := func(bucket, leafPath string) bool {
entries, _ := disk1.ListDir(bucket, leafPath, 1) entries, _ := disk1.ListDir(context.Background(), bucket, leafPath, 1)
return len(entries) == 0 return len(entries) == 0
} }

104
cmd/xl-storage-disk-id-check.go
View File

@ -84,8 +84,8 @@ func (p *xlStorageDiskIDCheck) checkDiskStale() error {
return errDiskNotFound return errDiskNotFound
} }
func (p *xlStorageDiskIDCheck) DiskInfo() (info DiskInfo, err error) { func (p *xlStorageDiskIDCheck) DiskInfo(ctx context.Context) (info DiskInfo, err error) {
info, err = p.storage.DiskInfo() info, err = p.storage.DiskInfo(ctx)
if err != nil { if err != nil {
return info, err return info, err
} }
@ -99,145 +99,145 @@ func (p *xlStorageDiskIDCheck) DiskInfo() (info DiskInfo, err error) {
return info, nil return info, nil
} }
func (p *xlStorageDiskIDCheck) MakeVolBulk(volumes ...string) (err error) { func (p *xlStorageDiskIDCheck) MakeVolBulk(ctx context.Context, volumes ...string) (err error) {
if err = p.checkDiskStale(); err != nil { if err = p.checkDiskStale(); err != nil {
return err return err
} }
return p.storage.MakeVolBulk(volumes...) return p.storage.MakeVolBulk(ctx, volumes...)
} }
func (p *xlStorageDiskIDCheck) MakeVol(volume string) (err error) { func (p *xlStorageDiskIDCheck) MakeVol(ctx context.Context, volume string) (err error) {
if err = p.checkDiskStale(); err != nil { if err = p.checkDiskStale(); err != nil {
return err return err
} }
return p.storage.MakeVol(volume) return p.storage.MakeVol(ctx, volume)
} }
func (p *xlStorageDiskIDCheck) ListVols() ([]VolInfo, error) { func (p *xlStorageDiskIDCheck) ListVols(ctx context.Context) ([]VolInfo, error) {
if err := p.checkDiskStale(); err != nil { if err := p.checkDiskStale(); err != nil {
return nil, err return nil, err
} }
return p.storage.ListVols() return p.storage.ListVols(ctx)
} }
func (p *xlStorageDiskIDCheck) StatVol(volume string) (vol VolInfo, err error) { func (p *xlStorageDiskIDCheck) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) {
if err = p.checkDiskStale(); err != nil { if err = p.checkDiskStale(); err != nil {
return vol, err return vol, err
} }
return p.storage.StatVol(volume) return p.storage.StatVol(ctx, volume)
} }
func (p *xlStorageDiskIDCheck) DeleteVol(volume string, forceDelete bool) (err error) { func (p *xlStorageDiskIDCheck) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) {
if err = p.checkDiskStale(); err != nil { if err = p.checkDiskStale(); err != nil {
return err return err
} }
return p.storage.DeleteVol(volume, forceDelete) return p.storage.DeleteVol(ctx, volume, forceDelete)
} }
func (p *xlStorageDiskIDCheck) WalkVersions(volume, dirPath string, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error) { func (p *xlStorageDiskIDCheck) WalkVersions(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error) {
if err := p.checkDiskStale(); err != nil { if err := p.checkDiskStale(); err != nil {
return nil, err return nil, err
} }
return p.storage.WalkVersions(volume, dirPath, marker, recursive, endWalkCh) return p.storage.WalkVersions(ctx, volume, dirPath, marker, recursive, endWalkCh)
} }
func (p *xlStorageDiskIDCheck) Walk(volume, dirPath string, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) { func (p *xlStorageDiskIDCheck) Walk(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) {
if err := p.checkDiskStale(); err != nil { if err := p.checkDiskStale(); err != nil {
return nil, err return nil, err
} }
return p.storage.Walk(volume, dirPath, marker, recursive, endWalkCh) return p.storage.Walk(ctx, volume, dirPath, marker, recursive, endWalkCh)
} }
func (p *xlStorageDiskIDCheck) WalkSplunk(volume, dirPath string, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error) { func (p *xlStorageDiskIDCheck) WalkSplunk(ctx context.Context, volume, dirPath, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error) {
if err := p.checkDiskStale(); err != nil { if err := p.checkDiskStale(); err != nil {
return nil, err return nil, err
} }
return p.storage.WalkSplunk(volume, dirPath, marker, endWalkCh) return p.storage.WalkSplunk(ctx, volume, dirPath, marker, endWalkCh)
} }
func (p *xlStorageDiskIDCheck) ListDir(volume, dirPath string, count int) ([]string, error) { func (p *xlStorageDiskIDCheck) ListDir(ctx context.Context, volume, dirPath string, count int) ([]string, error) {
if err := p.checkDiskStale(); err != nil { if err := p.checkDiskStale(); err != nil {
return nil, err return nil, err
} }
return p.storage.ListDir(volume, dirPath, count) return p.storage.ListDir(ctx, volume, dirPath, count)
} }
func (p *xlStorageDiskIDCheck) ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) { func (p *xlStorageDiskIDCheck) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
if err := p.checkDiskStale(); err != nil { if err := p.checkDiskStale(); err != nil {
return 0, err return 0, err
} }
return p.storage.ReadFile(volume, path, offset, buf, verifier) return p.storage.ReadFile(ctx, volume, path, offset, buf, verifier)
} }
func (p *xlStorageDiskIDCheck) AppendFile(volume string, path string, buf []byte) (err error) { func (p *xlStorageDiskIDCheck) AppendFile(ctx context.Context, volume string, path string, buf []byte) (err error) {
if err = p.checkDiskStale(); err != nil { if err = p.checkDiskStale(); err != nil {
return err return err
} }
return p.storage.AppendFile(volume, path, buf) return p.storage.AppendFile(ctx, volume, path, buf)
} }
func (p *xlStorageDiskIDCheck) CreateFile(volume, path string, size int64, reader io.Reader) error { func (p *xlStorageDiskIDCheck) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error {
if err := p.checkDiskStale(); err != nil { if err := p.checkDiskStale(); err != nil {
return err return err
} }
return p.storage.CreateFile(volume, path, size, reader) return p.storage.CreateFile(ctx, volume, path, size, reader)
} }
func (p *xlStorageDiskIDCheck) ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error) { func (p *xlStorageDiskIDCheck) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) {
if err := p.checkDiskStale(); err != nil { if err := p.checkDiskStale(); err != nil {
return nil, err return nil, err
} }
return p.storage.ReadFileStream(volume, path, offset, length) return p.storage.ReadFileStream(ctx, volume, path, offset, length)
} }
func (p *xlStorageDiskIDCheck) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error { func (p *xlStorageDiskIDCheck) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error {
if err := p.checkDiskStale(); err != nil { if err := p.checkDiskStale(); err != nil {
return err return err
} }
return p.storage.RenameFile(srcVolume, srcPath, dstVolume, dstPath) return p.storage.RenameFile(ctx, srcVolume, srcPath, dstVolume, dstPath)
} }
func (p *xlStorageDiskIDCheck) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath string) error { func (p *xlStorageDiskIDCheck) RenameData(ctx context.Context, srcVolume, srcPath, dataDir, dstVolume, dstPath string) error {
if err := p.checkDiskStale(); err != nil { if err := p.checkDiskStale(); err != nil {
return err return err
} }
return p.storage.RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath) return p.storage.RenameData(ctx, srcVolume, srcPath, dataDir, dstVolume, dstPath)
} }
func (p *xlStorageDiskIDCheck) CheckParts(volume string, path string, fi FileInfo) (err error) { func (p *xlStorageDiskIDCheck) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (err error) {
if err = p.checkDiskStale(); err != nil { if err = p.checkDiskStale(); err != nil {
return err return err
} }
return p.storage.CheckParts(volume, path, fi) return p.storage.CheckParts(ctx, volume, path, fi)
} }
func (p *xlStorageDiskIDCheck) CheckFile(volume string, path string) (err error) { func (p *xlStorageDiskIDCheck) CheckFile(ctx context.Context, volume string, path string) (err error) {
if err = p.checkDiskStale(); err != nil { if err = p.checkDiskStale(); err != nil {
return err return err
} }
return p.storage.CheckFile(volume, path) return p.storage.CheckFile(ctx, volume, path)
} }
func (p *xlStorageDiskIDCheck) DeleteFile(volume string, path string) (err error) { func (p *xlStorageDiskIDCheck) DeleteFile(ctx context.Context, volume string, path string) (err error) {
if err = p.checkDiskStale(); err != nil { if err = p.checkDiskStale(); err != nil {
return err return err
} }
return p.storage.DeleteFile(volume, path) return p.storage.DeleteFile(ctx, volume, path)
} }
func (p *xlStorageDiskIDCheck) DeleteVersions(volume string, versions []FileInfo) (errs []error) { func (p *xlStorageDiskIDCheck) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) (errs []error) {
if err := p.checkDiskStale(); err != nil { if err := p.checkDiskStale(); err != nil {
errs = make([]error, len(versions)) errs = make([]error, len(versions))
for i := range errs { for i := range errs {
@ -245,53 +245,53 @@ func (p *xlStorageDiskIDCheck) DeleteVersions(volume string, versions []FileInfo
} }
return errs return errs
} }
return p.storage.DeleteVersions(volume, versions) return p.storage.DeleteVersions(ctx, volume, versions)
} }
func (p *xlStorageDiskIDCheck) VerifyFile(volume, path string, fi FileInfo) error { func (p *xlStorageDiskIDCheck) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error {
if err := p.checkDiskStale(); err != nil { if err := p.checkDiskStale(); err != nil {
return err return err
} }
return p.storage.VerifyFile(volume, path, fi) return p.storage.VerifyFile(ctx, volume, path, fi)
} }
func (p *xlStorageDiskIDCheck) WriteAll(volume string, path string, reader io.Reader) (err error) { func (p *xlStorageDiskIDCheck) WriteAll(ctx context.Context, volume string, path string, reader io.Reader) (err error) {
if err = p.checkDiskStale(); err != nil { if err = p.checkDiskStale(); err != nil {
return err return err
} }
return p.storage.WriteAll(volume, path, reader) return p.storage.WriteAll(ctx, volume, path, reader)
} }
func (p *xlStorageDiskIDCheck) DeleteVersion(volume, path string, fi FileInfo) (err error) { func (p *xlStorageDiskIDCheck) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo) (err error) {
if err = p.checkDiskStale(); err != nil { if err = p.checkDiskStale(); err != nil {
return err return err
} }
return p.storage.DeleteVersion(volume, path, fi) return p.storage.DeleteVersion(ctx, volume, path, fi)
} }
func (p *xlStorageDiskIDCheck) WriteMetadata(volume, path string, fi FileInfo) (err error) { func (p *xlStorageDiskIDCheck) WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) (err error) {
if err = p.checkDiskStale(); err != nil { if err = p.checkDiskStale(); err != nil {
return err return err
} }
return p.storage.WriteMetadata(volume, path, fi) return p.storage.WriteMetadata(ctx, volume, path, fi)
} }
func (p *xlStorageDiskIDCheck) ReadVersion(volume, path, versionID string) (fi FileInfo, err error) { func (p *xlStorageDiskIDCheck) ReadVersion(ctx context.Context, volume, path, versionID string) (fi FileInfo, err error) {
if err = p.checkDiskStale(); err != nil { if err = p.checkDiskStale(); err != nil {
return fi, err return fi, err
} }
return p.storage.ReadVersion(volume, path, versionID) return p.storage.ReadVersion(ctx, volume, path, versionID)
} }
func (p *xlStorageDiskIDCheck) ReadAll(volume string, path string) (buf []byte, err error) { func (p *xlStorageDiskIDCheck) ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) {
if err = p.checkDiskStale(); err != nil { if err = p.checkDiskStale(); err != nil {
return nil, err return nil, err
} }
return p.storage.ReadAll(volume, path) return p.storage.ReadAll(ctx, volume, path)
} }

76
cmd/xl-storage.go
View File

@ -417,7 +417,7 @@ type DiskInfo struct {
// DiskInfo provides current information about disk space usage, // DiskInfo provides current information about disk space usage,
// total free inodes and underlying filesystem. // total free inodes and underlying filesystem.
func (s *xlStorage) DiskInfo() (info DiskInfo, err error) { func (s *xlStorage) DiskInfo(context.Context) (info DiskInfo, err error) {
atomic.AddInt32(&s.activeIOCount, 1) atomic.AddInt32(&s.activeIOCount, 1)
defer func() { defer func() {
atomic.AddInt32(&s.activeIOCount, -1) atomic.AddInt32(&s.activeIOCount, -1)
@ -548,9 +548,9 @@ func (s *xlStorage) SetDiskID(id string) {
// storage rest server for remote disks. // storage rest server for remote disks.
} }
func (s *xlStorage) MakeVolBulk(volumes ...string) (err error) { func (s *xlStorage) MakeVolBulk(ctx context.Context, volumes ...string) (err error) {
for _, volume := range volumes { for _, volume := range volumes {
if err = s.MakeVol(volume); err != nil { if err = s.MakeVol(ctx, volume); err != nil {
if os.IsPermission(err) { if os.IsPermission(err) {
return errVolumeAccessDenied return errVolumeAccessDenied
} }
@ -560,7 +560,7 @@ func (s *xlStorage) MakeVolBulk(volumes ...string) (err error) {
} }
// Make a volume entry. // Make a volume entry.
func (s *xlStorage) MakeVol(volume string) (err error) { func (s *xlStorage) MakeVol(ctx context.Context, volume string) (err error) {
if !isValidVolname(volume) { if !isValidVolname(volume) {
return errInvalidArgument return errInvalidArgument
} }
@ -594,7 +594,7 @@ func (s *xlStorage) MakeVol(volume string) (err error) {
} }
// ListVols - list volumes. // ListVols - list volumes.
func (s *xlStorage) ListVols() (volsInfo []VolInfo, err error) { func (s *xlStorage) ListVols(context.Context) (volsInfo []VolInfo, err error) {
atomic.AddInt32(&s.activeIOCount, 1) atomic.AddInt32(&s.activeIOCount, 1)
defer func() { defer func() {
atomic.AddInt32(&s.activeIOCount, -1) atomic.AddInt32(&s.activeIOCount, -1)
@ -654,7 +654,7 @@ func listVols(dirPath string) ([]VolInfo, error) {
} }
// StatVol - get volume info. // StatVol - get volume info.
func (s *xlStorage) StatVol(volume string) (volInfo VolInfo, err error) { func (s *xlStorage) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) {
atomic.AddInt32(&s.activeIOCount, 1) atomic.AddInt32(&s.activeIOCount, 1)
defer func() { defer func() {
atomic.AddInt32(&s.activeIOCount, -1) atomic.AddInt32(&s.activeIOCount, -1)
@ -686,7 +686,7 @@ func (s *xlStorage) StatVol(volume string) (volInfo VolInfo, err error) {
} }
// DeleteVol - delete a volume. // DeleteVol - delete a volume.
func (s *xlStorage) DeleteVol(volume string, forceDelete bool) (err error) { func (s *xlStorage) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) {
atomic.AddInt32(&s.activeIOCount, 1) atomic.AddInt32(&s.activeIOCount, 1)
defer func() { defer func() {
atomic.AddInt32(&s.activeIOCount, -1) atomic.AddInt32(&s.activeIOCount, -1)
@ -806,7 +806,7 @@ func (s *xlStorage) isLeafDir(volume, leafPath string) bool {
// sorted order, additionally along with metadata about each of those entries. // sorted order, additionally along with metadata about each of those entries.
// Implemented specifically for Splunk backend structure and List call with // Implemented specifically for Splunk backend structure and List call with
// delimiter as "guidSplunk" // delimiter as "guidSplunk"
func (s *xlStorage) WalkSplunk(volume, dirPath, marker string, endWalkCh <-chan struct{}) (ch chan FileInfo, err error) { func (s *xlStorage) WalkSplunk(ctx context.Context, volume, dirPath, marker string, endWalkCh <-chan struct{}) (ch chan FileInfo, err error) {
// Verify if volume is valid and it exists. // Verify if volume is valid and it exists.
volumeDir, err := s.getVolDir(volume) volumeDir, err := s.getVolDir(volume)
if err != nil { if err != nil {
@ -877,7 +877,7 @@ func (s *xlStorage) WalkSplunk(volume, dirPath, marker string, endWalkCh <-chan
// WalkVersions - is a sorted walker which returns file entries in lexically sorted order, // WalkVersions - is a sorted walker which returns file entries in lexically sorted order,
// additionally along with metadata version info about each of those entries. // additionally along with metadata version info about each of those entries.
func (s *xlStorage) WalkVersions(volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (ch chan FileInfoVersions, err error) { func (s *xlStorage) WalkVersions(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (ch chan FileInfoVersions, err error) {
atomic.AddInt32(&s.activeIOCount, 1) atomic.AddInt32(&s.activeIOCount, 1)
defer func() { defer func() {
atomic.AddInt32(&s.activeIOCount, -1) atomic.AddInt32(&s.activeIOCount, -1)
@ -912,7 +912,7 @@ func (s *xlStorage) WalkVersions(volume, dirPath, marker string, recursive bool,
go func() { go func() {
defer close(ch) defer close(ch)
listDir := func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string, delayIsLeaf bool) { listDir := func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string, delayIsLeaf bool) {
entries, err := s.ListDir(volume, dirPath, -1) entries, err := s.ListDir(ctx, volume, dirPath, -1)
if err != nil { if err != nil {
return false, nil, false return false, nil, false
} }
@ -962,7 +962,7 @@ func (s *xlStorage) WalkVersions(volume, dirPath, marker string, recursive bool,
// Walk - is a sorted walker which returns file entries in lexically // Walk - is a sorted walker which returns file entries in lexically
// sorted order, additionally along with metadata about each of those entries. // sorted order, additionally along with metadata about each of those entries.
func (s *xlStorage) Walk(volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (ch chan FileInfo, err error) { func (s *xlStorage) Walk(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (ch chan FileInfo, err error) {
atomic.AddInt32(&s.activeIOCount, 1) atomic.AddInt32(&s.activeIOCount, 1)
defer func() { defer func() {
atomic.AddInt32(&s.activeIOCount, -1) atomic.AddInt32(&s.activeIOCount, -1)
@ -997,7 +997,7 @@ func (s *xlStorage) Walk(volume, dirPath, marker string, recursive bool, endWalk
go func() { go func() {
defer close(ch) defer close(ch)
listDir := func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string, delayIsLeaf bool) { listDir := func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string, delayIsLeaf bool) {
entries, err := s.ListDir(volume, dirPath, -1) entries, err := s.ListDir(ctx, volume, dirPath, -1)
if err != nil { if err != nil {
return false, nil, false return false, nil, false
} }
@ -1046,7 +1046,7 @@ func (s *xlStorage) Walk(volume, dirPath, marker string, recursive bool, endWalk
// ListDir - return all the entries at the given directory path. // ListDir - return all the entries at the given directory path.
// If an entry is a directory it will be returned with a trailing SlashSeparator. // If an entry is a directory it will be returned with a trailing SlashSeparator.
func (s *xlStorage) ListDir(volume, dirPath string, count int) (entries []string, err error) { func (s *xlStorage) ListDir(ctx context.Context, volume, dirPath string, count int) (entries []string, err error) {
atomic.AddInt32(&s.activeIOCount, 1) atomic.AddInt32(&s.activeIOCount, 1)
defer func() { defer func() {
atomic.AddInt32(&s.activeIOCount, -1) atomic.AddInt32(&s.activeIOCount, -1)
@ -1082,10 +1082,10 @@ func (s *xlStorage) ListDir(volume, dirPath string, count int) (entries []string
// DeleteVersions deletes slice of versions, it can be same object // DeleteVersions deletes slice of versions, it can be same object
// or multiple objects. // or multiple objects.
func (s *xlStorage) DeleteVersions(volume string, versions []FileInfo) []error { func (s *xlStorage) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) []error {
errs := make([]error, len(versions)) errs := make([]error, len(versions))
for i, version := range versions { for i, version := range versions {
if err := s.DeleteVersion(volume, version.Name, version); err != nil { if err := s.DeleteVersion(ctx, volume, version.Name, version); err != nil {
errs[i] = err errs[i] = err
} }
} }
@ -1094,12 +1094,12 @@ func (s *xlStorage) DeleteVersions(volume string, versions []FileInfo) []error {
} }
// DeleteVersion - deletes FileInfo metadata for path at `xl.meta` // DeleteVersion - deletes FileInfo metadata for path at `xl.meta`
func (s *xlStorage) DeleteVersion(volume, path string, fi FileInfo) error { func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo) error {
if HasSuffix(path, SlashSeparator) { if HasSuffix(path, SlashSeparator) {
return s.DeleteFile(volume, path) return s.DeleteFile(ctx, volume, path)
} }
buf, err := s.ReadAll(volume, pathJoin(path, xlStorageFormatFile)) buf, err := s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile))
if err != nil { if err != nil {
return err return err
} }
@ -1152,7 +1152,7 @@ func (s *xlStorage) DeleteVersion(volume, path string, fi FileInfo) error {
} }
if !lastVersion { if !lastVersion {
return s.WriteAll(volume, pathJoin(path, xlStorageFormatFile), bytes.NewReader(buf)) return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), bytes.NewReader(buf))
} }
// Delete the meta file, if there are no more versions the // Delete the meta file, if there are no more versions the
@ -1166,8 +1166,8 @@ func (s *xlStorage) DeleteVersion(volume, path string, fi FileInfo) error {
} }
// WriteMetadata - writes FileInfo metadata for path at `xl.meta` // WriteMetadata - writes FileInfo metadata for path at `xl.meta`
func (s *xlStorage) WriteMetadata(volume, path string, fi FileInfo) error { func (s *xlStorage) WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) error {
buf, err := s.ReadAll(volume, pathJoin(path, xlStorageFormatFile)) buf, err := s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile))
if err != nil && err != errFileNotFound { if err != nil && err != errFileNotFound {
return err return err
} }
@ -1200,7 +1200,7 @@ func (s *xlStorage) WriteMetadata(volume, path string, fi FileInfo) error {
} }
} }
return s.WriteAll(volume, pathJoin(path, xlStorageFormatFile), bytes.NewReader(buf)) return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), bytes.NewReader(buf))
} }
func (s *xlStorage) renameLegacyMetadata(volume, path string) error { func (s *xlStorage) renameLegacyMetadata(volume, path string) error {
@ -1268,14 +1268,14 @@ func (s *xlStorage) renameLegacyMetadata(volume, path string) error {
} }
// ReadVersion - reads metadata and returns FileInfo at path `xl.meta` // ReadVersion - reads metadata and returns FileInfo at path `xl.meta`
func (s *xlStorage) ReadVersion(volume, path, versionID string) (fi FileInfo, err error) { func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID string) (fi FileInfo, err error) {
buf, err := s.ReadAll(volume, pathJoin(path, xlStorageFormatFile)) buf, err := s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile))
if err != nil { if err != nil {
if err == errFileNotFound { if err == errFileNotFound {
if err = s.renameLegacyMetadata(volume, path); err != nil { if err = s.renameLegacyMetadata(volume, path); err != nil {
return fi, err return fi, err
} }
buf, err = s.ReadAll(volume, pathJoin(path, xlStorageFormatFile)) buf, err = s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile))
if err != nil { if err != nil {
return fi, err return fi, err
} }
@ -1300,7 +1300,7 @@ func (s *xlStorage) ReadVersion(volume, path, versionID string) (fi FileInfo, er
// as an error to be reported. // as an error to be reported.
// This API is meant to be used on files which have small memory footprint, do // This API is meant to be used on files which have small memory footprint, do
// not use this on large files as it would cause server to crash. // not use this on large files as it would cause server to crash.
func (s *xlStorage) ReadAll(volume, path string) (buf []byte, err error) { func (s *xlStorage) ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) {
atomic.AddInt32(&s.activeIOCount, 1) atomic.AddInt32(&s.activeIOCount, 1)
defer func() { defer func() {
atomic.AddInt32(&s.activeIOCount, -1) atomic.AddInt32(&s.activeIOCount, -1)
@ -1363,7 +1363,7 @@ func (s *xlStorage) ReadAll(volume, path string) (buf []byte, err error) {
// //
// Additionally ReadFile also starts reading from an offset. ReadFile // Additionally ReadFile also starts reading from an offset. ReadFile
// semantics are same as io.ReadFull. // semantics are same as io.ReadFull.
func (s *xlStorage) ReadFile(volume, path string, offset int64, buffer []byte, verifier *BitrotVerifier) (int64, error) { func (s *xlStorage) ReadFile(ctx context.Context, volume string, path string, offset int64, buffer []byte, verifier *BitrotVerifier) (int64, error) {
if offset < 0 { if offset < 0 {
return 0, errInvalidArgument return 0, errInvalidArgument
} }
@ -1518,7 +1518,7 @@ func (s *xlStorage) openFile(volume, path string, mode int) (f *os.File, err err
} }
// ReadFileStream - Returns the read stream of the file. // ReadFileStream - Returns the read stream of the file.
func (s *xlStorage) ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error) { func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) {
if offset < 0 { if offset < 0 {
return nil, errInvalidArgument return nil, errInvalidArgument
} }
@ -1611,7 +1611,7 @@ func (c closeWrapper) Close() error {
} }
// CreateFile - creates the file. // CreateFile - creates the file.
func (s *xlStorage) CreateFile(volume, path string, fileSize int64, r io.Reader) (err error) { func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSize int64, r io.Reader) (err error) {
if fileSize < -1 { if fileSize < -1 {
return errInvalidArgument return errInvalidArgument
} }
@ -1720,7 +1720,7 @@ func (s *xlStorage) CreateFile(volume, path string, fileSize int64, r io.Reader)
return nil return nil
} }
func (s *xlStorage) WriteAll(volume, path string, reader io.Reader) (err error) { func (s *xlStorage) WriteAll(ctx context.Context, volume string, path string, reader io.Reader) (err error) {
atomic.AddInt32(&s.activeIOCount, 1) atomic.AddInt32(&s.activeIOCount, 1)
defer func() { defer func() {
atomic.AddInt32(&s.activeIOCount, -1) atomic.AddInt32(&s.activeIOCount, -1)
@ -1742,7 +1742,7 @@ func (s *xlStorage) WriteAll(volume, path string, reader io.Reader) (err error)
// AppendFile - append a byte array at path, if file doesn't exist at // AppendFile - append a byte array at path, if file doesn't exist at
// path this call explicitly creates it. // path this call explicitly creates it.
func (s *xlStorage) AppendFile(volume, path string, buf []byte) (err error) { func (s *xlStorage) AppendFile(ctx context.Context, volume string, path string, buf []byte) (err error) {
atomic.AddInt32(&s.activeIOCount, 1) atomic.AddInt32(&s.activeIOCount, 1)
defer func() { defer func() {
atomic.AddInt32(&s.activeIOCount, -1) atomic.AddInt32(&s.activeIOCount, -1)
@ -1764,7 +1764,7 @@ func (s *xlStorage) AppendFile(volume, path string, buf []byte) (err error) {
} }
// CheckParts check if path has necessary parts available. // CheckParts check if path has necessary parts available.
func (s *xlStorage) CheckParts(volume, path string, fi FileInfo) error { func (s *xlStorage) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error {
atomic.AddInt32(&s.activeIOCount, 1) atomic.AddInt32(&s.activeIOCount, 1)
defer func() { defer func() {
atomic.AddInt32(&s.activeIOCount, -1) atomic.AddInt32(&s.activeIOCount, -1)
@ -1809,7 +1809,7 @@ func (s *xlStorage) CheckParts(volume, path string, fi FileInfo) error {
} }
// CheckFile check if path has necessary metadata. // CheckFile check if path has necessary metadata.
func (s *xlStorage) CheckFile(volume, path string) error { func (s *xlStorage) CheckFile(ctx context.Context, volume string, path string) error {
atomic.AddInt32(&s.activeIOCount, 1) atomic.AddInt32(&s.activeIOCount, 1)
defer func() { defer func() {
atomic.AddInt32(&s.activeIOCount, -1) atomic.AddInt32(&s.activeIOCount, -1)
@ -1913,7 +1913,7 @@ func deleteFile(basePath, deletePath string, recursive bool) error {
} }
// DeleteFile - delete a file at path. // DeleteFile - delete a file at path.
func (s *xlStorage) DeleteFile(volume, path string) (err error) { func (s *xlStorage) DeleteFile(ctx context.Context, volume string, path string) (err error) {
atomic.AddInt32(&s.activeIOCount, 1) atomic.AddInt32(&s.activeIOCount, 1)
defer func() { defer func() {
atomic.AddInt32(&s.activeIOCount, -1) atomic.AddInt32(&s.activeIOCount, -1)
@ -1988,7 +1988,7 @@ func (s *xlStorage) DeleteFileBulk(volume string, paths []string) (errs []error,
} }
// RenameData - rename source path to destination path atomically, metadata and data directory. // RenameData - rename source path to destination path atomically, metadata and data directory.
func (s *xlStorage) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath string) (err error) { func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath, dataDir, dstVolume, dstPath string) (err error) {
atomic.AddInt32(&s.activeIOCount, 1) atomic.AddInt32(&s.activeIOCount, 1)
defer func() { defer func() {
atomic.AddInt32(&s.activeIOCount, -1) atomic.AddInt32(&s.activeIOCount, -1)
@ -2170,7 +2170,7 @@ func (s *xlStorage) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath s
return errFileCorrupt return errFileCorrupt
} }
if err = s.WriteAll(srcVolume, pathJoin(srcPath, xlStorageFormatFile), bytes.NewReader(dstBuf)); err != nil { if err = s.WriteAll(ctx, srcVolume, pathJoin(srcPath, xlStorageFormatFile), bytes.NewReader(dstBuf)); err != nil {
return err return err
} }
@ -2201,7 +2201,7 @@ func (s *xlStorage) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath s
} }
// RenameFile - rename source path to destination path atomically. // RenameFile - rename source path to destination path atomically.
func (s *xlStorage) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err error) { func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) (err error) {
atomic.AddInt32(&s.activeIOCount, 1) atomic.AddInt32(&s.activeIOCount, 1)
defer func() { defer func() {
atomic.AddInt32(&s.activeIOCount, -1) atomic.AddInt32(&s.activeIOCount, -1)
@ -2357,7 +2357,7 @@ func (s *xlStorage) bitrotVerify(partPath string, partSize int64, algo BitrotAlg
} }
} }
func (s *xlStorage) VerifyFile(volume, path string, fi FileInfo) (err error) { func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (err error) {
atomic.AddInt32(&s.activeIOCount, 1) atomic.AddInt32(&s.activeIOCount, 1)
defer func() { defer func() {
atomic.AddInt32(&s.activeIOCount, -1) atomic.AddInt32(&s.activeIOCount, -1)

129
cmd/xl-storage_test.go
View File

@ -18,6 +18,7 @@ package cmd
import ( import (
"bytes" "bytes"
"context"
"crypto/rand" "crypto/rand"
"fmt" "fmt"
"io" "io"
@ -126,12 +127,12 @@ func newXLStorageTestSetup() (*xlStorageDiskIDCheck, string, error) {
if err != nil { if err != nil {
return nil, "", err return nil, "", err
} }
err = storage.MakeVol(minioMetaBucket) err = storage.MakeVol(context.Background(), minioMetaBucket)
if err != nil { if err != nil {
return nil, "", err return nil, "", err
} }
// Create a sample format.json file // Create a sample format.json file
err = storage.WriteAll(minioMetaBucket, formatConfigFile, bytes.NewBufferString(`{"version":"1","format":"xl","id":"592a41c2-b7cc-4130-b883-c4b5cb15965b","xl":{"version":"3","this":"da017d62-70e3-45f1-8a1a-587707e69ad1","sets":[["e07285a6-8c73-4962-89c6-047fb939f803","33b8d431-482d-4376-b63c-626d229f0a29","cff6513a-4439-4dc1-bcaa-56c9e880c352","da017d62-70e3-45f1-8a1a-587707e69ad1","9c9f21d5-1f15-4737-bce6-835faa0d9626","0a59b346-1424-4fc2-9fa2-a2e80541d0c1","7924a3dc-b69a-4971-9a2e-014966d6aebb","4d2b8dd9-4e48-444b-bdca-c89194b26042"]],"distributionAlgo":"CRCMOD"}}`)) err = storage.WriteAll(context.Background(), minioMetaBucket, formatConfigFile, bytes.NewBufferString(`{"version":"1","format":"xl","id":"592a41c2-b7cc-4130-b883-c4b5cb15965b","xl":{"version":"3","this":"da017d62-70e3-45f1-8a1a-587707e69ad1","sets":[["e07285a6-8c73-4962-89c6-047fb939f803","33b8d431-482d-4376-b63c-626d229f0a29","cff6513a-4439-4dc1-bcaa-56c9e880c352","da017d62-70e3-45f1-8a1a-587707e69ad1","9c9f21d5-1f15-4737-bce6-835faa0d9626","0a59b346-1424-4fc2-9fa2-a2e80541d0c1","7924a3dc-b69a-4971-9a2e-014966d6aebb","4d2b8dd9-4e48-444b-bdca-c89194b26042"]],"distributionAlgo":"CRCMOD"}}`))
if err != nil { if err != nil {
return nil, "", err return nil, "", err
} }
@ -262,16 +263,16 @@ func TestXLStorageReadAll(t *testing.T) {
defer os.RemoveAll(path) defer os.RemoveAll(path)
// Create files for the test cases. // Create files for the test cases.
if err = xlStorage.MakeVol("exists"); err != nil { if err = xlStorage.MakeVol(context.Background(), "exists"); err != nil {
t.Fatalf("Unable to create a volume \"exists\", %s", err) t.Fatalf("Unable to create a volume \"exists\", %s", err)
} }
if err = xlStorage.AppendFile("exists", "as-directory/as-file", []byte("Hello, World")); err != nil { if err = xlStorage.AppendFile(context.Background(), "exists", "as-directory/as-file", []byte("Hello, World")); err != nil {
t.Fatalf("Unable to create a file \"as-directory/as-file\", %s", err) t.Fatalf("Unable to create a file \"as-directory/as-file\", %s", err)
} }
if err = xlStorage.AppendFile("exists", "as-file", []byte("Hello, World")); err != nil { if err = xlStorage.AppendFile(context.Background(), "exists", "as-file", []byte("Hello, World")); err != nil {
t.Fatalf("Unable to create a file \"as-file\", %s", err) t.Fatalf("Unable to create a file \"as-file\", %s", err)
} }
if err = xlStorage.AppendFile("exists", "as-file-parent", []byte("Hello, World")); err != nil { if err = xlStorage.AppendFile(context.Background(), "exists", "as-file-parent", []byte("Hello, World")); err != nil {
t.Fatalf("Unable to create a file \"as-file-parent\", %s", err) t.Fatalf("Unable to create a file \"as-file-parent\", %s", err)
} }
@ -328,7 +329,7 @@ func TestXLStorageReadAll(t *testing.T) {
var dataRead []byte var dataRead []byte
// Run through all the test cases and validate for ReadAll. // Run through all the test cases and validate for ReadAll.
for i, testCase := range testCases { for i, testCase := range testCases {
dataRead, err = xlStorage.ReadAll(testCase.volume, testCase.path) dataRead, err = xlStorage.ReadAll(context.Background(), testCase.volume, testCase.path)
if err != testCase.err { if err != testCase.err {
t.Fatalf("TestXLStorage %d: Expected err \"%s\", got err \"%s\"", i+1, testCase.err, err) t.Fatalf("TestXLStorage %d: Expected err \"%s\", got err \"%s\"", i+1, testCase.err, err)
} }
@ -434,7 +435,7 @@ func TestXLStorageMakeVol(t *testing.T) {
} }
for i, testCase := range testCases { for i, testCase := range testCases {
if err := xlStorage.MakeVol(testCase.volName); err != testCase.expectedErr { if err := xlStorage.MakeVol(context.Background(), testCase.volName); err != testCase.expectedErr {
t.Fatalf("TestXLStorage %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) t.Fatalf("TestXLStorage %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
} }
} }
@ -470,7 +471,7 @@ func TestXLStorageMakeVol(t *testing.T) {
t.Fatalf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err) t.Fatalf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err)
} }
if err := xlStorageNew.MakeVol("test-vol"); err != errDiskAccessDenied { if err := xlStorageNew.MakeVol(context.Background(), "test-vol"); err != errDiskAccessDenied {
t.Fatalf("expected: %s, got: %s", errDiskAccessDenied, err) t.Fatalf("expected: %s, got: %s", errDiskAccessDenied, err)
} }
} }
@ -486,7 +487,7 @@ func TestXLStorageDeleteVol(t *testing.T) {
defer os.RemoveAll(path) defer os.RemoveAll(path)
// Setup test environment. // Setup test environment.
if err = xlStorage.MakeVol("success-vol"); err != nil { if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil {
t.Fatalf("Unable to create volume, %s", err) t.Fatalf("Unable to create volume, %s", err)
} }
@ -530,7 +531,7 @@ func TestXLStorageDeleteVol(t *testing.T) {
} }
for i, testCase := range testCases { for i, testCase := range testCases {
if err = xlStorage.DeleteVol(testCase.volName, false); err != testCase.expectedErr { if err = xlStorage.DeleteVol(context.Background(), testCase.volName, false); err != testCase.expectedErr {
t.Fatalf("TestXLStorage: %d, expected: %s, got: %s", i+1, testCase.expectedErr, err) t.Fatalf("TestXLStorage: %d, expected: %s, got: %s", i+1, testCase.expectedErr, err)
} }
} }
@ -569,7 +570,7 @@ func TestXLStorageDeleteVol(t *testing.T) {
t.Fatalf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err) t.Fatalf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err)
} }
if err = xlStorageNew.DeleteVol("mybucket", false); err != errDiskAccessDenied { if err = xlStorageNew.DeleteVol(context.Background(), "mybucket", false); err != errDiskAccessDenied {
t.Fatalf("expected: Permission error, got: %s", err) t.Fatalf("expected: Permission error, got: %s", err)
} }
} }
@ -583,7 +584,7 @@ func TestXLStorageDeleteVol(t *testing.T) {
// TestXLStorage for delete on an removed disk. // TestXLStorage for delete on an removed disk.
// should fail with disk not found. // should fail with disk not found.
err = xlStorageDeletedStorage.DeleteVol("Del-Vol", false) err = xlStorageDeletedStorage.DeleteVol(context.Background(), "Del-Vol", false)
if err != errDiskNotFound { if err != errDiskNotFound {
t.Errorf("Expected: \"Disk not found\", got \"%s\"", err) t.Errorf("Expected: \"Disk not found\", got \"%s\"", err)
} }
@ -599,7 +600,7 @@ func TestXLStorageStatVol(t *testing.T) {
defer os.RemoveAll(path) defer os.RemoveAll(path)
// Setup test environment. // Setup test environment.
if err = xlStorage.MakeVol("success-vol"); err != nil { if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil {
t.Fatalf("Unable to create volume, %s", err) t.Fatalf("Unable to create volume, %s", err)
} }
@ -626,7 +627,7 @@ func TestXLStorageStatVol(t *testing.T) {
for i, testCase := range testCases { for i, testCase := range testCases {
var volInfo VolInfo var volInfo VolInfo
volInfo, err = xlStorage.StatVol(testCase.volName) volInfo, err = xlStorage.StatVol(context.Background(), testCase.volName)
if err != testCase.expectedErr { if err != testCase.expectedErr {
t.Fatalf("TestXLStorage case : %d, Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) t.Fatalf("TestXLStorage case : %d, Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
} }
@ -648,7 +649,7 @@ func TestXLStorageStatVol(t *testing.T) {
// TestXLStorage for delete on an removed disk. // TestXLStorage for delete on an removed disk.
// should fail with disk not found. // should fail with disk not found.
_, err = xlStorageDeletedStorage.StatVol("Stat vol") _, err = xlStorageDeletedStorage.StatVol(context.Background(), "Stat vol")
if err != errDiskNotFound { if err != errDiskNotFound {
t.Errorf("Expected: \"Disk not found\", got \"%s\"", err) t.Errorf("Expected: \"Disk not found\", got \"%s\"", err)
} }
@ -664,18 +665,18 @@ func TestXLStorageListVols(t *testing.T) {
var volInfos []VolInfo var volInfos []VolInfo
// TestXLStorage empty list vols. // TestXLStorage empty list vols.
if volInfos, err = xlStorage.ListVols(); err != nil { if volInfos, err = xlStorage.ListVols(context.Background()); err != nil {
t.Fatalf("expected: <nil>, got: %s", err) t.Fatalf("expected: <nil>, got: %s", err)
} else if len(volInfos) != 1 { } else if len(volInfos) != 1 {
t.Fatalf("expected: one entry, got: %s", volInfos) t.Fatalf("expected: one entry, got: %s", volInfos)
} }
// TestXLStorage non-empty list vols. // TestXLStorage non-empty list vols.
if err = xlStorage.MakeVol("success-vol"); err != nil { if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil {
t.Fatalf("Unable to create volume, %s", err) t.Fatalf("Unable to create volume, %s", err)
} }
volInfos, err = xlStorage.ListVols() volInfos, err = xlStorage.ListVols(context.Background())
if err != nil { if err != nil {
t.Fatalf("expected: <nil>, got: %s", err) t.Fatalf("expected: <nil>, got: %s", err)
} }
@ -696,7 +697,7 @@ func TestXLStorageListVols(t *testing.T) {
// removing the path and simulating disk failure // removing the path and simulating disk failure
os.RemoveAll(path) os.RemoveAll(path)
// should fail with errDiskNotFound. // should fail with errDiskNotFound.
if _, err = xlStorage.ListVols(); err != errDiskNotFound { if _, err = xlStorage.ListVols(context.Background()); err != errDiskNotFound {
t.Errorf("Expected to fail with \"%s\", but instead failed with \"%s\"", errDiskNotFound, err) t.Errorf("Expected to fail with \"%s\", but instead failed with \"%s\"", errDiskNotFound, err)
} }
} }
@ -718,13 +719,13 @@ func TestXLStorageXlStorageListDir(t *testing.T) {
// removing the disk, used to recreate disk not found error. // removing the disk, used to recreate disk not found error.
os.RemoveAll(diskPath) os.RemoveAll(diskPath)
// Setup test environment. // Setup test environment.
if err = xlStorage.MakeVol("success-vol"); err != nil { if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil {
t.Fatalf("Unable to create volume, %s", err) t.Fatalf("Unable to create volume, %s", err)
} }
if err = xlStorage.AppendFile("success-vol", "abc/def/ghi/success-file", []byte("Hello, world")); err != nil { if err = xlStorage.AppendFile(context.Background(), "success-vol", "abc/def/ghi/success-file", []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err) t.Fatalf("Unable to create file, %s", err)
} }
if err = xlStorage.AppendFile("success-vol", "abc/xyz/ghi/success-file", []byte("Hello, world")); err != nil { if err = xlStorage.AppendFile(context.Background(), "success-vol", "abc/xyz/ghi/success-file", []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err) t.Fatalf("Unable to create file, %s", err)
} }
@ -783,7 +784,7 @@ func TestXLStorageXlStorageListDir(t *testing.T) {
for i, testCase := range testCases { for i, testCase := range testCases {
var dirList []string var dirList []string
dirList, err = xlStorage.ListDir(testCase.srcVol, testCase.srcPath, -1) dirList, err = xlStorage.ListDir(context.Background(), testCase.srcVol, testCase.srcPath, -1)
if err != testCase.expectedErr { if err != testCase.expectedErr {
t.Fatalf("TestXLStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) t.Fatalf("TestXLStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
} }
@ -816,14 +817,14 @@ func TestXLStorageXlStorageListDir(t *testing.T) {
t.Fatalf("Unable to initialize xlStorage, %s", err) t.Fatalf("Unable to initialize xlStorage, %s", err)
} }
if err = xlStorageNew.DeleteFile("mybucket", "myobject"); err != errFileAccessDenied { if err = xlStorageNew.DeleteFile(context.Background(), "mybucket", "myobject"); err != errFileAccessDenied {
t.Errorf("expected: %s, got: %s", errFileAccessDenied, err) t.Errorf("expected: %s, got: %s", errFileAccessDenied, err)
} }
} }
// TestXLStorage for delete on an removed disk. // TestXLStorage for delete on an removed disk.
// should fail with disk not found. // should fail with disk not found.
err = xlStorageDeletedStorage.DeleteFile("del-vol", "my-file") err = xlStorageDeletedStorage.DeleteFile(context.Background(), "del-vol", "my-file")
if err != errDiskNotFound { if err != errDiskNotFound {
t.Errorf("Expected: \"Disk not found\", got \"%s\"", err) t.Errorf("Expected: \"Disk not found\", got \"%s\"", err)
} }
@ -846,17 +847,17 @@ func TestXLStorageDeleteFile(t *testing.T) {
// removing the disk, used to recreate disk not found error. // removing the disk, used to recreate disk not found error.
os.RemoveAll(diskPath) os.RemoveAll(diskPath)
// Setup test environment. // Setup test environment.
if err = xlStorage.MakeVol("success-vol"); err != nil { if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil {
t.Fatalf("Unable to create volume, %s", err) t.Fatalf("Unable to create volume, %s", err)
} }
if err = xlStorage.AppendFile("success-vol", "success-file", []byte("Hello, world")); err != nil { if err = xlStorage.AppendFile(context.Background(), "success-vol", "success-file", []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err) t.Fatalf("Unable to create file, %s", err)
} }
if err = xlStorage.MakeVol("no-permissions"); err != nil { if err = xlStorage.MakeVol(context.Background(), "no-permissions"); err != nil {
t.Fatalf("Unable to create volume, %s", err.Error()) t.Fatalf("Unable to create volume, %s", err.Error())
} }
if err = xlStorage.AppendFile("no-permissions", "dir/file", []byte("Hello, world")); err != nil { if err = xlStorage.AppendFile(context.Background(), "no-permissions", "dir/file", []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err.Error()) t.Fatalf("Unable to create file, %s", err.Error())
} }
// Parent directory must have write permissions, this is read + execute. // Parent directory must have write permissions, this is read + execute.
@ -915,7 +916,7 @@ func TestXLStorageDeleteFile(t *testing.T) {
} }
for i, testCase := range testCases { for i, testCase := range testCases {
if err = xlStorage.DeleteFile(testCase.srcVol, testCase.srcPath); err != testCase.expectedErr { if err = xlStorage.DeleteFile(context.Background(), testCase.srcVol, testCase.srcPath); err != testCase.expectedErr {
t.Errorf("TestXLStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) t.Errorf("TestXLStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
} }
} }
@ -940,14 +941,14 @@ func TestXLStorageDeleteFile(t *testing.T) {
t.Fatalf("Unable to initialize xlStorage, %s", err) t.Fatalf("Unable to initialize xlStorage, %s", err)
} }
if err = xlStorageNew.DeleteFile("mybucket", "myobject"); err != errFileAccessDenied { if err = xlStorageNew.DeleteFile(context.Background(), "mybucket", "myobject"); err != errFileAccessDenied {
t.Errorf("expected: %s, got: %s", errFileAccessDenied, err) t.Errorf("expected: %s, got: %s", errFileAccessDenied, err)
} }
} }
// TestXLStorage for delete on an removed disk. // TestXLStorage for delete on an removed disk.
// should fail with disk not found. // should fail with disk not found.
err = xlStorageDeletedStorage.DeleteFile("del-vol", "my-file") err = xlStorageDeletedStorage.DeleteFile(context.Background(), "del-vol", "my-file")
if err != errDiskNotFound { if err != errDiskNotFound {
t.Errorf("Expected: \"Disk not found\", got \"%s\"", err) t.Errorf("Expected: \"Disk not found\", got \"%s\"", err)
} }
@ -964,7 +965,7 @@ func TestXLStorageReadFile(t *testing.T) {
volume := "success-vol" volume := "success-vol"
// Setup test environment. // Setup test environment.
if err = xlStorage.MakeVol(volume); err != nil { if err = xlStorage.MakeVol(context.Background(), volume); err != nil {
t.Fatalf("Unable to create volume, %s", err) t.Fatalf("Unable to create volume, %s", err)
} }
@ -1048,7 +1049,7 @@ func TestXLStorageReadFile(t *testing.T) {
v := NewBitrotVerifier(SHA256, getSHA256Sum([]byte("hello, world"))) v := NewBitrotVerifier(SHA256, getSHA256Sum([]byte("hello, world")))
// Create test files for further reading. // Create test files for further reading.
for i, appendFile := range appendFiles { for i, appendFile := range appendFiles {
err = xlStorage.AppendFile(volume, appendFile.fileName, []byte("hello, world")) err = xlStorage.AppendFile(context.Background(), volume, appendFile.fileName, []byte("hello, world"))
if err != appendFile.expectedErr { if err != appendFile.expectedErr {
t.Fatalf("Creating file failed: %d %#v, expected: %s, got: %s", i+1, appendFile, appendFile.expectedErr, err) t.Fatalf("Creating file failed: %d %#v, expected: %s, got: %s", i+1, appendFile, appendFile.expectedErr, err)
} }
@ -1057,7 +1058,7 @@ func TestXLStorageReadFile(t *testing.T) {
{ {
buf := make([]byte, 5) buf := make([]byte, 5)
// Test for negative offset. // Test for negative offset.
if _, err = xlStorage.ReadFile(volume, "myobject", -1, buf, v); err == nil { if _, err = xlStorage.ReadFile(context.Background(), volume, "myobject", -1, buf, v); err == nil {
t.Fatalf("expected: error, got: <nil>") t.Fatalf("expected: error, got: <nil>")
} }
} }
@ -1067,7 +1068,7 @@ func TestXLStorageReadFile(t *testing.T) {
var n int64 var n int64
// Common read buffer. // Common read buffer.
var buf = make([]byte, testCase.bufSize) var buf = make([]byte, testCase.bufSize)
n, err = xlStorage.ReadFile(testCase.volume, testCase.fileName, testCase.offset, buf, v) n, err = xlStorage.ReadFile(context.Background(), testCase.volume, testCase.fileName, testCase.offset, buf, v)
if err != nil && testCase.expectedErr != nil { if err != nil && testCase.expectedErr != nil {
// Validate if the type string of the errors are an exact match. // Validate if the type string of the errors are an exact match.
if err.Error() != testCase.expectedErr.Error() { if err.Error() != testCase.expectedErr.Error() {
@ -1140,7 +1141,7 @@ func TestXLStorageReadFile(t *testing.T) {
// Common read buffer. // Common read buffer.
var buf = make([]byte, 10) var buf = make([]byte, 10)
if _, err = xlStoragePermStorage.ReadFile("mybucket", "myobject", 0, buf, v); err != errFileAccessDenied { if _, err = xlStoragePermStorage.ReadFile(context.Background(), "mybucket", "myobject", 0, buf, v); err != errFileAccessDenied {
t.Errorf("expected: %s, got: %s", errFileAccessDenied, err) t.Errorf("expected: %s, got: %s", errFileAccessDenied, err)
} }
} }
@ -1182,7 +1183,7 @@ func TestXLStorageReadFileWithVerify(t *testing.T) {
os.RemoveAll(path) os.RemoveAll(path)
t.Fatalf("Unable to create xlStorage test setup, %s", err) t.Fatalf("Unable to create xlStorage test setup, %s", err)
} }
if err = xlStorage.MakeVol(volume); err != nil { if err = xlStorage.MakeVol(context.Background(), volume); err != nil {
os.RemoveAll(path) os.RemoveAll(path)
t.Fatalf("Unable to create volume %s: %v", volume, err) t.Fatalf("Unable to create volume %s: %v", volume, err)
} }
@ -1191,7 +1192,7 @@ func TestXLStorageReadFileWithVerify(t *testing.T) {
os.RemoveAll(path) os.RemoveAll(path)
t.Fatalf("Unable to create generate random data: %v", err) t.Fatalf("Unable to create generate random data: %v", err)
} }
if err = xlStorage.AppendFile(volume, object, data); err != nil { if err = xlStorage.AppendFile(context.Background(), volume, object, data); err != nil {
os.RemoveAll(path) os.RemoveAll(path)
t.Fatalf("Unable to create object: %v", err) t.Fatalf("Unable to create object: %v", err)
} }
@ -1204,7 +1205,7 @@ func TestXLStorageReadFileWithVerify(t *testing.T) {
} }
buffer := make([]byte, test.length) buffer := make([]byte, test.length)
n, err := xlStorage.ReadFile(volume, test.file, int64(test.offset), buffer, NewBitrotVerifier(test.algorithm, h.Sum(nil))) n, err := xlStorage.ReadFile(context.Background(), volume, test.file, int64(test.offset), buffer, NewBitrotVerifier(test.algorithm, h.Sum(nil)))
switch { switch {
case err == nil && test.expError != nil: case err == nil && test.expError != nil:
@ -1227,7 +1228,7 @@ func TestXLStorageFormatFileChange(t *testing.T) {
} }
defer os.RemoveAll(path) defer os.RemoveAll(path)
if err = xlStorage.MakeVol(volume); err != nil { if err = xlStorage.MakeVol(context.Background(), volume); err != nil {
t.Fatalf("MakeVol failed with %s", err) t.Fatalf("MakeVol failed with %s", err)
} }
@ -1236,7 +1237,7 @@ func TestXLStorageFormatFileChange(t *testing.T) {
t.Fatalf("ioutil.WriteFile failed with %s", err) t.Fatalf("ioutil.WriteFile failed with %s", err)
} }
err = xlStorage.MakeVol(volume) err = xlStorage.MakeVol(context.Background(), volume)
if err != errVolumeExists { if err != errVolumeExists {
t.Fatalf("MakeVol expected to fail with errDiskNotFound but failed with %s", err) t.Fatalf("MakeVol expected to fail with errDiskNotFound but failed with %s", err)
} }
@ -1252,7 +1253,7 @@ func TestXLStorageAppendFile(t *testing.T) {
defer os.RemoveAll(path) defer os.RemoveAll(path)
// Setup test environment. // Setup test environment.
if err = xlStorage.MakeVol("success-vol"); err != nil { if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil {
t.Fatalf("Unable to create volume, %s", err) t.Fatalf("Unable to create volume, %s", err)
} }
@ -1282,7 +1283,7 @@ func TestXLStorageAppendFile(t *testing.T) {
} }
for i, testCase := range testCases { for i, testCase := range testCases {
if err = xlStorage.AppendFile("success-vol", testCase.fileName, []byte("hello, world")); err != testCase.expectedErr { if err = xlStorage.AppendFile(context.Background(), "success-vol", testCase.fileName, []byte("hello, world")); err != testCase.expectedErr {
t.Errorf("Case: %d, expected: %s, got: %s", i+1, testCase.expectedErr, err) t.Errorf("Case: %d, expected: %s, got: %s", i+1, testCase.expectedErr, err)
} }
} }
@ -1308,14 +1309,14 @@ func TestXLStorageAppendFile(t *testing.T) {
t.Fatalf("Unable to initialize xlStorage, %s", err) t.Fatalf("Unable to initialize xlStorage, %s", err)
} }
if err = xlStoragePermStorage.AppendFile("mybucket", "myobject", []byte("hello, world")); err != errFileAccessDenied { if err = xlStoragePermStorage.AppendFile(context.Background(), "mybucket", "myobject", []byte("hello, world")); err != errFileAccessDenied {
t.Fatalf("expected: Permission error, got: %s", err) t.Fatalf("expected: Permission error, got: %s", err)
} }
} }
// TestXLStorage case with invalid volume name. // TestXLStorage case with invalid volume name.
// A valid volume name should be atleast of size 3. // A valid volume name should be atleast of size 3.
err = xlStorage.AppendFile("bn", "yes", []byte("hello, world")) err = xlStorage.AppendFile(context.Background(), "bn", "yes", []byte("hello, world"))
if err != errVolumeNotFound { if err != errVolumeNotFound {
t.Fatalf("expected: \"Invalid argument error\", got: \"%s\"", err) t.Fatalf("expected: \"Invalid argument error\", got: \"%s\"", err)
} }
@ -1331,32 +1332,32 @@ func TestXLStorageRenameFile(t *testing.T) {
defer os.RemoveAll(path) defer os.RemoveAll(path)
// Setup test environment. // Setup test environment.
if err := xlStorage.MakeVol("src-vol"); err != nil { if err := xlStorage.MakeVol(context.Background(), "src-vol"); err != nil {
t.Fatalf("Unable to create volume, %s", err) t.Fatalf("Unable to create volume, %s", err)
} }
if err := xlStorage.MakeVol("dest-vol"); err != nil { if err := xlStorage.MakeVol(context.Background(), "dest-vol"); err != nil {
t.Fatalf("Unable to create volume, %s", err) t.Fatalf("Unable to create volume, %s", err)
} }
if err := xlStorage.AppendFile("src-vol", "file1", []byte("Hello, world")); err != nil { if err := xlStorage.AppendFile(context.Background(), "src-vol", "file1", []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err) t.Fatalf("Unable to create file, %s", err)
} }
if err := xlStorage.AppendFile("src-vol", "file2", []byte("Hello, world")); err != nil { if err := xlStorage.AppendFile(context.Background(), "src-vol", "file2", []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err) t.Fatalf("Unable to create file, %s", err)
} }
if err := xlStorage.AppendFile("src-vol", "file3", []byte("Hello, world")); err != nil { if err := xlStorage.AppendFile(context.Background(), "src-vol", "file3", []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err) t.Fatalf("Unable to create file, %s", err)
} }
if err := xlStorage.AppendFile("src-vol", "file4", []byte("Hello, world")); err != nil { if err := xlStorage.AppendFile(context.Background(), "src-vol", "file4", []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err) t.Fatalf("Unable to create file, %s", err)
} }
if err := xlStorage.AppendFile("src-vol", "file5", []byte("Hello, world")); err != nil { if err := xlStorage.AppendFile(context.Background(), "src-vol", "file5", []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err) t.Fatalf("Unable to create file, %s", err)
} }
if err := xlStorage.AppendFile("src-vol", "path/to/file1", []byte("Hello, world")); err != nil { if err := xlStorage.AppendFile(context.Background(), "src-vol", "path/to/file1", []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err) t.Fatalf("Unable to create file, %s", err)
} }
@ -1533,7 +1534,7 @@ func TestXLStorageRenameFile(t *testing.T) {
} }
for i, testCase := range testCases { for i, testCase := range testCases {
if err := xlStorage.RenameFile(testCase.srcVol, testCase.srcPath, testCase.destVol, testCase.destPath); err != testCase.expectedErr { if err := xlStorage.RenameFile(context.Background(), testCase.srcVol, testCase.srcPath, testCase.destVol, testCase.destPath); err != testCase.expectedErr {
t.Fatalf("TestXLStorage %d: Expected the error to be : \"%v\", got: \"%v\".", i+1, testCase.expectedErr, err) t.Fatalf("TestXLStorage %d: Expected the error to be : \"%v\", got: \"%v\".", i+1, testCase.expectedErr, err)
} }
} }
@ -1549,15 +1550,15 @@ func TestXLStorageCheckFile(t *testing.T) {
defer os.RemoveAll(path) defer os.RemoveAll(path)
// Setup test environment. // Setup test environment.
if err := xlStorage.MakeVol("success-vol"); err != nil { if err := xlStorage.MakeVol(context.Background(), "success-vol"); err != nil {
t.Fatalf("Unable to create volume, %s", err) t.Fatalf("Unable to create volume, %s", err)
} }
if err := xlStorage.AppendFile("success-vol", pathJoin("success-file", xlStorageFormatFile), []byte("Hello, world")); err != nil { if err := xlStorage.AppendFile(context.Background(), "success-vol", pathJoin("success-file", xlStorageFormatFile), []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err) t.Fatalf("Unable to create file, %s", err)
} }
if err := xlStorage.AppendFile("success-vol", pathJoin("path/to/success-file", xlStorageFormatFile), []byte("Hello, world")); err != nil { if err := xlStorage.AppendFile(context.Background(), "success-vol", pathJoin("path/to/success-file", xlStorageFormatFile), []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err) t.Fatalf("Unable to create file, %s", err)
} }
@ -1611,7 +1612,7 @@ func TestXLStorageCheckFile(t *testing.T) {
} }
for i, testCase := range testCases { for i, testCase := range testCases {
if err := xlStorage.CheckFile(testCase.srcVol, testCase.srcPath); err != testCase.expectedErr { if err := xlStorage.CheckFile(context.Background(), testCase.srcVol, testCase.srcPath); err != testCase.expectedErr {
t.Fatalf("TestXLStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) t.Fatalf("TestXLStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
} }
} }
@ -1634,7 +1635,7 @@ func TestXLStorageVerifyFile(t *testing.T) {
volName := "testvol" volName := "testvol"
fileName := "testfile" fileName := "testfile"
if err := xlStorage.MakeVol(volName); err != nil { if err := xlStorage.MakeVol(context.Background(), volName); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -1648,7 +1649,7 @@ func TestXLStorageVerifyFile(t *testing.T) {
h := algo.New() h := algo.New()
h.Write(data) h.Write(data)
hashBytes := h.Sum(nil) hashBytes := h.Sum(nil)
if err := xlStorage.WriteAll(volName, fileName, bytes.NewBuffer(data)); err != nil { if err := xlStorage.WriteAll(context.Background(), volName, fileName, bytes.NewBuffer(data)); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := xlStorage.storage.bitrotVerify(pathJoin(path, volName, fileName), size, algo, hashBytes, 0); err != nil { if err := xlStorage.storage.bitrotVerify(pathJoin(path, volName, fileName), size, algo, hashBytes, 0); err != nil {
@ -1656,7 +1657,7 @@ func TestXLStorageVerifyFile(t *testing.T) {
} }
// 2) Whole-file bitrot check on corrupted file // 2) Whole-file bitrot check on corrupted file
if err := xlStorage.AppendFile(volName, fileName, []byte("a")); err != nil { if err := xlStorage.AppendFile(context.Background(), volName, fileName, []byte("a")); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -1670,7 +1671,7 @@ func TestXLStorageVerifyFile(t *testing.T) {
t.Fatal("expected to fail bitrot check") t.Fatal("expected to fail bitrot check")
} }
if err := xlStorage.DeleteFile(volName, fileName); err != nil { if err := xlStorage.DeleteFile(context.Background(), volName, fileName); err != nil {
t.Fatal(err) t.Fatal(err)
} }

9
cmd/xl-storage_unix_test.go
View File

@ -19,6 +19,7 @@
package cmd package cmd
import ( import (
"context"
"io/ioutil" "io/ioutil"
"os" "os"
"path" "path"
@ -56,7 +57,7 @@ func TestIsValidUmaskVol(t *testing.T) {
// Attempt to create a volume to verify the permissions later. // Attempt to create a volume to verify the permissions later.
// MakeVol creates 0777. // MakeVol creates 0777.
if err = disk.MakeVol(testCase.volName); err != nil { if err = disk.MakeVol(context.Background(), testCase.volName); err != nil {
t.Fatalf("Creating a volume failed with %s expected to pass.", err) t.Fatalf("Creating a volume failed with %s expected to pass.", err)
} }
defer os.RemoveAll(tmpPath) defer os.RemoveAll(tmpPath)
@ -98,7 +99,7 @@ func TestIsValidUmaskFile(t *testing.T) {
// Attempt to create a volume to verify the permissions later. // Attempt to create a volume to verify the permissions later.
// MakeVol creates directory with 0777 perms. // MakeVol creates directory with 0777 perms.
if err = disk.MakeVol(testCase.volName); err != nil { if err = disk.MakeVol(context.Background(), testCase.volName); err != nil {
t.Fatalf("Creating a volume failed with %s expected to pass.", err) t.Fatalf("Creating a volume failed with %s expected to pass.", err)
} }
@ -106,12 +107,12 @@ func TestIsValidUmaskFile(t *testing.T) {
// Attempt to create a file to verify the permissions later. // Attempt to create a file to verify the permissions later.
// AppendFile creates file with 0666 perms. // AppendFile creates file with 0666 perms.
if err = disk.AppendFile(testCase.volName, pathJoin("hello-world.txt", xlStorageFormatFile), []byte("Hello World")); err != nil { if err = disk.AppendFile(context.Background(), testCase.volName, pathJoin("hello-world.txt", xlStorageFormatFile), []byte("Hello World")); err != nil {
t.Fatalf("Create a file `test` failed with %s expected to pass.", err) t.Fatalf("Create a file `test` failed with %s expected to pass.", err)
} }
// CheckFile - stat the file. // CheckFile - stat the file.
if err := disk.CheckFile(testCase.volName, "hello-world.txt"); err != nil { if err := disk.CheckFile(context.Background(), testCase.volName, "hello-world.txt"); err != nil {
t.Fatalf("Stat failed with %s expected to pass.", err) t.Fatalf("Stat failed with %s expected to pass.", err)
} }
} }

13
cmd/xl-storage_windows_test.go
View File

@ -20,6 +20,7 @@ package cmd
import ( import (
"bytes" "bytes"
"context"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"os" "os"
@ -54,20 +55,20 @@ func TestUNCPaths(t *testing.T) {
} }
// Create volume to use in conjunction with other StorageAPI's file API(s) // Create volume to use in conjunction with other StorageAPI's file API(s)
err = fs.MakeVol("voldir") err = fs.MakeVol(context.Background(), "voldir")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
for i, test := range testCases { for i, test := range testCases {
t.Run(fmt.Sprint(i), func(t *testing.T) { t.Run(fmt.Sprint(i), func(t *testing.T) {
err = fs.AppendFile("voldir", test.objName, []byte("hello")) err = fs.AppendFile(context.Background(), "voldir", test.objName, []byte("hello"))
if err != nil && test.pass { if err != nil && test.pass {
t.Error(err) t.Error(err)
} else if err == nil && !test.pass { } else if err == nil && !test.pass {
t.Error(err) t.Error(err)
} }
fs.DeleteFile("voldir", test.objName) fs.DeleteFile(context.Background(), "voldir", test.objName)
}) })
} }
} }
@ -89,19 +90,19 @@ func TestUNCPathENOTDIR(t *testing.T) {
} }
// Create volume to use in conjunction with other StorageAPI's file API(s) // Create volume to use in conjunction with other StorageAPI's file API(s)
err = fs.MakeVol("voldir") err = fs.MakeVol(context.Background(), "voldir")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
err = fs.AppendFile("voldir", "/file", []byte("hello")) err = fs.AppendFile(context.Background(), "voldir", "/file", []byte("hello"))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
// Try to create a file that includes a file in its path components. // Try to create a file that includes a file in its path components.
// In *nix, this returns syscall.ENOTDIR while in windows we receive the following error. // In *nix, this returns syscall.ENOTDIR while in windows we receive the following error.
err = fs.AppendFile("voldir", "/file/obj1", []byte("hello")) err = fs.AppendFile(context.Background(), "voldir", "/file/obj1", []byte("hello"))
if err != errFileAccessDenied { if err != errFileAccessDenied {
t.Errorf("expected: %s, got: %s", errFileAccessDenied, err) t.Errorf("expected: %s, got: %s", errFileAccessDenied, err)
} }