MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-23 21:55:53 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add storage layer contexts (#10321)

Browse Source 
Add context to all (non-trivial) calls to the storage layer. 

Contexts are propagated through the REST client.

- `context.TODO()` is left in place for the places where it needs to be added to the caller.
- `endWalkCh` could probably be removed from the walkers, but no changes so far.

The "dangerous" part is that now a caller disconnecting *will* propagate down,  so a 
"delete" operation will now be interrupted. In some cases we might want to disconnect 
this functionality so the operation completes if it has started, leaving the system in a cleaner state.
This commit is contained in:
Klaus Post 2020-09-04 09:45:06 -07:00 committed by GitHub
parent 0037951b6e
commit 2d58a8d861
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
36 changed files with 466 additions and 467 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cmd/bitrot-streaming.go
View File

@ -18,6 +18,7 @@ package cmd
import (
"bytes"
"context"
"encoding/hex"
"fmt"
"hash"
@ -80,7 +81,7 @@ func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length i
bitrotSumsTotalSize := ceilFrac(length, shardSize) * int64(h.Size()) // Size used for storing bitrot checksums.
totalFileSize = bitrotSumsTotalSize + length
}
err := disk.CreateFile(volume, filePath, totalFileSize, r)
err := disk.CreateFile(context.TODO(), volume, filePath, totalFileSize, r)
r.CloseWithError(err)
close(bw.canClose)
}()
@ -118,7 +119,7 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
// For the first ReadAt() call we need to open the stream for reading.
b.currOffset = offset
streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
b.rc, err = b.disk.ReadFileStream(b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
if err != nil {
return 0, err
}

5
cmd/bitrot-whole.go
View File

@ -17,6 +17,7 @@
package cmd
import (
"context"
"fmt"
"hash"
"io"
@ -34,7 +35,7 @@ type wholeBitrotWriter struct {
}
func (b *wholeBitrotWriter) Write(p []byte) (int, error) {
err := b.disk.AppendFile(b.volume, b.filePath, p)
err := b.disk.AppendFile(context.TODO(), b.volume, b.filePath, p)
if err != nil {
logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err))
return 0, err
@ -69,7 +70,7 @@ type wholeBitrotReader struct {
func (b *wholeBitrotReader) ReadAt(buf []byte, offset int64) (n int, err error) {
if b.buf == nil {
b.buf = make([]byte, b.tillOffset-offset)
if _, err := b.disk.ReadFile(b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
if _, err := b.disk.ReadFile(context.TODO(), b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err))
return 0, err
}

3
cmd/bitrot_test.go
View File

@ -17,6 +17,7 @@
package cmd
import (
"context"
"io"
"io/ioutil"
"log"
@ -39,7 +40,7 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
t.Fatal(err)
}
disk.MakeVol(volume)
disk.MakeVol(context.Background(), volume)
writer := newBitrotWriter(disk, volume, filePath, 35, bitrotAlgo, 10)

4
cmd/bootstrap-peer-server.go
View File

@ -131,7 +131,7 @@ func (client *bootstrapRESTClient) callWithContext(ctx context.Context, method s
values = make(url.Values)
}
respBody, err = client.restClient.CallWithContext(ctx, method, values, body, length)
respBody, err = client.restClient.Call(ctx, method, values, body, length)
if err == nil {
return respBody, nil
}
@ -234,7 +234,7 @@ func newBootstrapRESTClient(endpoint Endpoint) *bootstrapRESTClient {
ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
// Instantiate a new rest client for healthcheck
// to avoid recursive healthCheckFn()
respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).CallWithContext(ctx, bootstrapRESTMethodHealth, nil, nil, -1)
respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).Call(ctx, bootstrapRESTMethodHealth, nil, nil, -1)
xhttp.DrainBody(respBody)
cancel()
var ne *rest.NetworkError

14
cmd/erasure-bucket.go
View File

@ -49,7 +49,7 @@ func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket stri
index := index
g.Go(func() error {
if storageDisks[index] != nil {
if err := storageDisks[index].MakeVol(bucket); err != nil {
if err := storageDisks[index].MakeVol(ctx, bucket); err != nil {
if err != errVolumeExists {
logger.LogIf(ctx, err)
}
@ -75,7 +75,7 @@ func undoDeleteBucket(storageDisks []StorageAPI, bucket string) {
}
index := index
g.Go(func() error {
_ = storageDisks[index].MakeVol(bucket)
_ = storageDisks[index].MakeVol(context.Background(), bucket)
return nil
}, index)
}
@ -92,7 +92,7 @@ func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string) (
bucketErrs = append(bucketErrs, errDiskNotFound)
continue
}
volInfo, serr := disk.StatVol(bucketName)
volInfo, serr := disk.StatVol(ctx, bucketName)
if serr == nil {
return BucketInfo(volInfo), nil
}
@ -129,7 +129,7 @@ func (er erasureObjects) listBuckets(ctx context.Context) (bucketsInfo []BucketI
continue
}
var volsInfo []VolInfo
volsInfo, err = disk.ListVols()
volsInfo, err = disk.ListVols(ctx)
if err == nil {
// NOTE: The assumption here is that volumes across all disks in
// readQuorum have consistent view i.e they all have same number
@ -181,10 +181,10 @@ func deleteDanglingBucket(ctx context.Context, storageDisks []StorageAPI, dErrs
for index, err := range dErrs {
if err == errVolumeNotEmpty {
// Attempt to delete bucket again.
if derr := storageDisks[index].DeleteVol(bucket, false); derr == errVolumeNotEmpty {
if derr := storageDisks[index].DeleteVol(ctx, bucket, false); derr == errVolumeNotEmpty {
_ = cleanupDir(ctx, storageDisks[index], bucket, "")
_ = storageDisks[index].DeleteVol(bucket, false)
_ = storageDisks[index].DeleteVol(ctx, bucket, false)
// Cleanup all the previously incomplete multiparts.
_ = cleanupDir(ctx, storageDisks[index], minioMetaMultipartBucket, bucket)
@ -204,7 +204,7 @@ func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, forceD
index := index
g.Go(func() error {
if storageDisks[index] != nil {
if err := storageDisks[index].DeleteVol(bucket, forceDelete); err != nil {
if err := storageDisks[index].DeleteVol(ctx, bucket, forceDelete); err != nil {
return err
}
err := cleanupDir(ctx, storageDisks[index], minioMetaMultipartBucket, bucket)

2
cmd/erasure-common.go
View File

@ -77,7 +77,7 @@ func (er erasureObjects) isObject(ctx context.Context, bucket, prefix string) (o
return errDiskNotFound
}
// Check if 'prefix' is an object on this 'disk', else continue the check the next disk
return storageDisks[index].CheckFile(bucket, prefix)
return storageDisks[index].CheckFile(ctx, bucket, prefix)
}, index)
}

2
cmd/erasure-decode_test.go
View File

@ -28,7 +28,7 @@ import (
humanize "github.com/dustin/go-humanize"
)
func (a badDisk) ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
func (a badDisk) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
return 0, errFaultyDisk
}

8
cmd/erasure-encode_test.go
View File

@ -32,11 +32,11 @@ func (a badDisk) String() string {
return "bad-disk"
}
func (a badDisk) AppendFile(volume string, path string, buf []byte) error {
func (a badDisk) AppendFile(ctx context.Context, volume string, path string, buf []byte) error {
return errFaultyDisk
}
func (a badDisk) ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error) {
func (a badDisk) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) {
return nil, errFaultyDisk
}
@ -44,7 +44,7 @@ func (a badDisk) UpdateBloomFilter(ctx context.Context, oldest, current uint64)
return nil, errFaultyDisk
}
func (a badDisk) CreateFile(volume, path string, size int64, reader io.Reader) error {
func (a badDisk) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error {
return errFaultyDisk
}
@ -195,7 +195,7 @@ func benchmarkErasureEncode(data, parity, dataDown, parityDown int, size int64,
if disk == OfflineDisk {
continue
}
disk.DeleteFile("testbucket", "object")
disk.DeleteFile(context.Background(), "testbucket", "object")
writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize())
}
_, err := erasure.Encode(context.Background(), bytes.NewReader(content), writers, buffer, erasure.dataBlocks+1)

4
cmd/erasure-healing-common.go
View File

@ -175,9 +175,9 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
// disk has a valid xl.meta but may not have all the
// parts. This is considered an outdated disk, since
// it needs healing too.
dataErrs[i] = onlineDisk.VerifyFile(bucket, object, partsMetadata[i])
dataErrs[i] = onlineDisk.VerifyFile(ctx, bucket, object, partsMetadata[i])
case madmin.HealNormalScan:
dataErrs[i] = onlineDisk.CheckParts(bucket, object, partsMetadata[i])
dataErrs[i] = onlineDisk.CheckParts(ctx, bucket, object, partsMetadata[i])
}
if dataErrs[i] == nil {

2
cmd/erasure-healing-common_test.go
View File

@ -210,7 +210,7 @@ func TestListOnlineDisks(t *testing.T) {
// and check if that disk
// appears in outDatedDisks.
tamperedIndex = index
dErr := erasureDisks[index].DeleteFile(bucket, pathJoin(object, fi.DataDir, "part.1"))
dErr := erasureDisks[index].DeleteFile(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
if dErr != nil {
t.Fatalf("Test %d: Failed to delete %s - %v", i+1,
filepath.Join(object, "part.1"), dErr)

14
cmd/erasure-healing.go
View File

@ -78,7 +78,7 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints
afterState[index] = madmin.DriveStateOffline
return errDiskNotFound
}
if _, serr := storageDisks[index].StatVol(bucket); serr != nil {
if _, serr := storageDisks[index].StatVol(ctx, bucket); serr != nil {
if serr == errDiskNotFound {
beforeState[index] = madmin.DriveStateOffline
afterState[index] = madmin.DriveStateOffline
@ -136,7 +136,7 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints
index := index
g.Go(func() error {
if beforeState[index] == madmin.DriveStateMissing {
makeErr := storageDisks[index].MakeVol(bucket)
makeErr := storageDisks[index].MakeVol(ctx, bucket)
if makeErr == nil {
afterState[index] = madmin.DriveStateOk
}
@ -171,7 +171,7 @@ func listAllBuckets(storageDisks []StorageAPI, healBuckets map[string]VolInfo) (
continue
}
var volsInfo []VolInfo
volsInfo, err = disk.ListVols()
volsInfo, err = disk.ListVols(context.TODO())
if err != nil {
if IsErrIgnored(err, bucketMetadataOpIgnoredErrs...) {
continue
@ -472,7 +472,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
}
// Attempt a rename now from healed data to final location.
if err = disk.RenameData(minioMetaTmpBucket, tmpID, partsMetadata[i].DataDir, bucket, object); err != nil {
if err = disk.RenameData(ctx, minioMetaTmpBucket, tmpID, partsMetadata[i].DataDir, bucket, object); err != nil {
if err != errIsNotRegular && err != errFileNotFound {
logger.LogIf(ctx, err)
}
@ -525,7 +525,7 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
wg.Add(1)
go func(index int, disk StorageAPI) {
defer wg.Done()
_ = disk.DeleteFile(bucket, object)
_ = disk.DeleteFile(ctx, bucket, object)
}(index, disk)
}
wg.Wait()
@ -557,7 +557,7 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
for i, err := range errs {
if err == errVolumeNotFound || err == errFileNotFound {
// Bucket or prefix/directory not found
merr := storageDisks[i].MakeVol(pathJoin(bucket, object))
merr := storageDisks[i].MakeVol(ctx, pathJoin(bucket, object))
switch merr {
case nil, errVolumeExists:
hr.After.Drives[i].State = madmin.DriveStateOk
@ -642,7 +642,7 @@ func statAllDirs(ctx context.Context, storageDisks []StorageAPI, bucket, prefix
}
index := index
g.Go(func() error {
entries, err := storageDisks[index].ListDir(bucket, prefix, 1)
entries, err := storageDisks[index].ListDir(ctx, bucket, prefix, 1)
if err != nil {
return err
}

32
cmd/erasure-healing_test.go
View File

@ -66,7 +66,7 @@ func TestHealing(t *testing.T) {
}
disk := er.getDisks()[0]
fileInfoPreHeal, err := disk.ReadVersion(bucket, object, "")
fileInfoPreHeal, err := disk.ReadVersion(context.Background(), bucket, object, "")
if err != nil {
t.Fatal(err)
}
@ -83,7 +83,7 @@ func TestHealing(t *testing.T) {
t.Fatal(err)
}
fileInfoPostHeal, err := disk.ReadVersion(bucket, object, "")
fileInfoPostHeal, err := disk.ReadVersion(context.Background(), bucket, object, "")
if err != nil {
t.Fatal(err)
}
@ -102,7 +102,7 @@ func TestHealing(t *testing.T) {
// gone down when an object was replaced by a new object.
fileInfoOutDated := fileInfoPreHeal
fileInfoOutDated.ModTime = time.Now()
err = disk.WriteMetadata(bucket, object, fileInfoOutDated)
err = disk.WriteMetadata(context.Background(), bucket, object, fileInfoOutDated)
if err != nil {
t.Fatal(err)
}
@ -112,7 +112,7 @@ func TestHealing(t *testing.T) {
t.Fatal(err)
}
fileInfoPostHeal, err = disk.ReadVersion(bucket, object, "")
fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "")
if err != nil {
t.Fatal(err)
}
@ -134,7 +134,7 @@ func TestHealing(t *testing.T) {
t.Fatal(err)
}
// Stat the bucket to make sure that it was created.
_, err = er.getDisks()[0].StatVol(bucket)
_, err = er.getDisks()[0].StatVol(context.Background(), bucket)
if err != nil {
t.Fatal(err)
}
@ -200,7 +200,7 @@ func TestHealObjectCorrupted(t *testing.T) {
er := z.zones[0].sets[0]
erasureDisks := er.getDisks()
firstDisk := erasureDisks[0]
err = firstDisk.DeleteFile(bucket, pathJoin(object, xlStorageFormatFile))
err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile))
if err != nil {
t.Fatalf("Failed to delete a file - %v", err)
}
@ -216,16 +216,16 @@ func TestHealObjectCorrupted(t *testing.T) {
t.Fatalf("Failed to getLatestFileInfo - %v", err)
}
if err = firstDisk.CheckFile(bucket, object); err != nil {
if err = firstDisk.CheckFile(context.Background(), bucket, object); err != nil {
t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
}
err = firstDisk.DeleteFile(bucket, pathJoin(object, fi.DataDir, "part.1"))
err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
if err != nil {
t.Errorf("Failure during deleting part.1 - %v", err)
}
err = firstDisk.WriteAll(bucket, pathJoin(object, fi.DataDir, "part.1"), bytes.NewReader([]byte{}))
err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bytes.NewReader([]byte{}))
if err != nil {
t.Errorf("Failure during creating part.1 - %v", err)
}
@ -245,13 +245,13 @@ func TestHealObjectCorrupted(t *testing.T) {
t.Fatalf("FileInfo not equal after healing")
}
err = firstDisk.DeleteFile(bucket, pathJoin(object, fi.DataDir, "part.1"))
err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
if err != nil {
t.Errorf("Failure during deleting part.1 - %v", err)
}
bdata := bytes.Repeat([]byte("b"), int(nfi.Size))
err = firstDisk.WriteAll(bucket, pathJoin(object, fi.DataDir, "part.1"), bytes.NewReader(bdata))
err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bytes.NewReader(bdata))
if err != nil {
t.Errorf("Failure during creating part.1 - %v", err)
}
@ -274,7 +274,7 @@ func TestHealObjectCorrupted(t *testing.T) {
// Test 4: checks if HealObject returns an error when xl.meta is not found
// in more than read quorum number of disks, to create a corrupted situation.
for i := 0; i <= len(er.getDisks())/2; i++ {
er.getDisks()[i].DeleteFile(bucket, pathJoin(object, xlStorageFormatFile))
er.getDisks()[i].DeleteFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile))
}
// Try healing now, expect to receive errFileNotFound.
@ -350,7 +350,7 @@ func TestHealObjectErasure(t *testing.T) {
t.Fatalf("Failed to complete multipart upload - %v", err)
}
err = firstDisk.DeleteFile(bucket, pathJoin(object, xlStorageFormatFile))
err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile))
if err != nil {
t.Fatalf("Failed to delete a file - %v", err)
}
@ -360,7 +360,7 @@ func TestHealObjectErasure(t *testing.T) {
t.Fatalf("Failed to heal object - %v", err)
}
if err = firstDisk.CheckFile(bucket, object); err != nil {
if err = firstDisk.CheckFile(context.Background(), bucket, object); err != nil {
t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
}
@ -421,7 +421,7 @@ func TestHealEmptyDirectoryErasure(t *testing.T) {
z := obj.(*erasureZones)
er := z.zones[0].sets[0]
firstDisk := er.getDisks()[0]
err = firstDisk.DeleteFile(bucket, object)
err = firstDisk.DeleteFile(context.Background(), bucket, object)
if err != nil {
t.Fatalf("Failed to delete a file - %v", err)
}
@ -433,7 +433,7 @@ func TestHealEmptyDirectoryErasure(t *testing.T) {
}
// Check if the empty directory is restored in the first disk
_, err = firstDisk.StatVol(pathJoin(bucket, object))
_, err = firstDisk.StatVol(context.Background(), pathJoin(bucket, object))
if err != nil {
t.Fatalf("Expected object to be present but stat failed - %v", err)
}

2
cmd/erasure-metadata-utils.go
View File

@ -126,7 +126,7 @@ func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, ve
if disks[index] == nil {
return errDiskNotFound
}
metadataArray[index], err = disks[index].ReadVersion(bucket, object, versionID)
metadataArray[index], err = disks[index].ReadVersion(ctx, bucket, object, versionID)
if err != nil {
if err != errFileNotFound && err != errVolumeNotFound && err != errFileVersionNotFound {
logger.GetReqInfo(ctx).AppendTags("disk", disks[index].String())

4
cmd/erasure-metadata.go
View File

@ -273,7 +273,7 @@ func renameFileInfo(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry
if disks[index] == nil {
return errDiskNotFound
}
if err := disks[index].RenameData(srcBucket, srcEntry, "", dstBucket, dstEntry); err != nil {
if err := disks[index].RenameData(ctx, srcBucket, srcEntry, "", dstBucket, dstEntry); err != nil {
if !IsErrIgnored(err, ignoredErr...) {
return err
}
@ -304,7 +304,7 @@ func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, bucket, prefix
}
// Pick one FileInfo for a disk at index.
files[index].Erasure.Index = index + 1
return disks[index].WriteMetadata(bucket, prefix, files[index])
return disks[index].WriteMetadata(ctx, bucket, prefix, files[index])
}, index)
}

12
cmd/erasure-multipart.go
View File

@ -63,7 +63,7 @@ func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir stri
// Ignoring failure to remove parts that weren't present in CompleteMultipartUpload
// requests. xl.meta is the authoritative source of truth on which parts constitute
// the object. The presence of parts that don't belong in the object doesn't affect correctness.
_ = storageDisks[index].DeleteFile(minioMetaMultipartBucket, curpartPath)
_ = storageDisks[index].DeleteFile(context.TODO(), minioMetaMultipartBucket, curpartPath)
return nil
}, index)
}
@ -99,18 +99,18 @@ func (er erasureObjects) cleanupStaleMultipartUploads(ctx context.Context, clean
// Remove the old multipart uploads on the given disk.
func (er erasureObjects) cleanupStaleMultipartUploadsOnDisk(ctx context.Context, disk StorageAPI, expiry time.Duration) {
now := time.Now()
shaDirs, err := disk.ListDir(minioMetaMultipartBucket, "", -1)
shaDirs, err := disk.ListDir(ctx, minioMetaMultipartBucket, "", -1)
if err != nil {
return
}
for _, shaDir := range shaDirs {
uploadIDDirs, err := disk.ListDir(minioMetaMultipartBucket, shaDir, -1)
uploadIDDirs, err := disk.ListDir(ctx, minioMetaMultipartBucket, shaDir, -1)
if err != nil {
continue
}
for _, uploadIDDir := range uploadIDDirs {
uploadIDPath := pathJoin(shaDir, uploadIDDir)
fi, err := disk.ReadVersion(minioMetaMultipartBucket, uploadIDPath, "")
fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, uploadIDPath, "")
if err != nil {
continue
}
@ -139,7 +139,7 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
if disk == nil {
continue
}
uploadIDs, err = disk.ListDir(minioMetaMultipartBucket, er.getMultipartSHADir(bucket, object), -1)
uploadIDs, err = disk.ListDir(ctx, minioMetaMultipartBucket, er.getMultipartSHADir(bucket, object), -1)
if err != nil {
if err == errDiskNotFound {
continue
@ -172,7 +172,7 @@ retry:
if populatedUploadIds.Contains(uploadID) {
continue
}
fi, err := disk.ReadVersion(minioMetaMultipartBucket, pathJoin(er.getUploadIDDir(bucket, object, uploadID)), "")
fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, pathJoin(er.getUploadIDDir(bucket, object, uploadID)), "")
if err != nil {
if err == errDiskNotFound || err == errFileNotFound {
goto retry

16
cmd/erasure-object.go
View File

@ -49,7 +49,7 @@ func (er erasureObjects) putObjectDir(ctx context.Context, bucket, object string
}
index := index
g.Go(func() error {
err := storageDisks[index].MakeVol(pathJoin(bucket, object))
err := storageDisks[index].MakeVol(ctx, pathJoin(bucket, object))
if err != nil && err != errVolumeExists {
return err
}
@ -362,7 +362,7 @@ func (er erasureObjects) getObjectInfoDir(ctx context.Context, bucket, object st
index := index
g.Go(func() error {
// Check if 'prefix' is an object on this 'disk'.
entries, err := storageDisks[index].ListDir(bucket, object, 1)
entries, err := storageDisks[index].ListDir(ctx, bucket, object, 1)
if err != nil {
return err
}
@ -459,7 +459,7 @@ func undoRename(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry str
index := index
g.Go(func() error {
if errs[index] == nil {
_ = disks[index].RenameFile(dstBucket, dstEntry, srcBucket, srcEntry)
_ = disks[index].RenameFile(context.TODO(), dstBucket, dstEntry, srcBucket, srcEntry)
}
return nil
}, index)
@ -479,7 +479,7 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, da
if disks[index] == nil {
return errDiskNotFound
}
if err := disks[index].RenameData(srcBucket, srcEntry, dataDir, dstBucket, dstEntry); err != nil {
if err := disks[index].RenameData(ctx, srcBucket, srcEntry, dataDir, dstBucket, dstEntry); err != nil {
if !IsErrIgnored(err, ignoredErr...) {
return err
}
@ -504,7 +504,7 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, da
ug.Go(func() error {
// Undo all the partial rename operations.
if errs[index] == nil {
_ = disks[index].RenameData(dstBucket, dstEntry, dataDir, srcBucket, srcEntry)
_ = disks[index].RenameData(context.Background(), dstBucket, dstEntry, dataDir, srcBucket, srcEntry)
}
return nil
}, index)
@ -532,7 +532,7 @@ func rename(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBuc
if disks[index] == nil {
return errDiskNotFound
}
if err := disks[index].RenameFile(srcBucket, srcEntry, dstBucket, dstEntry); err != nil {
if err := disks[index].RenameFile(ctx, srcBucket, srcEntry, dstBucket, dstEntry); err != nil {
if !IsErrIgnored(err, ignoredErr...) {
return err
}
@ -776,7 +776,7 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object
if disks[index] == nil {
return errDiskNotFound
}
err := disks[index].DeleteVersion(bucket, object, fi)
err := disks[index].DeleteVersion(ctx, bucket, object, fi)
if err != nil && err != errVolumeNotFound {
return err
}
@ -899,7 +899,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
wg.Add(1)
go func(index int, disk StorageAPI) {
defer wg.Done()
delObjErrs[index] = disk.DeleteVersions(bucket, versions)
delObjErrs[index] = disk.DeleteVersions(ctx, bucket, versions)
}(index, disk)
}

10
cmd/erasure-sets.go
View File

@ -140,7 +140,7 @@ func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
// Close the internal connection to avoid connection leaks.
disk.Close()
if errors.Is(err, errUnformattedDisk) {
info, derr := disk.DiskInfo()
info, derr := disk.DiskInfo(context.TODO())
if derr != nil && info.RootDisk {
return nil, nil, fmt.Errorf("Disk: %s returned %w but its a root disk refusing to use it",
disk, derr) // make sure to '%w' to wrap the error
@ -981,7 +981,7 @@ func (s *erasureSets) startMergeWalksVersionsN(ctx context.Context, bucket, pref
// Disk can be offline
continue
}
entryCh, err := disk.WalkVersions(bucket, prefix, marker, recursive, endWalkCh)
entryCh, err := disk.WalkVersions(GlobalContext, bucket, prefix, marker, recursive, endWalkCh)
if err != nil {
logger.LogIf(ctx, err)
// Disk walk returned error, ignore it.
@ -1012,7 +1012,7 @@ func (s *erasureSets) startMergeWalksN(ctx context.Context, bucket, prefix, mark
// Disk can be offline
continue
}
entryCh, err := disk.Walk(bucket, prefix, marker, recursive, endWalkCh)
entryCh, err := disk.Walk(GlobalContext, bucket, prefix, marker, recursive, endWalkCh)
if err != nil {
// Disk walk returned error, ignore it.
continue
@ -1042,7 +1042,7 @@ func (s *erasureSets) startSplunkMergeWalksN(ctx context.Context, bucket, prefix
// Disk can be offline
continue
}
entryCh, err := disk.WalkSplunk(bucket, prefix, marker, endWalkCh)
entryCh, err := disk.WalkSplunk(GlobalContext, bucket, prefix, marker, endWalkCh)
if err != nil {
// Disk walk returned error, ignore it.
continue
@ -1300,7 +1300,7 @@ func getHealDiskInfos(storageDisks []StorageAPI, errs []error) ([]DiskInfo, []er
return errDiskNotFound
}
var err error
infos[index], err = storageDisks[index].DiskInfo()
infos[index], err = storageDisks[index].DiskInfo(context.TODO())
return err
}, index)
}

4
cmd/erasure.go
View File

@ -139,7 +139,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di
// Storage disk is empty, perhaps ignored disk or not available.
return errDiskNotFound
}
info, err := disks[index].DiskInfo()
info, err := disks[index].DiskInfo(context.TODO())
if err != nil {
if !IsErr(err, baseErrs...) {
reqInfo := (&logger.ReqInfo{}).AppendTags("disk", disks[index].String())
@ -282,7 +282,7 @@ func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []Buc
if d == nil || !d.IsOnline() {
continue
}
di, err := d.DiskInfo()
di, err := d.DiskInfo(ctx)
if err != nil {
logger.LogIf(ctx, err)
continue

2
cmd/erasure_test.go
View File

@ -134,7 +134,7 @@ func newErasureTestSetup(dataBlocks int, parityBlocks int, blockSize int64) (*er
if err != nil {
return nil, err
}
err = disks[i].MakeVol("testbucket")
err = disks[i].MakeVol(context.Background(), "testbucket")
if err != nil {
return nil, err
}

12
cmd/format-erasure.go
View File

@ -355,15 +355,15 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3) error {
tmpFormat := mustGetUUID()
// Purge any existing temporary file, okay to ignore errors here.
defer disk.DeleteFile(minioMetaBucket, tmpFormat)
defer disk.DeleteFile(context.TODO(), minioMetaBucket, tmpFormat)
// write to unique file.
if err = disk.WriteAll(minioMetaBucket, tmpFormat, bytes.NewReader(formatBytes)); err != nil {
if err = disk.WriteAll(context.TODO(), minioMetaBucket, tmpFormat, bytes.NewReader(formatBytes)); err != nil {
return err
}
// Rename file `uuid.json` --> `format.json`.
if err = disk.RenameFile(minioMetaBucket, tmpFormat, minioMetaBucket, formatConfigFile); err != nil {
if err = disk.RenameFile(context.TODO(), minioMetaBucket, tmpFormat, minioMetaBucket, formatConfigFile); err != nil {
return err
}
@ -392,13 +392,13 @@ func isHiddenDirectories(vols ...VolInfo) bool {
// loadFormatErasure - loads format.json from disk.
func loadFormatErasure(disk StorageAPI) (format *formatErasureV3, err error) {
buf, err := disk.ReadAll(minioMetaBucket, formatConfigFile)
buf, err := disk.ReadAll(context.TODO(), minioMetaBucket, formatConfigFile)
if err != nil {
// 'file not found' and 'volume not found' as
// same. 'volume not found' usually means its a fresh disk.
if err == errFileNotFound || err == errVolumeNotFound {
var vols []VolInfo
vols, err = disk.ListVols()
vols, err = disk.ListVols(context.TODO())
if err != nil {
return nil, err
}
@ -862,7 +862,7 @@ func makeFormatErasureMetaVolumes(disk StorageAPI) error {
return errDiskNotFound
}
// Attempt to create MinIO internal buckets.
return disk.MakeVolBulk(minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, dataUsageBucket)
return disk.MakeVolBulk(context.TODO(), minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, dataUsageBucket)
}
// Get all UUIDs which are present in reference format should

2
cmd/global-heal.go
View File

@ -131,7 +131,7 @@ func healErasureSet(ctx context.Context, setIndex int, xlObj *erasureObjects, se
continue
}
entryCh, err := disk.WalkVersions(bucket.Name, "", "", true, ctx.Done())
entryCh, err := disk.WalkVersions(ctx, bucket.Name, "", "", true, ctx.Done())
if err != nil {
// Disk walk returned error, ignore it.
continue

4
cmd/lock-rest-client.go
View File

@ -63,7 +63,7 @@ func (client *lockRESTClient) callWithContext(ctx context.Context, method string
values = make(url.Values)
}
respBody, err = client.restClient.Call(method, values, body, length)
respBody, err = client.restClient.Call(ctx, method, values, body, length)
if err == nil {
return respBody, nil
}
@ -158,7 +158,7 @@ func newlockRESTClient(endpoint Endpoint) *lockRESTClient {
ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
// Instantiate a new rest client for healthcheck
// to avoid recursive healthCheckFn()
respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).CallWithContext(ctx, lockRESTMethodHealth, nil, nil, -1)
respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).Call(ctx, lockRESTMethodHealth, nil, nil, -1)
xhttp.DrainBody(respBody)
cancel()
var ne *rest.NetworkError

104
cmd/naughty-disk_test.go
View File

@ -94,139 +94,139 @@ func (d *naughtyDisk) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageC
return d.disk.CrawlAndGetDataUsage(ctx, cache)
}
func (d *naughtyDisk) DiskInfo() (info DiskInfo, err error) {
func (d *naughtyDisk) DiskInfo(ctx context.Context) (info DiskInfo, err error) {
if err := d.calcError(); err != nil {
return info, err
}
return d.disk.DiskInfo()
return d.disk.DiskInfo(ctx)
}
func (d *naughtyDisk) MakeVolBulk(volumes ...string) (err error) {
func (d *naughtyDisk) MakeVolBulk(ctx context.Context, volumes ...string) (err error) {
if err := d.calcError(); err != nil {
return err
}
return d.disk.MakeVolBulk(volumes...)
return d.disk.MakeVolBulk(ctx, volumes...)
}
func (d *naughtyDisk) MakeVol(volume string) (err error) {
func (d *naughtyDisk) MakeVol(ctx context.Context, volume string) (err error) {
if err := d.calcError(); err != nil {
return err
}
return d.disk.MakeVol(volume)
return d.disk.MakeVol(ctx, volume)
}
func (d *naughtyDisk) ListVols() (vols []VolInfo, err error) {
func (d *naughtyDisk) ListVols(ctx context.Context) (vols []VolInfo, err error) {
if err := d.calcError(); err != nil {
return nil, err
}
return d.disk.ListVols()
return d.disk.ListVols(ctx)
}
func (d *naughtyDisk) StatVol(volume string) (volInfo VolInfo, err error) {
func (d *naughtyDisk) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) {
if err := d.calcError(); err != nil {
return VolInfo{}, err
}
return d.disk.StatVol(volume)
return d.disk.StatVol(ctx, volume)
}
func (d *naughtyDisk) DeleteVol(volume string, forceDelete bool) (err error) {
func (d *naughtyDisk) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) {
if err := d.calcError(); err != nil {
return err
}
return d.disk.DeleteVol(volume, forceDelete)
return d.disk.DeleteVol(ctx, volume, forceDelete)
}
func (d *naughtyDisk) WalkSplunk(volume, path, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error) {
func (d *naughtyDisk) WalkSplunk(ctx context.Context, volume, dirPath, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error) {
if err := d.calcError(); err != nil {
return nil, err
}
return d.disk.WalkSplunk(volume, path, marker, endWalkCh)
return d.disk.WalkSplunk(ctx, volume, dirPath, marker, endWalkCh)
}
func (d *naughtyDisk) WalkVersions(volume, path, marker string, recursive bool, endWalkVersionsCh <-chan struct{}) (chan FileInfoVersions, error) {
func (d *naughtyDisk) WalkVersions(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error) {
if err := d.calcError(); err != nil {
return nil, err
}
return d.disk.WalkVersions(volume, path, marker, recursive, endWalkVersionsCh)
return d.disk.WalkVersions(ctx, volume, dirPath, marker, recursive, endWalkCh)
}
func (d *naughtyDisk) Walk(volume, path, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) {
func (d *naughtyDisk) Walk(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) {
if err := d.calcError(); err != nil {
return nil, err
}
return d.disk.Walk(volume, path, marker, recursive, endWalkCh)
return d.disk.Walk(ctx, volume, dirPath, marker, recursive, endWalkCh)
}
func (d *naughtyDisk) ListDir(volume, path string, count int) (entries []string, err error) {
func (d *naughtyDisk) ListDir(ctx context.Context, volume, dirPath string, count int) (entries []string, err error) {
if err := d.calcError(); err != nil {
return []string{}, err
}
return d.disk.ListDir(volume, path, count)
return d.disk.ListDir(ctx, volume, dirPath, count)
}
func (d *naughtyDisk) ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
func (d *naughtyDisk) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
if err := d.calcError(); err != nil {
return 0, err
}
return d.disk.ReadFile(volume, path, offset, buf, verifier)
return d.disk.ReadFile(ctx, volume, path, offset, buf, verifier)
}
func (d *naughtyDisk) ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error) {
func (d *naughtyDisk) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) {
if err := d.calcError(); err != nil {
return nil, err
}
return d.disk.ReadFileStream(volume, path, offset, length)
return d.disk.ReadFileStream(ctx, volume, path, offset, length)
}
func (d *naughtyDisk) CreateFile(volume, path string, size int64, reader io.Reader) error {
func (d *naughtyDisk) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error {
if err := d.calcError(); err != nil {
return err
}
return d.disk.CreateFile(volume, path, size, reader)
return d.disk.CreateFile(ctx, volume, path, size, reader)
}
func (d *naughtyDisk) AppendFile(volume, path string, buf []byte) error {
func (d *naughtyDisk) AppendFile(ctx context.Context, volume string, path string, buf []byte) error {
if err := d.calcError(); err != nil {
return err
}
return d.disk.AppendFile(volume, path, buf)
return d.disk.AppendFile(ctx, volume, path, buf)
}
func (d *naughtyDisk) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath string) error {
func (d *naughtyDisk) RenameData(ctx context.Context, srcVolume, srcPath, dataDir, dstVolume, dstPath string) error {
if err := d.calcError(); err != nil {
return err
}
return d.disk.RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath)
return d.disk.RenameData(ctx, srcVolume, srcPath, dataDir, dstVolume, dstPath)
}
func (d *naughtyDisk) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error {
func (d *naughtyDisk) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error {
if err := d.calcError(); err != nil {
return err
}
return d.disk.RenameFile(srcVolume, srcPath, dstVolume, dstPath)
return d.disk.RenameFile(ctx, srcVolume, srcPath, dstVolume, dstPath)
}
func (d *naughtyDisk) CheckParts(volume string, path string, fi FileInfo) (err error) {
func (d *naughtyDisk) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (err error) {
if err := d.calcError(); err != nil {
return err
}
return d.disk.CheckParts(volume, path, fi)
return d.disk.CheckParts(ctx, volume, path, fi)
}
func (d *naughtyDisk) CheckFile(volume string, path string) (err error) {
func (d *naughtyDisk) CheckFile(ctx context.Context, volume string, path string) (err error) {
if err := d.calcError(); err != nil {
return err
}
return d.disk.CheckFile(volume, path)
return d.disk.CheckFile(ctx, volume, path)
}
func (d *naughtyDisk) DeleteFile(volume string, path string) (err error) {
func (d *naughtyDisk) DeleteFile(ctx context.Context, volume string, path string) (err error) {
if err := d.calcError(); err != nil {
return err
}
return d.disk.DeleteFile(volume, path)
return d.disk.DeleteFile(ctx, volume, path)
}
func (d *naughtyDisk) DeleteVersions(volume string, versions []FileInfo) []error {
func (d *naughtyDisk) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) []error {
if err := d.calcError(); err != nil {
errs := make([]error, len(versions))
for i := range errs {
@ -234,47 +234,47 @@ func (d *naughtyDisk) DeleteVersions(volume string, versions []FileInfo) []error
}
return errs
}
return d.disk.DeleteVersions(volume, versions)
return d.disk.DeleteVersions(ctx, volume, versions)
}
func (d *naughtyDisk) WriteMetadata(volume, path string, fi FileInfo) (err error) {
func (d *naughtyDisk) WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) (err error) {
if err := d.calcError(); err != nil {
return err
}
return d.disk.WriteMetadata(volume, path, fi)
return d.disk.WriteMetadata(ctx, volume, path, fi)
}
func (d *naughtyDisk) DeleteVersion(volume string, path string, fi FileInfo) (err error) {
func (d *naughtyDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo) (err error) {
if err := d.calcError(); err != nil {
return err
}
return d.disk.DeleteVersion(volume, path, fi)
return d.disk.DeleteVersion(ctx, volume, path, fi)
}
func (d *naughtyDisk) ReadVersion(volume string, path string, versionID string) (fi FileInfo, err error) {
func (d *naughtyDisk) ReadVersion(ctx context.Context, volume, path, versionID string) (fi FileInfo, err error) {
if err := d.calcError(); err != nil {
return FileInfo{}, err
}
return d.disk.ReadVersion(volume, path, versionID)
return d.disk.ReadVersion(ctx, volume, path, versionID)
}
func (d *naughtyDisk) WriteAll(volume string, path string, reader io.Reader) (err error) {
func (d *naughtyDisk) WriteAll(ctx context.Context, volume string, path string, reader io.Reader) (err error) {
if err := d.calcError(); err != nil {
return err
}
return d.disk.WriteAll(volume, path, reader)
return d.disk.WriteAll(ctx, volume, path, reader)
}
func (d *naughtyDisk) ReadAll(volume string, path string) (buf []byte, err error) {
func (d *naughtyDisk) ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) {
if err := d.calcError(); err != nil {
return nil, err
}
return d.disk.ReadAll(volume, path)
return d.disk.ReadAll(ctx, volume, path)
}
func (d *naughtyDisk) VerifyFile(volume, path string, fi FileInfo) error {
func (d *naughtyDisk) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error {
if err := d.calcError(); err != nil {
return err
}
return d.disk.VerifyFile(volume, path, fi)
return d.disk.VerifyFile(ctx, volume, path, fi)
}

6
cmd/object-api-common.go
View File

@ -99,7 +99,7 @@ func cleanupDir(ctx context.Context, storage StorageAPI, volume, dirPath string)
delFunc = func(entryPath string) error {
if !HasSuffix(entryPath, SlashSeparator) {
// Delete the file entry.
err := storage.DeleteFile(volume, entryPath)
err := storage.DeleteFile(ctx, volume, entryPath)
if err != errDiskNotFound && err != errUnformattedDisk {
logger.LogIf(ctx, err)
}
@ -107,7 +107,7 @@ func cleanupDir(ctx context.Context, storage StorageAPI, volume, dirPath string)
}
// If it's a directory, list and call delFunc() for each entry.
entries, err := storage.ListDir(volume, entryPath, -1)
entries, err := storage.ListDir(ctx, volume, entryPath, -1)
// If entryPath prefix never existed, safe to ignore.
if err == errFileNotFound {
return nil
@ -120,7 +120,7 @@ func cleanupDir(ctx context.Context, storage StorageAPI, volume, dirPath string)
// Entry path is empty, just delete it.
if len(entries) == 0 {
err = storage.DeleteFile(volume, entryPath)
err = storage.DeleteFile(ctx, volume, entryPath)
if err != errDiskNotFound && err != errUnformattedDisk {
logger.LogIf(ctx, err)
}

4
cmd/peer-rest-client.go
View File

@ -62,7 +62,7 @@ func (client *peerRESTClient) callWithContext(ctx context.Context, method string
values = make(url.Values)
}
respBody, err = client.restClient.CallWithContext(ctx, method, values, body, length)
respBody, err = client.restClient.Call(ctx, method, values, body, length)
if err == nil {
return respBody, nil
}
@ -882,7 +882,7 @@ func newPeerRESTClient(peer *xnet.Host) *peerRESTClient {
ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
// Instantiate a new rest client for healthcheck
// to avoid recursive healthCheckFn()
respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).CallWithContext(ctx, peerRESTMethodHealth, nil, nil, -1)
respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).Call(ctx, peerRESTMethodHealth, nil, nil, -1)
xhttp.DrainBody(respBody)
cancel()
var ne *rest.NetworkError

10
cmd/rest/client.go
View File

@ -95,8 +95,8 @@ func (e restError) Timeout() bool {
return true
}
// CallWithContext - make a REST call with context.
func (c *Client) CallWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (reply io.ReadCloser, err error) {
// Call - make a REST call with context.
func (c *Client) Call(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (reply io.ReadCloser, err error) {
if !c.IsOnline() {
return nil, &NetworkError{Err: &url.Error{Op: method, URL: c.url.String(), Err: restError("remote server offline")}}
}
@ -153,12 +153,6 @@ func (c *Client) CallWithContext(ctx context.Context, method string, values url.
return resp.Body, nil
}
// Call - make a REST call.
func (c *Client) Call(method string, values url.Values, body io.Reader, length int64) (reply io.ReadCloser, err error) {
ctx := context.Background()
return c.CallWithContext(ctx, method, values, body, length)
}
// Close closes all idle connections of the underlying http client
func (c *Client) Close() {
atomic.StoreInt32(&c.connected, closed)

56
cmd/storage-interface.go
View File

@ -35,47 +35,47 @@ type StorageAPI interface {
GetDiskID() (string, error)
SetDiskID(id string)
DiskInfo() (info DiskInfo, err error)
DiskInfo(ctx context.Context) (info DiskInfo, err error)
CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache) (dataUsageCache, error)
// Volume operations.
MakeVol(volume string) (err error)
MakeVolBulk(volumes ...string) (err error)
ListVols() (vols []VolInfo, err error)
StatVol(volume string) (vol VolInfo, err error)
DeleteVol(volume string, forceDelete bool) (err error)
MakeVol(ctx context.Context, volume string) (err error)
MakeVolBulk(ctx context.Context, volumes ...string) (err error)
ListVols(ctx context.Context) (vols []VolInfo, err error)
StatVol(ctx context.Context, volume string) (vol VolInfo, err error)
DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error)
// WalkVersions in sorted order directly on disk.
WalkVersions(volume, dirPath string, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error)
WalkVersions(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error)
// Walk in sorted order directly on disk.
Walk(volume, dirPath string, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error)
Walk(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error)
// Walk in sorted order directly on disk.
WalkSplunk(volume, dirPath string, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error)
WalkSplunk(ctx context.Context, volume, dirPath, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error)
// Metadata operations
DeleteVersion(volume, path string, fi FileInfo) error
DeleteVersions(volume string, versions []FileInfo) []error
WriteMetadata(volume, path string, fi FileInfo) error
ReadVersion(volume, path, versionID string) (FileInfo, error)
RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath string) error
DeleteVersion(ctx context.Context, volume, path string, fi FileInfo) error
DeleteVersions(ctx context.Context, volume string, versions []FileInfo) []error
WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) error
ReadVersion(ctx context.Context, volume, path, versionID string) (FileInfo, error)
RenameData(ctx context.Context, srcVolume, srcPath, dataDir, dstVolume, dstPath string) error
// File operations.
ListDir(volume, dirPath string, count int) ([]string, error)
ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error)
AppendFile(volume string, path string, buf []byte) (err error)
CreateFile(volume, path string, size int64, reader io.Reader) error
ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error)
RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error
CheckParts(volume string, path string, fi FileInfo) error
CheckFile(volume string, path string) (err error)
DeleteFile(volume string, path string) (err error)
VerifyFile(volume, path string, fi FileInfo) error
ListDir(ctx context.Context, volume, dirPath string, count int) ([]string, error)
ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error)
AppendFile(ctx context.Context, volume string, path string, buf []byte) (err error)
CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error
ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error)
RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error
CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error
CheckFile(ctx context.Context, volume string, path string) (err error)
DeleteFile(ctx context.Context, volume string, path string) (err error)
VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error
// Write all data, syncs the data to disk.
WriteAll(volume string, path string, reader io.Reader) (err error)
WriteAll(ctx context.Context, volume string, path string, reader io.Reader) (err error)
// Read all.
ReadAll(volume string, path string) (buf []byte, err error)
ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error)
}
// storageReader is an io.Reader view of a disk
@ -86,7 +86,7 @@ type storageReader struct {
}
func (r *storageReader) Read(p []byte) (n int, err error) {
nn, err := r.storage.ReadFile(r.volume, r.path, r.offset, p, nil)
nn, err := r.storage.ReadFile(context.TODO(), r.volume, r.path, r.offset, p, nil)
r.offset += nn
n = int(nn)
@ -103,7 +103,7 @@ type storageWriter struct {
}
func (w *storageWriter) Write(p []byte) (n int, err error) {
err = w.storage.AppendFile(w.volume, w.path, p)
err = w.storage.AppendFile(context.TODO(), w.volume, w.path, p)
if err == nil {
n = len(p)
}

130
cmd/storage-rest-client.go
View File

@ -119,7 +119,7 @@ type storageRESTClient struct {
// Wrapper to restClient.Call to handle network errors, in case of network error the connection is makred disconnected
// permanently. The only way to restore the storage connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
// after verifying format.json
func (client *storageRESTClient) call(method string, values url.Values, body io.Reader, length int64) (io.ReadCloser, error) {
func (client *storageRESTClient) call(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (io.ReadCloser, error) {
if !client.IsOnline() {
return nil, errDiskNotFound
}
@ -127,7 +127,7 @@ func (client *storageRESTClient) call(method string, values url.Values, body io.
values = make(url.Values)
}
values.Set(storageRESTDiskID, client.diskID)
respBody, err := client.restClient.Call(method, values, body, length)
respBody, err := client.restClient.Call(ctx, method, values, body, length)
if err == nil {
return respBody, nil
}
@ -157,9 +157,7 @@ func (client *storageRESTClient) Hostname() string {
func (client *storageRESTClient) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache) (dataUsageCache, error) {
b := cache.serialize()
respBody, err := client.call(storageRESTMethodCrawlAndGetDataUsage,
url.Values{},
bytes.NewBuffer(b), int64(len(b)))
respBody, err := client.call(ctx, storageRESTMethodCrawlAndGetDataUsage, url.Values{}, bytes.NewBuffer(b), int64(len(b)))
defer http.DrainBody(respBody)
if err != nil {
return cache, err
@ -185,8 +183,8 @@ func (client *storageRESTClient) SetDiskID(id string) {
}
// DiskInfo - fetch disk information for a remote disk.
func (client *storageRESTClient) DiskInfo() (info DiskInfo, err error) {
respBody, err := client.call(storageRESTMethodDiskInfo, nil, nil, -1)
func (client *storageRESTClient) DiskInfo(ctx context.Context) (info DiskInfo, err error) {
respBody, err := client.call(ctx, storageRESTMethodDiskInfo, nil, nil, -1)
if err != nil {
return
}
@ -202,81 +200,81 @@ func (client *storageRESTClient) DiskInfo() (info DiskInfo, err error) {
}
// MakeVolBulk - create multiple volumes in a bulk operation.
func (client *storageRESTClient) MakeVolBulk(volumes ...string) (err error) {
func (client *storageRESTClient) MakeVolBulk(ctx context.Context, volumes ...string) (err error) {
values := make(url.Values)
values.Set(storageRESTVolumes, strings.Join(volumes, ","))
respBody, err := client.call(storageRESTMethodMakeVolBulk, values, nil, -1)
respBody, err := client.call(ctx, storageRESTMethodMakeVolBulk, values, nil, -1)
defer http.DrainBody(respBody)
return err
}
// MakeVol - create a volume on a remote disk.
func (client *storageRESTClient) MakeVol(volume string) (err error) {
func (client *storageRESTClient) MakeVol(ctx context.Context, volume string) (err error) {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
respBody, err := client.call(storageRESTMethodMakeVol, values, nil, -1)
respBody, err := client.call(ctx, storageRESTMethodMakeVol, values, nil, -1)
defer http.DrainBody(respBody)
return err
}
// ListVols - List all volumes on a remote disk.
func (client *storageRESTClient) ListVols() (volinfo []VolInfo, err error) {
respBody, err := client.call(storageRESTMethodListVols, nil, nil, -1)
func (client *storageRESTClient) ListVols(ctx context.Context) (vols []VolInfo, err error) {
respBody, err := client.call(ctx, storageRESTMethodListVols, nil, nil, -1)
if err != nil {
return
}
defer http.DrainBody(respBody)
err = gob.NewDecoder(respBody).Decode(&volinfo)
return volinfo, err
err = gob.NewDecoder(respBody).Decode(&vols)
return vols, err
}
// StatVol - get volume info over the network.
func (client *storageRESTClient) StatVol(volume string) (volInfo VolInfo, err error) {
func (client *storageRESTClient) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
respBody, err := client.call(storageRESTMethodStatVol, values, nil, -1)
respBody, err := client.call(ctx, storageRESTMethodStatVol, values, nil, -1)
if err != nil {
return
}
defer http.DrainBody(respBody)
err = gob.NewDecoder(respBody).Decode(&volInfo)
return volInfo, err
err = gob.NewDecoder(respBody).Decode(&vol)
return vol, err
}
// DeleteVol - Deletes a volume over the network.
func (client *storageRESTClient) DeleteVol(volume string, forceDelete bool) (err error) {
func (client *storageRESTClient) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
if forceDelete {
values.Set(storageRESTForceDelete, "true")
}
respBody, err := client.call(storageRESTMethodDeleteVol, values, nil, -1)
respBody, err := client.call(ctx, storageRESTMethodDeleteVol, values, nil, -1)
defer http.DrainBody(respBody)
return err
}
// AppendFile - append to a file.
func (client *storageRESTClient) AppendFile(volume, path string, buffer []byte) error {
func (client *storageRESTClient) AppendFile(ctx context.Context, volume string, path string, buf []byte) error {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path)
reader := bytes.NewReader(buffer)
respBody, err := client.call(storageRESTMethodAppendFile, values, reader, -1)
reader := bytes.NewReader(buf)
respBody, err := client.call(ctx, storageRESTMethodAppendFile, values, reader, -1)
defer http.DrainBody(respBody)
return err
}
func (client *storageRESTClient) CreateFile(volume, path string, length int64, r io.Reader) error {
func (client *storageRESTClient) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path)
values.Set(storageRESTLength, strconv.Itoa(int(length)))
respBody, err := client.call(storageRESTMethodCreateFile, values, ioutil.NopCloser(r), length)
values.Set(storageRESTLength, strconv.Itoa(int(size)))
respBody, err := client.call(ctx, storageRESTMethodCreateFile, values, ioutil.NopCloser(reader), size)
defer http.DrainBody(respBody)
return err
}
func (client *storageRESTClient) WriteMetadata(volume, path string, fi FileInfo) error {
func (client *storageRESTClient) WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) error {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path)
@ -286,12 +284,12 @@ func (client *storageRESTClient) WriteMetadata(volume, path string, fi FileInfo)
return err
}
respBody, err := client.call(storageRESTMethodWriteMetadata, values, &reader, -1)
respBody, err := client.call(ctx, storageRESTMethodWriteMetadata, values, &reader, -1)
defer http.DrainBody(respBody)
return err
}
func (client *storageRESTClient) DeleteVersion(volume, path string, fi FileInfo) error {
func (client *storageRESTClient) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo) error {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path)
@ -301,33 +299,33 @@ func (client *storageRESTClient) DeleteVersion(volume, path string, fi FileInfo)
return err
}
respBody, err := client.call(storageRESTMethodDeleteVersion, values, &buffer, -1)
respBody, err := client.call(ctx, storageRESTMethodDeleteVersion, values, &buffer, -1)
defer http.DrainBody(respBody)
return err
}
// WriteAll - write all data to a file.
func (client *storageRESTClient) WriteAll(volume, path string, reader io.Reader) error {
func (client *storageRESTClient) WriteAll(ctx context.Context, volume string, path string, reader io.Reader) error {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path)
respBody, err := client.call(storageRESTMethodWriteAll, values, reader, -1)
respBody, err := client.call(ctx, storageRESTMethodWriteAll, values, reader, -1)
defer http.DrainBody(respBody)
return err
}
// CheckFile - stat a file metadata.
func (client *storageRESTClient) CheckFile(volume, path string) error {
func (client *storageRESTClient) CheckFile(ctx context.Context, volume string, path string) error {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path)
respBody, err := client.call(storageRESTMethodCheckFile, values, nil, -1)
respBody, err := client.call(ctx, storageRESTMethodCheckFile, values, nil, -1)
defer http.DrainBody(respBody)
return err
}
// CheckParts - stat all file parts.
func (client *storageRESTClient) CheckParts(volume, path string, fi FileInfo) error {
func (client *storageRESTClient) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path)
@ -338,32 +336,32 @@ func (client *storageRESTClient) CheckParts(volume, path string, fi FileInfo) er
return err
}
respBody, err := client.call(storageRESTMethodCheckParts, values, &reader, -1)
respBody, err := client.call(ctx, storageRESTMethodCheckParts, values, &reader, -1)
defer http.DrainBody(respBody)
return err
}
// RenameData - rename source path to destination path atomically, metadata and data file.
func (client *storageRESTClient) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath string) (err error) {
func (client *storageRESTClient) RenameData(ctx context.Context, srcVolume, srcPath, dataDir, dstVolume, dstPath string) (err error) {
values := make(url.Values)
values.Set(storageRESTSrcVolume, srcVolume)
values.Set(storageRESTSrcPath, srcPath)
values.Set(storageRESTDataDir, dataDir)
values.Set(storageRESTDstVolume, dstVolume)
values.Set(storageRESTDstPath, dstPath)
respBody, err := client.call(storageRESTMethodRenameData, values, nil, -1)
respBody, err := client.call(ctx, storageRESTMethodRenameData, values, nil, -1)
defer http.DrainBody(respBody)
return err
}
func (client *storageRESTClient) ReadVersion(volume, path, versionID string) (fi FileInfo, err error) {
func (client *storageRESTClient) ReadVersion(ctx context.Context, volume, path, versionID string) (fi FileInfo, err error) {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path)
values.Set(storageRESTVersionID, versionID)
respBody, err := client.call(storageRESTMethodReadVersion, values, nil, -1)
respBody, err := client.call(ctx, storageRESTMethodReadVersion, values, nil, -1)
if err != nil {
return fi, err
}
@ -374,11 +372,11 @@ func (client *storageRESTClient) ReadVersion(volume, path, versionID string) (fi
}
// ReadAll - reads all contents of a file.
func (client *storageRESTClient) ReadAll(volume, path string) ([]byte, error) {
func (client *storageRESTClient) ReadAll(ctx context.Context, volume string, path string) ([]byte, error) {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path)
respBody, err := client.call(storageRESTMethodReadAll, values, nil, -1)
respBody, err := client.call(ctx, storageRESTMethodReadAll, values, nil, -1)
if err != nil {
return nil, err
}
@ -387,13 +385,13 @@ func (client *storageRESTClient) ReadAll(volume, path string) ([]byte, error) {
}
// ReadFileStream - returns a reader for the requested file.
func (client *storageRESTClient) ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error) {
func (client *storageRESTClient) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path)
values.Set(storageRESTOffset, strconv.Itoa(int(offset)))
values.Set(storageRESTLength, strconv.Itoa(int(length)))
respBody, err := client.call(storageRESTMethodReadFileStream, values, nil, -1)
respBody, err := client.call(ctx, storageRESTMethodReadFileStream, values, nil, -1)
if err != nil {
return nil, err
}
@ -401,12 +399,12 @@ func (client *storageRESTClient) ReadFileStream(volume, path string, offset, len
}
// ReadFile - reads section of a file.
func (client *storageRESTClient) ReadFile(volume, path string, offset int64, buffer []byte, verifier *BitrotVerifier) (int64, error) {
func (client *storageRESTClient) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (int64, error) {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path)
values.Set(storageRESTOffset, strconv.Itoa(int(offset)))
values.Set(storageRESTLength, strconv.Itoa(len(buffer)))
values.Set(storageRESTLength, strconv.Itoa(len(buf)))
if verifier != nil {
values.Set(storageRESTBitrotAlgo, verifier.algorithm.String())
values.Set(storageRESTBitrotHash, hex.EncodeToString(verifier.sum))
@ -414,21 +412,21 @@ func (client *storageRESTClient) ReadFile(volume, path string, offset int64, buf
values.Set(storageRESTBitrotAlgo, "")
values.Set(storageRESTBitrotHash, "")
}
respBody, err := client.call(storageRESTMethodReadFile, values, nil, -1)
respBody, err := client.call(ctx, storageRESTMethodReadFile, values, nil, -1)
if err != nil {
return 0, err
}
defer http.DrainBody(respBody)
n, err := io.ReadFull(respBody, buffer)
n, err := io.ReadFull(respBody, buf)
return int64(n), err
}
func (client *storageRESTClient) WalkSplunk(volume, dirPath, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error) {
func (client *storageRESTClient) WalkSplunk(ctx context.Context, volume, dirPath, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error) {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
values.Set(storageRESTDirPath, dirPath)
values.Set(storageRESTMarkerPath, marker)
respBody, err := client.call(storageRESTMethodWalkSplunk, values, nil, -1)
respBody, err := client.call(ctx, storageRESTMethodWalkSplunk, values, nil, -1)
if err != nil {
return nil, err
}
@ -457,13 +455,13 @@ func (client *storageRESTClient) WalkSplunk(volume, dirPath, marker string, endW
return ch, nil
}
func (client *storageRESTClient) WalkVersions(volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error) {
func (client *storageRESTClient) WalkVersions(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error) {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
values.Set(storageRESTDirPath, dirPath)
values.Set(storageRESTMarkerPath, marker)
values.Set(storageRESTRecursive, strconv.FormatBool(recursive))
respBody, err := client.call(storageRESTMethodWalkVersions, values, nil, -1)
respBody, err := client.call(ctx, storageRESTMethodWalkVersions, values, nil, -1)
if err != nil {
return nil, err
}
@ -494,13 +492,13 @@ func (client *storageRESTClient) WalkVersions(volume, dirPath, marker string, re
return ch, nil
}
func (client *storageRESTClient) Walk(volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) {
func (client *storageRESTClient) Walk(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
values.Set(storageRESTDirPath, dirPath)
values.Set(storageRESTMarkerPath, marker)
values.Set(storageRESTRecursive, strconv.FormatBool(recursive))
respBody, err := client.call(storageRESTMethodWalk, values, nil, -1)
respBody, err := client.call(ctx, storageRESTMethodWalk, values, nil, -1)
if err != nil {
return nil, err
}
@ -530,12 +528,12 @@ func (client *storageRESTClient) Walk(volume, dirPath, marker string, recursive
}
// ListDir - lists a directory.
func (client *storageRESTClient) ListDir(volume, dirPath string, count int) (entries []string, err error) {
func (client *storageRESTClient) ListDir(ctx context.Context, volume, dirPath string, count int) (entries []string, err error) {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
values.Set(storageRESTDirPath, dirPath)
values.Set(storageRESTCount, strconv.Itoa(count))
respBody, err := client.call(storageRESTMethodListDir, values, nil, -1)
respBody, err := client.call(ctx, storageRESTMethodListDir, values, nil, -1)
if err != nil {
return nil, err
}
@ -545,17 +543,17 @@ func (client *storageRESTClient) ListDir(volume, dirPath string, count int) (ent
}
// DeleteFile - deletes a file.
func (client *storageRESTClient) DeleteFile(volume, path string) error {
func (client *storageRESTClient) DeleteFile(ctx context.Context, volume string, path string) error {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path)
respBody, err := client.call(storageRESTMethodDeleteFile, values, nil, -1)
respBody, err := client.call(ctx, storageRESTMethodDeleteFile, values, nil, -1)
defer http.DrainBody(respBody)
return err
}
// DeleteVersions - deletes list of specified versions if present
func (client *storageRESTClient) DeleteVersions(volume string, versions []FileInfo) (errs []error) {
func (client *storageRESTClient) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) (errs []error) {
if len(versions) == 0 {
return errs
}
@ -572,7 +570,7 @@ func (client *storageRESTClient) DeleteVersions(volume string, versions []FileIn
errs = make([]error, len(versions))
respBody, err := client.call(storageRESTMethodDeleteVersions, values, &buffer, -1)
respBody, err := client.call(ctx, storageRESTMethodDeleteVersions, values, &buffer, -1)
defer http.DrainBody(respBody)
if err != nil {
for i := range errs {
@ -605,18 +603,18 @@ func (client *storageRESTClient) DeleteVersions(volume string, versions []FileIn
}
// RenameFile - renames a file.
func (client *storageRESTClient) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err error) {
func (client *storageRESTClient) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) (err error) {
values := make(url.Values)
values.Set(storageRESTSrcVolume, srcVolume)
values.Set(storageRESTSrcPath, srcPath)
values.Set(storageRESTDstVolume, dstVolume)
values.Set(storageRESTDstPath, dstPath)
respBody, err := client.call(storageRESTMethodRenameFile, values, nil, -1)
respBody, err := client.call(ctx, storageRESTMethodRenameFile, values, nil, -1)
defer http.DrainBody(respBody)
return err
}
func (client *storageRESTClient) VerifyFile(volume, path string, fi FileInfo) error {
func (client *storageRESTClient) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path)
@ -626,7 +624,7 @@ func (client *storageRESTClient) VerifyFile(volume, path string, fi FileInfo) er
return err
}
respBody, err := client.call(storageRESTMethodVerifyFile, values, &reader, -1)
respBody, err := client.call(ctx, storageRESTMethodVerifyFile, values, &reader, -1)
defer http.DrainBody(respBody)
if err != nil {
return err
@ -674,7 +672,7 @@ func newStorageRESTClient(endpoint Endpoint) *storageRESTClient {
ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
// Instantiate a new rest client for healthcheck
// to avoid recursive healthCheckFn()
respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).CallWithContext(ctx, storageRESTMethodHealth, nil, nil, -1)
respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).Call(ctx, storageRESTMethodHealth, nil, nil, -1)
xhttp.DrainBody(respBody)
cancel()
return !errors.Is(err, context.DeadlineExceeded) && toStorageErr(err) != errDiskNotFound

52
cmd/storage-rest-server.go
View File

@ -138,7 +138,7 @@ func (s *storageRESTServer) DiskInfoHandler(w http.ResponseWriter, r *http.Reque
if !s.IsValid(w, r) {
return
}
info, err := s.storage.DiskInfo()
info, err := s.storage.DiskInfo(r.Context())
if err != nil {
info.Error = err.Error()
}
@ -179,7 +179,7 @@ func (s *storageRESTServer) MakeVolHandler(w http.ResponseWriter, r *http.Reques
}
vars := mux.Vars(r)
volume := vars[storageRESTVolume]
err := s.storage.MakeVol(volume)
err := s.storage.MakeVol(r.Context(), volume)
if err != nil {
s.writeErrorResponse(w, err)
}
@ -192,7 +192,7 @@ func (s *storageRESTServer) MakeVolBulkHandler(w http.ResponseWriter, r *http.Re
}
vars := mux.Vars(r)
volumes := strings.Split(vars[storageRESTVolumes], ",")
err := s.storage.MakeVolBulk(volumes...)
err := s.storage.MakeVolBulk(r.Context(), volumes...)
if err != nil {
s.writeErrorResponse(w, err)
}
@ -203,7 +203,7 @@ func (s *storageRESTServer) ListVolsHandler(w http.ResponseWriter, r *http.Reque
if !s.IsValid(w, r) {
return
}
infos, err := s.storage.ListVols()
infos, err := s.storage.ListVols(r.Context())
if err != nil {
s.writeErrorResponse(w, err)
return
@ -219,7 +219,7 @@ func (s *storageRESTServer) StatVolHandler(w http.ResponseWriter, r *http.Reques
}
vars := mux.Vars(r)
volume := vars[storageRESTVolume]
info, err := s.storage.StatVol(volume)
info, err := s.storage.StatVol(r.Context(), volume)
if err != nil {
s.writeErrorResponse(w, err)
return
@ -236,7 +236,7 @@ func (s *storageRESTServer) DeleteVolHandler(w http.ResponseWriter, r *http.Requ
vars := mux.Vars(r)
volume := vars[storageRESTVolume]
forceDelete := vars[storageRESTForceDelete] == "true"
err := s.storage.DeleteVol(volume, forceDelete)
err := s.storage.DeleteVol(r.Context(), volume, forceDelete)
if err != nil {
s.writeErrorResponse(w, err)
}
@ -257,7 +257,7 @@ func (s *storageRESTServer) AppendFileHandler(w http.ResponseWriter, r *http.Req
s.writeErrorResponse(w, err)
return
}
err = s.storage.AppendFile(volume, filePath, buf)
err = s.storage.AppendFile(r.Context(), volume, filePath, buf)
if err != nil {
s.writeErrorResponse(w, err)
}
@ -278,7 +278,7 @@ func (s *storageRESTServer) CreateFileHandler(w http.ResponseWriter, r *http.Req
s.writeErrorResponse(w, err)
return
}
err = s.storage.CreateFile(volume, filePath, int64(fileSize), r.Body)
err = s.storage.CreateFile(r.Context(), volume, filePath, int64(fileSize), r.Body)
if err != nil {
s.writeErrorResponse(w, err)
}
@ -304,7 +304,7 @@ func (s *storageRESTServer) DeleteVersionHandler(w http.ResponseWriter, r *http.
return
}
err := s.storage.DeleteVersion(volume, filePath, fi)
err := s.storage.DeleteVersion(r.Context(), volume, filePath, fi)
if err != nil {
s.writeErrorResponse(w, err)
}
@ -320,7 +320,7 @@ func (s *storageRESTServer) ReadVersionHandler(w http.ResponseWriter, r *http.Re
filePath := vars[storageRESTFilePath]
versionID := vars[storageRESTVersionID]
fi, err := s.storage.ReadVersion(volume, filePath, versionID)
fi, err := s.storage.ReadVersion(r.Context(), volume, filePath, versionID)
if err != nil {
s.writeErrorResponse(w, err)
return
@ -351,7 +351,7 @@ func (s *storageRESTServer) WriteMetadataHandler(w http.ResponseWriter, r *http.
return
}
err = s.storage.WriteMetadata(volume, filePath, fi)
err = s.storage.WriteMetadata(r.Context(), volume, filePath, fi)
if err != nil {
s.writeErrorResponse(w, err)
}
@ -371,7 +371,7 @@ func (s *storageRESTServer) WriteAllHandler(w http.ResponseWriter, r *http.Reque
return
}
err := s.storage.WriteAll(volume, filePath, io.LimitReader(r.Body, r.ContentLength))
err := s.storage.WriteAll(r.Context(), volume, filePath, io.LimitReader(r.Body, r.ContentLength))
if err != nil {
s.writeErrorResponse(w, err)
}
@ -397,7 +397,7 @@ func (s *storageRESTServer) CheckPartsHandler(w http.ResponseWriter, r *http.Req
return
}
if err := s.storage.CheckParts(volume, filePath, fi); err != nil {
if err := s.storage.CheckParts(r.Context(), volume, filePath, fi); err != nil {
s.writeErrorResponse(w, err)
}
}
@ -411,7 +411,7 @@ func (s *storageRESTServer) CheckFileHandler(w http.ResponseWriter, r *http.Requ
volume := vars[storageRESTVolume]
filePath := vars[storageRESTFilePath]
if err := s.storage.CheckFile(volume, filePath); err != nil {
if err := s.storage.CheckFile(r.Context(), volume, filePath); err != nil {
s.writeErrorResponse(w, err)
}
}
@ -425,7 +425,7 @@ func (s *storageRESTServer) ReadAllHandler(w http.ResponseWriter, r *http.Reques
volume := vars[storageRESTVolume]
filePath := vars[storageRESTFilePath]
buf, err := s.storage.ReadAll(volume, filePath)
buf, err := s.storage.ReadAll(r.Context(), volume, filePath)
if err != nil {
s.writeErrorResponse(w, err)
return
@ -469,7 +469,7 @@ func (s *storageRESTServer) ReadFileHandler(w http.ResponseWriter, r *http.Reque
verifier = NewBitrotVerifier(BitrotAlgorithmFromString(vars[storageRESTBitrotAlgo]), hash)
}
buf := make([]byte, length)
_, err = s.storage.ReadFile(volume, filePath, int64(offset), buf, verifier)
_, err = s.storage.ReadFile(r.Context(), volume, filePath, int64(offset), buf, verifier)
if err != nil {
s.writeErrorResponse(w, err)
return
@ -498,7 +498,7 @@ func (s *storageRESTServer) ReadFileStreamHandler(w http.ResponseWriter, r *http
return
}
rc, err := s.storage.ReadFileStream(volume, filePath, int64(offset), int64(length))
rc, err := s.storage.ReadFileStream(r.Context(), volume, filePath, int64(offset), int64(length))
if err != nil {
s.writeErrorResponse(w, err)
return
@ -524,7 +524,7 @@ func (s *storageRESTServer) WalkSplunkHandler(w http.ResponseWriter, r *http.Req
setEventStreamHeaders(w)
encoder := gob.NewEncoder(w)
fch, err := s.storage.WalkSplunk(volume, dirPath, markerPath, r.Context().Done())
fch, err := s.storage.WalkSplunk(r.Context(), volume, dirPath, markerPath, r.Context().Done())
if err != nil {
s.writeErrorResponse(w, err)
return
@ -552,7 +552,7 @@ func (s *storageRESTServer) WalkVersionsHandler(w http.ResponseWriter, r *http.R
setEventStreamHeaders(w)
encoder := gob.NewEncoder(w)
fch, err := s.storage.WalkVersions(volume, dirPath, markerPath, recursive, r.Context().Done())
fch, err := s.storage.WalkVersions(r.Context(), volume, dirPath, markerPath, recursive, r.Context().Done())
if err != nil {
s.writeErrorResponse(w, err)
return
@ -580,7 +580,7 @@ func (s *storageRESTServer) WalkHandler(w http.ResponseWriter, r *http.Request)
setEventStreamHeaders(w)
encoder := gob.NewEncoder(w)
fch, err := s.storage.Walk(volume, dirPath, markerPath, recursive, r.Context().Done())
fch, err := s.storage.Walk(r.Context(), volume, dirPath, markerPath, recursive, r.Context().Done())
if err != nil {
s.writeErrorResponse(w, err)
return
@ -604,7 +604,7 @@ func (s *storageRESTServer) ListDirHandler(w http.ResponseWriter, r *http.Reques
return
}
entries, err := s.storage.ListDir(volume, dirPath, count)
entries, err := s.storage.ListDir(r.Context(), volume, dirPath, count)
if err != nil {
s.writeErrorResponse(w, err)
return
@ -622,7 +622,7 @@ func (s *storageRESTServer) DeleteFileHandler(w http.ResponseWriter, r *http.Req
volume := vars[storageRESTVolume]
filePath := vars[storageRESTFilePath]
err := s.storage.DeleteFile(volume, filePath)
err := s.storage.DeleteFile(r.Context(), volume, filePath)
if err != nil {
s.writeErrorResponse(w, err)
}
@ -663,7 +663,7 @@ func (s *storageRESTServer) DeleteVersionsHandler(w http.ResponseWriter, r *http
setEventStreamHeaders(w)
encoder := gob.NewEncoder(w)
done := keepHTTPResponseAlive(w)
errs := s.storage.DeleteVersions(volume, versions)
errs := s.storage.DeleteVersions(r.Context(), volume, versions)
done(nil)
for idx := range versions {
if errs[idx] != nil {
@ -685,7 +685,7 @@ func (s *storageRESTServer) RenameDataHandler(w http.ResponseWriter, r *http.Req
dataDir := vars[storageRESTDataDir]
dstVolume := vars[storageRESTDstVolume]
dstFilePath := vars[storageRESTDstPath]
err := s.storage.RenameData(srcVolume, srcFilePath, dataDir, dstVolume, dstFilePath)
err := s.storage.RenameData(r.Context(), srcVolume, srcFilePath, dataDir, dstVolume, dstFilePath)
if err != nil {
s.writeErrorResponse(w, err)
}
@ -701,7 +701,7 @@ func (s *storageRESTServer) RenameFileHandler(w http.ResponseWriter, r *http.Req
srcFilePath := vars[storageRESTSrcPath]
dstVolume := vars[storageRESTDstVolume]
dstFilePath := vars[storageRESTDstPath]
err := s.storage.RenameFile(srcVolume, srcFilePath, dstVolume, dstFilePath)
err := s.storage.RenameFile(r.Context(), srcVolume, srcFilePath, dstVolume, dstFilePath)
if err != nil {
s.writeErrorResponse(w, err)
}
@ -809,7 +809,7 @@ func (s *storageRESTServer) VerifyFileHandler(w http.ResponseWriter, r *http.Req
setEventStreamHeaders(w)
encoder := gob.NewEncoder(w)
done := keepHTTPResponseAlive(w)
err = s.storage.VerifyFile(volume, filePath, fi)
err = s.storage.VerifyFile(r.Context(), volume, filePath, fi)
done(nil)
vresp := &VerifyFileResp{}
if err != nil {

61
cmd/storage-rest_test.go
View File

@ -17,6 +17,7 @@
package cmd
import (
"context"
"io/ioutil"
"net/http/httptest"
"os"
@ -43,7 +44,7 @@ func testStorageAPIDiskInfo(t *testing.T, storage StorageAPI) {
}
for i, testCase := range testCases {
_, err := storage.DiskInfo()
_, err := storage.DiskInfo(context.Background())
expectErr := (err != nil)
if expectErr != testCase.expectErr {
@ -66,7 +67,7 @@ func testStorageAPIMakeVol(t *testing.T, storage StorageAPI) {
}
for i, testCase := range testCases {
err := storage.MakeVol(testCase.volumeName)
err := storage.MakeVol(context.Background(), testCase.volumeName)
expectErr := (err != nil)
if expectErr != testCase.expectErr {
@ -87,13 +88,13 @@ func testStorageAPIListVols(t *testing.T, storage StorageAPI) {
for i, testCase := range testCases {
for _, volumeName := range testCase.volumeNames {
err := storage.MakeVol(volumeName)
err := storage.MakeVol(context.Background(), volumeName)
if err != nil {
t.Fatalf("unexpected error %v", err)
}
}
result, err := storage.ListVols()
result, err := storage.ListVols(context.Background())
expectErr := (err != nil)
if expectErr != testCase.expectErr {
@ -109,7 +110,7 @@ func testStorageAPIListVols(t *testing.T, storage StorageAPI) {
}
func testStorageAPIStatVol(t *testing.T, storage StorageAPI) {
err := storage.MakeVol("foo")
err := storage.MakeVol(context.Background(), "foo")
if err != nil {
t.Fatalf("unexpected error %v", err)
}
@ -124,7 +125,7 @@ func testStorageAPIStatVol(t *testing.T, storage StorageAPI) {
}
for i, testCase := range testCases {
result, err := storage.StatVol(testCase.volumeName)
result, err := storage.StatVol(context.Background(), testCase.volumeName)
expectErr := (err != nil)
if expectErr != testCase.expectErr {
@ -140,7 +141,7 @@ func testStorageAPIStatVol(t *testing.T, storage StorageAPI) {
}
func testStorageAPIDeleteVol(t *testing.T, storage StorageAPI) {
err := storage.MakeVol("foo")
err := storage.MakeVol(context.Background(), "foo")
if err != nil {
t.Fatalf("unexpected error %v", err)
}
@ -155,7 +156,7 @@ func testStorageAPIDeleteVol(t *testing.T, storage StorageAPI) {
}
for i, testCase := range testCases {
err := storage.DeleteVol(testCase.volumeName, false)
err := storage.DeleteVol(context.Background(), testCase.volumeName, false)
expectErr := (err != nil)
if expectErr != testCase.expectErr {
@ -165,11 +166,11 @@ func testStorageAPIDeleteVol(t *testing.T, storage StorageAPI) {
}
func testStorageAPICheckFile(t *testing.T, storage StorageAPI) {
err := storage.MakeVol("foo")
err := storage.MakeVol(context.Background(), "foo")
if err != nil {
t.Fatalf("unexpected error %v", err)
}
err = storage.AppendFile("foo", pathJoin("myobject", xlStorageFormatFile), []byte("foo"))
err = storage.AppendFile(context.Background(), "foo", pathJoin("myobject", xlStorageFormatFile), []byte("foo"))
if err != nil {
t.Fatalf("unexpected error %v", err)
}
@ -185,7 +186,7 @@ func testStorageAPICheckFile(t *testing.T, storage StorageAPI) {
}
for i, testCase := range testCases {
err := storage.CheckFile(testCase.volumeName, testCase.objectName)
err := storage.CheckFile(context.Background(), testCase.volumeName, testCase.objectName)
expectErr := (err != nil)
if expectErr != testCase.expectErr {
@ -195,11 +196,11 @@ func testStorageAPICheckFile(t *testing.T, storage StorageAPI) {
}
func testStorageAPIListDir(t *testing.T, storage StorageAPI) {
err := storage.MakeVol("foo")
err := storage.MakeVol(context.Background(), "foo")
if err != nil {
t.Fatalf("unexpected error %v", err)
}
err = storage.AppendFile("foo", "path/to/myobject", []byte("foo"))
err = storage.AppendFile(context.Background(), "foo", "path/to/myobject", []byte("foo"))
if err != nil {
t.Fatalf("unexpected error %v", err)
}
@ -216,7 +217,7 @@ func testStorageAPIListDir(t *testing.T, storage StorageAPI) {
}
for i, testCase := range testCases {
result, err := storage.ListDir(testCase.volumeName, testCase.prefix, -1)
result, err := storage.ListDir(context.Background(), testCase.volumeName, testCase.prefix, -1)
expectErr := (err != nil)
if expectErr != testCase.expectErr {
@ -232,11 +233,11 @@ func testStorageAPIListDir(t *testing.T, storage StorageAPI) {
}
func testStorageAPIReadAll(t *testing.T, storage StorageAPI) {
err := storage.MakeVol("foo")
err := storage.MakeVol(context.Background(), "foo")
if err != nil {
t.Fatalf("unexpected error %v", err)
}
err = storage.AppendFile("foo", "myobject", []byte("foo"))
err = storage.AppendFile(context.Background(), "foo", "myobject", []byte("foo"))
if err != nil {
t.Fatalf("unexpected error %v", err)
}
@ -253,7 +254,7 @@ func testStorageAPIReadAll(t *testing.T, storage StorageAPI) {
}
for i, testCase := range testCases {
result, err := storage.ReadAll(testCase.volumeName, testCase.objectName)
result, err := storage.ReadAll(context.Background(), testCase.volumeName, testCase.objectName)
expectErr := (err != nil)
if expectErr != testCase.expectErr {
@ -269,11 +270,11 @@ func testStorageAPIReadAll(t *testing.T, storage StorageAPI) {
}
func testStorageAPIReadFile(t *testing.T, storage StorageAPI) {
err := storage.MakeVol("foo")
err := storage.MakeVol(context.Background(), "foo")
if err != nil {
t.Fatalf("unexpected error %v", err)
}
err = storage.AppendFile("foo", "myobject", []byte("foo"))
err = storage.AppendFile(context.Background(), "foo", "myobject", []byte("foo"))
if err != nil {
t.Fatalf("unexpected error %v", err)
}
@ -294,7 +295,7 @@ func testStorageAPIReadFile(t *testing.T, storage StorageAPI) {
result := make([]byte, 100)
for i, testCase := range testCases {
result = result[testCase.offset:3]
_, err := storage.ReadFile(testCase.volumeName, testCase.objectName, testCase.offset, result, nil)
_, err := storage.ReadFile(context.Background(), testCase.volumeName, testCase.objectName, testCase.offset, result, nil)
expectErr := (err != nil)
if expectErr != testCase.expectErr {
@ -310,7 +311,7 @@ func testStorageAPIReadFile(t *testing.T, storage StorageAPI) {
}
func testStorageAPIAppendFile(t *testing.T, storage StorageAPI) {
err := storage.MakeVol("foo")
err := storage.MakeVol(context.Background(), "foo")
if err != nil {
t.Fatalf("unexpected error %v", err)
}
@ -328,7 +329,7 @@ func testStorageAPIAppendFile(t *testing.T, storage StorageAPI) {
}
for i, testCase := range testCases {
err := storage.AppendFile(testCase.volumeName, testCase.objectName, testCase.data)
err := storage.AppendFile(context.Background(), testCase.volumeName, testCase.objectName, testCase.data)
expectErr := (err != nil)
if expectErr != testCase.expectErr {
@ -338,12 +339,12 @@ func testStorageAPIAppendFile(t *testing.T, storage StorageAPI) {
}
func testStorageAPIDeleteFile(t *testing.T, storage StorageAPI) {
err := storage.MakeVol("foo")
err := storage.MakeVol(context.Background(), "foo")
if err != nil {
t.Fatalf("unexpected error %v", err)
}
err = storage.AppendFile("foo", "myobject", []byte("foo"))
err = storage.AppendFile(context.Background(), "foo", "myobject", []byte("foo"))
if err != nil {
t.Fatalf("unexpected error %v", err)
}
@ -361,7 +362,7 @@ func testStorageAPIDeleteFile(t *testing.T, storage StorageAPI) {
}
for i, testCase := range testCases {
err := storage.DeleteFile(testCase.volumeName, testCase.objectName)
err := storage.DeleteFile(context.Background(), testCase.volumeName, testCase.objectName)
expectErr := (err != nil)
if expectErr != testCase.expectErr {
@ -371,22 +372,22 @@ func testStorageAPIDeleteFile(t *testing.T, storage StorageAPI) {
}
func testStorageAPIRenameFile(t *testing.T, storage StorageAPI) {
err := storage.MakeVol("foo")
err := storage.MakeVol(context.Background(), "foo")
if err != nil {
t.Fatalf("unexpected error %v", err)
}
err = storage.MakeVol("bar")
err = storage.MakeVol(context.Background(), "bar")
if err != nil {
t.Fatalf("unexpected error %v", err)
}
err = storage.AppendFile("foo", "myobject", []byte("foo"))
err = storage.AppendFile(context.Background(), "foo", "myobject", []byte("foo"))
if err != nil {
t.Fatalf("unexpected error %v", err)
}
err = storage.AppendFile("foo", "otherobject", []byte("foo"))
err = storage.AppendFile(context.Background(), "foo", "otherobject", []byte("foo"))
if err != nil {
t.Fatalf("unexpected error %v", err)
}
@ -405,7 +406,7 @@ func testStorageAPIRenameFile(t *testing.T, storage StorageAPI) {
}
for i, testCase := range testCases {
err := storage.RenameFile(testCase.volumeName, testCase.objectName, testCase.destVolumeName, testCase.destObjectName)
err := storage.RenameFile(context.Background(), testCase.volumeName, testCase.objectName, testCase.destVolumeName, testCase.destObjectName)
expectErr := (err != nil)
if expectErr != testCase.expectErr {

16
cmd/tree-walk_test.go
View File

@ -69,14 +69,14 @@ func TestFilterMatchingPrefix(t *testing.T) {
// Helper function that creates a volume and files in it.
func createNamespace(disk StorageAPI, volume string, files []string) error {
// Make a volume.
err := disk.MakeVol(volume)
err := disk.MakeVol(context.Background(), volume)
if err != nil {
return err
}
// Create files.
for _, file := range files {
err = disk.AppendFile(volume, file, []byte{})
err = disk.AppendFile(context.Background(), volume, file, []byte{})
if err != nil {
return err
}
@ -88,7 +88,7 @@ func createNamespace(disk StorageAPI, volume string, files []string) error {
// disks - used for doing disk.ListDir()
func listDirFactory(ctx context.Context, disk StorageAPI, isLeaf IsLeafFunc) ListDirFunc {
return func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string, delayIsLeaf bool) {
entries, err := disk.ListDir(volume, dirPath, -1)
entries, err := disk.ListDir(ctx, volume, dirPath, -1)
if err != nil {
return false, nil, false
}
@ -163,7 +163,7 @@ func TestTreeWalk(t *testing.T) {
}
isLeafDir := func(bucket, leafPath string) bool {
entries, _ := disk.ListDir(bucket, leafPath, 1)
entries, _ := disk.ListDir(context.Background(), bucket, leafPath, 1)
return len(entries) == 0
}
@ -207,7 +207,7 @@ func TestTreeWalkTimeout(t *testing.T) {
}
isLeafDir := func(bucket, leafPath string) bool {
entries, _ := disk.ListDir(bucket, leafPath, 1)
entries, _ := disk.ListDir(context.Background(), bucket, leafPath, 1)
return len(entries) == 0
}
@ -275,7 +275,7 @@ func TestRecursiveTreeWalk(t *testing.T) {
}
isLeafDir := func(bucket, leafPath string) bool {
entries, _ := disk1.ListDir(bucket, leafPath, 1)
entries, _ := disk1.ListDir(context.Background(), bucket, leafPath, 1)
return len(entries) == 0
}
@ -389,7 +389,7 @@ func TestSortedness(t *testing.T) {
}
isLeafDir := func(bucket, leafPath string) bool {
entries, _ := disk1.ListDir(bucket, leafPath, 1)
entries, _ := disk1.ListDir(context.Background(), bucket, leafPath, 1)
return len(entries) == 0
}
@ -469,7 +469,7 @@ func TestTreeWalkIsEnd(t *testing.T) {
}
isLeafDir := func(bucket, leafPath string) bool {
entries, _ := disk1.ListDir(bucket, leafPath, 1)
entries, _ := disk1.ListDir(context.Background(), bucket, leafPath, 1)
return len(entries) == 0
}

104
cmd/xl-storage-disk-id-check.go
View File

@ -84,8 +84,8 @@ func (p *xlStorageDiskIDCheck) checkDiskStale() error {
return errDiskNotFound
}
func (p *xlStorageDiskIDCheck) DiskInfo() (info DiskInfo, err error) {
info, err = p.storage.DiskInfo()
func (p *xlStorageDiskIDCheck) DiskInfo(ctx context.Context) (info DiskInfo, err error) {
info, err = p.storage.DiskInfo(ctx)
if err != nil {
return info, err
}
@ -99,145 +99,145 @@ func (p *xlStorageDiskIDCheck) DiskInfo() (info DiskInfo, err error) {
return info, nil
}
func (p *xlStorageDiskIDCheck) MakeVolBulk(volumes ...string) (err error) {
func (p *xlStorageDiskIDCheck) MakeVolBulk(ctx context.Context, volumes ...string) (err error) {
if err = p.checkDiskStale(); err != nil {
return err
}
return p.storage.MakeVolBulk(volumes...)
return p.storage.MakeVolBulk(ctx, volumes...)
}
func (p *xlStorageDiskIDCheck) MakeVol(volume string) (err error) {
func (p *xlStorageDiskIDCheck) MakeVol(ctx context.Context, volume string) (err error) {
if err = p.checkDiskStale(); err != nil {
return err
}
return p.storage.MakeVol(volume)
return p.storage.MakeVol(ctx, volume)
}
func (p *xlStorageDiskIDCheck) ListVols() ([]VolInfo, error) {
func (p *xlStorageDiskIDCheck) ListVols(ctx context.Context) ([]VolInfo, error) {
if err := p.checkDiskStale(); err != nil {
return nil, err
}
return p.storage.ListVols()
return p.storage.ListVols(ctx)
}
func (p *xlStorageDiskIDCheck) StatVol(volume string) (vol VolInfo, err error) {
func (p *xlStorageDiskIDCheck) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) {
if err = p.checkDiskStale(); err != nil {
return vol, err
}
return p.storage.StatVol(volume)
return p.storage.StatVol(ctx, volume)
}
func (p *xlStorageDiskIDCheck) DeleteVol(volume string, forceDelete bool) (err error) {
func (p *xlStorageDiskIDCheck) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) {
if err = p.checkDiskStale(); err != nil {
return err
}
return p.storage.DeleteVol(volume, forceDelete)
return p.storage.DeleteVol(ctx, volume, forceDelete)
}
func (p *xlStorageDiskIDCheck) WalkVersions(volume, dirPath string, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error) {
func (p *xlStorageDiskIDCheck) WalkVersions(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error) {
if err := p.checkDiskStale(); err != nil {
return nil, err
}
return p.storage.WalkVersions(volume, dirPath, marker, recursive, endWalkCh)
return p.storage.WalkVersions(ctx, volume, dirPath, marker, recursive, endWalkCh)
}
func (p *xlStorageDiskIDCheck) Walk(volume, dirPath string, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) {
func (p *xlStorageDiskIDCheck) Walk(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) {
if err := p.checkDiskStale(); err != nil {
return nil, err
}
return p.storage.Walk(volume, dirPath, marker, recursive, endWalkCh)
return p.storage.Walk(ctx, volume, dirPath, marker, recursive, endWalkCh)
}
func (p *xlStorageDiskIDCheck) WalkSplunk(volume, dirPath string, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error) {
func (p *xlStorageDiskIDCheck) WalkSplunk(ctx context.Context, volume, dirPath, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error) {
if err := p.checkDiskStale(); err != nil {
return nil, err
}
return p.storage.WalkSplunk(volume, dirPath, marker, endWalkCh)
return p.storage.WalkSplunk(ctx, volume, dirPath, marker, endWalkCh)
}
func (p *xlStorageDiskIDCheck) ListDir(volume, dirPath string, count int) ([]string, error) {
func (p *xlStorageDiskIDCheck) ListDir(ctx context.Context, volume, dirPath string, count int) ([]string, error) {
if err := p.checkDiskStale(); err != nil {
return nil, err
}
return p.storage.ListDir(volume, dirPath, count)
return p.storage.ListDir(ctx, volume, dirPath, count)
}
func (p *xlStorageDiskIDCheck) ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
func (p *xlStorageDiskIDCheck) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
if err := p.checkDiskStale(); err != nil {
return 0, err
}
return p.storage.ReadFile(volume, path, offset, buf, verifier)
return p.storage.ReadFile(ctx, volume, path, offset, buf, verifier)
}
func (p *xlStorageDiskIDCheck) AppendFile(volume string, path string, buf []byte) (err error) {
func (p *xlStorageDiskIDCheck) AppendFile(ctx context.Context, volume string, path string, buf []byte) (err error) {
if err = p.checkDiskStale(); err != nil {
return err
}
return p.storage.AppendFile(volume, path, buf)
return p.storage.AppendFile(ctx, volume, path, buf)
}
func (p *xlStorageDiskIDCheck) CreateFile(volume, path string, size int64, reader io.Reader) error {
func (p *xlStorageDiskIDCheck) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error {
if err := p.checkDiskStale(); err != nil {
return err
}
return p.storage.CreateFile(volume, path, size, reader)
return p.storage.CreateFile(ctx, volume, path, size, reader)
}
func (p *xlStorageDiskIDCheck) ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error) {
func (p *xlStorageDiskIDCheck) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) {
if err := p.checkDiskStale(); err != nil {
return nil, err
}
return p.storage.ReadFileStream(volume, path, offset, length)
return p.storage.ReadFileStream(ctx, volume, path, offset, length)
}
func (p *xlStorageDiskIDCheck) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error {
func (p *xlStorageDiskIDCheck) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error {
if err := p.checkDiskStale(); err != nil {
return err
}
return p.storage.RenameFile(srcVolume, srcPath, dstVolume, dstPath)
return p.storage.RenameFile(ctx, srcVolume, srcPath, dstVolume, dstPath)
}
func (p *xlStorageDiskIDCheck) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath string) error {
func (p *xlStorageDiskIDCheck) RenameData(ctx context.Context, srcVolume, srcPath, dataDir, dstVolume, dstPath string) error {
if err := p.checkDiskStale(); err != nil {
return err
}
return p.storage.RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath)
return p.storage.RenameData(ctx, srcVolume, srcPath, dataDir, dstVolume, dstPath)
}
func (p *xlStorageDiskIDCheck) CheckParts(volume string, path string, fi FileInfo) (err error) {
func (p *xlStorageDiskIDCheck) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (err error) {
if err = p.checkDiskStale(); err != nil {
return err
}
return p.storage.CheckParts(volume, path, fi)
return p.storage.CheckParts(ctx, volume, path, fi)
}
func (p *xlStorageDiskIDCheck) CheckFile(volume string, path string) (err error) {
func (p *xlStorageDiskIDCheck) CheckFile(ctx context.Context, volume string, path string) (err error) {
if err = p.checkDiskStale(); err != nil {
return err
}
return p.storage.CheckFile(volume, path)
return p.storage.CheckFile(ctx, volume, path)
}
func (p *xlStorageDiskIDCheck) DeleteFile(volume string, path string) (err error) {
func (p *xlStorageDiskIDCheck) DeleteFile(ctx context.Context, volume string, path string) (err error) {
if err = p.checkDiskStale(); err != nil {
return err
}
return p.storage.DeleteFile(volume, path)
return p.storage.DeleteFile(ctx, volume, path)
}
func (p *xlStorageDiskIDCheck) DeleteVersions(volume string, versions []FileInfo) (errs []error) {
func (p *xlStorageDiskIDCheck) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) (errs []error) {
if err := p.checkDiskStale(); err != nil {
errs = make([]error, len(versions))
for i := range errs {
@ -245,53 +245,53 @@ func (p *xlStorageDiskIDCheck) DeleteVersions(volume string, versions []FileInfo
}
return errs
}
return p.storage.DeleteVersions(volume, versions)
return p.storage.DeleteVersions(ctx, volume, versions)
}
func (p *xlStorageDiskIDCheck) VerifyFile(volume, path string, fi FileInfo) error {
func (p *xlStorageDiskIDCheck) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error {
if err := p.checkDiskStale(); err != nil {
return err
}
return p.storage.VerifyFile(volume, path, fi)
return p.storage.VerifyFile(ctx, volume, path, fi)
}
func (p *xlStorageDiskIDCheck) WriteAll(volume string, path string, reader io.Reader) (err error) {
func (p *xlStorageDiskIDCheck) WriteAll(ctx context.Context, volume string, path string, reader io.Reader) (err error) {
if err = p.checkDiskStale(); err != nil {
return err
}
return p.storage.WriteAll(volume, path, reader)
return p.storage.WriteAll(ctx, volume, path, reader)
}
func (p *xlStorageDiskIDCheck) DeleteVersion(volume, path string, fi FileInfo) (err error) {
func (p *xlStorageDiskIDCheck) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo) (err error) {
if err = p.checkDiskStale(); err != nil {
return err
}
return p.storage.DeleteVersion(volume, path, fi)
return p.storage.DeleteVersion(ctx, volume, path, fi)
}
func (p *xlStorageDiskIDCheck) WriteMetadata(volume, path string, fi FileInfo) (err error) {
func (p *xlStorageDiskIDCheck) WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) (err error) {
if err = p.checkDiskStale(); err != nil {
return err
}
return p.storage.WriteMetadata(volume, path, fi)
return p.storage.WriteMetadata(ctx, volume, path, fi)
}
func (p *xlStorageDiskIDCheck) ReadVersion(volume, path, versionID string) (fi FileInfo, err error) {
func (p *xlStorageDiskIDCheck) ReadVersion(ctx context.Context, volume, path, versionID string) (fi FileInfo, err error) {
if err = p.checkDiskStale(); err != nil {
return fi, err
}
return p.storage.ReadVersion(volume, path, versionID)
return p.storage.ReadVersion(ctx, volume, path, versionID)
}
func (p *xlStorageDiskIDCheck) ReadAll(volume string, path string) (buf []byte, err error) {
func (p *xlStorageDiskIDCheck) ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) {
if err = p.checkDiskStale(); err != nil {
return nil, err
}
return p.storage.ReadAll(volume, path)
return p.storage.ReadAll(ctx, volume, path)
}

76
cmd/xl-storage.go
View File

@ -417,7 +417,7 @@ type DiskInfo struct {
// DiskInfo provides current information about disk space usage,
// total free inodes and underlying filesystem.
func (s *xlStorage) DiskInfo() (info DiskInfo, err error) {
func (s *xlStorage) DiskInfo(context.Context) (info DiskInfo, err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@ -548,9 +548,9 @@ func (s *xlStorage) SetDiskID(id string) {
// storage rest server for remote disks.
}
func (s *xlStorage) MakeVolBulk(volumes ...string) (err error) {
func (s *xlStorage) MakeVolBulk(ctx context.Context, volumes ...string) (err error) {
for _, volume := range volumes {
if err = s.MakeVol(volume); err != nil {
if err = s.MakeVol(ctx, volume); err != nil {
if os.IsPermission(err) {
return errVolumeAccessDenied
}
@ -560,7 +560,7 @@ func (s *xlStorage) MakeVolBulk(volumes ...string) (err error) {
}
// Make a volume entry.
func (s *xlStorage) MakeVol(volume string) (err error) {
func (s *xlStorage) MakeVol(ctx context.Context, volume string) (err error) {
if !isValidVolname(volume) {
return errInvalidArgument
}
@ -594,7 +594,7 @@ func (s *xlStorage) MakeVol(volume string) (err error) {
}
// ListVols - list volumes.
func (s *xlStorage) ListVols() (volsInfo []VolInfo, err error) {
func (s *xlStorage) ListVols(context.Context) (volsInfo []VolInfo, err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@ -654,7 +654,7 @@ func listVols(dirPath string) ([]VolInfo, error) {
}
// StatVol - get volume info.
func (s *xlStorage) StatVol(volume string) (volInfo VolInfo, err error) {
func (s *xlStorage) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@ -686,7 +686,7 @@ func (s *xlStorage) StatVol(volume string) (volInfo VolInfo, err error) {
}
// DeleteVol - delete a volume.
func (s *xlStorage) DeleteVol(volume string, forceDelete bool) (err error) {
func (s *xlStorage) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@ -806,7 +806,7 @@ func (s *xlStorage) isLeafDir(volume, leafPath string) bool {
// sorted order, additionally along with metadata about each of those entries.
// Implemented specifically for Splunk backend structure and List call with
// delimiter as "guidSplunk"
func (s *xlStorage) WalkSplunk(volume, dirPath, marker string, endWalkCh <-chan struct{}) (ch chan FileInfo, err error) {
func (s *xlStorage) WalkSplunk(ctx context.Context, volume, dirPath, marker string, endWalkCh <-chan struct{}) (ch chan FileInfo, err error) {
// Verify if volume is valid and it exists.
volumeDir, err := s.getVolDir(volume)
if err != nil {
@ -877,7 +877,7 @@ func (s *xlStorage) WalkSplunk(volume, dirPath, marker string, endWalkCh <-chan
// WalkVersions - is a sorted walker which returns file entries in lexically sorted order,
// additionally along with metadata version info about each of those entries.
func (s *xlStorage) WalkVersions(volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (ch chan FileInfoVersions, err error) {
func (s *xlStorage) WalkVersions(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (ch chan FileInfoVersions, err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@ -912,7 +912,7 @@ func (s *xlStorage) WalkVersions(volume, dirPath, marker string, recursive bool,
go func() {
defer close(ch)
listDir := func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string, delayIsLeaf bool) {
entries, err := s.ListDir(volume, dirPath, -1)
entries, err := s.ListDir(ctx, volume, dirPath, -1)
if err != nil {
return false, nil, false
}
@ -962,7 +962,7 @@ func (s *xlStorage) WalkVersions(volume, dirPath, marker string, recursive bool,
// Walk - is a sorted walker which returns file entries in lexically
// sorted order, additionally along with metadata about each of those entries.
func (s *xlStorage) Walk(volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (ch chan FileInfo, err error) {
func (s *xlStorage) Walk(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (ch chan FileInfo, err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@ -997,7 +997,7 @@ func (s *xlStorage) Walk(volume, dirPath, marker string, recursive bool, endWalk
go func() {
defer close(ch)
listDir := func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string, delayIsLeaf bool) {
entries, err := s.ListDir(volume, dirPath, -1)
entries, err := s.ListDir(ctx, volume, dirPath, -1)
if err != nil {
return false, nil, false
}
@ -1046,7 +1046,7 @@ func (s *xlStorage) Walk(volume, dirPath, marker string, recursive bool, endWalk
// ListDir - return all the entries at the given directory path.
// If an entry is a directory it will be returned with a trailing SlashSeparator.
func (s *xlStorage) ListDir(volume, dirPath string, count int) (entries []string, err error) {
func (s *xlStorage) ListDir(ctx context.Context, volume, dirPath string, count int) (entries []string, err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@ -1082,10 +1082,10 @@ func (s *xlStorage) ListDir(volume, dirPath string, count int) (entries []string
// DeleteVersions deletes slice of versions, it can be same object
// or multiple objects.
func (s *xlStorage) DeleteVersions(volume string, versions []FileInfo) []error {
func (s *xlStorage) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) []error {
errs := make([]error, len(versions))
for i, version := range versions {
if err := s.DeleteVersion(volume, version.Name, version); err != nil {
if err := s.DeleteVersion(ctx, volume, version.Name, version); err != nil {
errs[i] = err
}
}
@ -1094,12 +1094,12 @@ func (s *xlStorage) DeleteVersions(volume string, versions []FileInfo) []error {
}
// DeleteVersion - deletes FileInfo metadata for path at `xl.meta`
func (s *xlStorage) DeleteVersion(volume, path string, fi FileInfo) error {
func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo) error {
if HasSuffix(path, SlashSeparator) {
return s.DeleteFile(volume, path)
return s.DeleteFile(ctx, volume, path)
}
buf, err := s.ReadAll(volume, pathJoin(path, xlStorageFormatFile))
buf, err := s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile))
if err != nil {
return err
}
@ -1152,7 +1152,7 @@ func (s *xlStorage) DeleteVersion(volume, path string, fi FileInfo) error {
}
if !lastVersion {
return s.WriteAll(volume, pathJoin(path, xlStorageFormatFile), bytes.NewReader(buf))
return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), bytes.NewReader(buf))
}
// Delete the meta file, if there are no more versions the
@ -1166,8 +1166,8 @@ func (s *xlStorage) DeleteVersion(volume, path string, fi FileInfo) error {
}
// WriteMetadata - writes FileInfo metadata for path at `xl.meta`
func (s *xlStorage) WriteMetadata(volume, path string, fi FileInfo) error {
buf, err := s.ReadAll(volume, pathJoin(path, xlStorageFormatFile))
func (s *xlStorage) WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) error {
buf, err := s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile))
if err != nil && err != errFileNotFound {
return err
}
@ -1200,7 +1200,7 @@ func (s *xlStorage) WriteMetadata(volume, path string, fi FileInfo) error {
}
}
return s.WriteAll(volume, pathJoin(path, xlStorageFormatFile), bytes.NewReader(buf))
return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), bytes.NewReader(buf))
}
func (s *xlStorage) renameLegacyMetadata(volume, path string) error {
@ -1268,14 +1268,14 @@ func (s *xlStorage) renameLegacyMetadata(volume, path string) error {
}
// ReadVersion - reads metadata and returns FileInfo at path `xl.meta`
func (s *xlStorage) ReadVersion(volume, path, versionID string) (fi FileInfo, err error) {
buf, err := s.ReadAll(volume, pathJoin(path, xlStorageFormatFile))
func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID string) (fi FileInfo, err error) {
buf, err := s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile))
if err != nil {
if err == errFileNotFound {
if err = s.renameLegacyMetadata(volume, path); err != nil {
return fi, err
}
buf, err = s.ReadAll(volume, pathJoin(path, xlStorageFormatFile))
buf, err = s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile))
if err != nil {
return fi, err
}
@ -1300,7 +1300,7 @@ func (s *xlStorage) ReadVersion(volume, path, versionID string) (fi FileInfo, er
// as an error to be reported.
// This API is meant to be used on files which have small memory footprint, do
// not use this on large files as it would cause server to crash.
func (s *xlStorage) ReadAll(volume, path string) (buf []byte, err error) {
func (s *xlStorage) ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@ -1363,7 +1363,7 @@ func (s *xlStorage) ReadAll(volume, path string) (buf []byte, err error) {
//
// Additionally ReadFile also starts reading from an offset. ReadFile
// semantics are same as io.ReadFull.
func (s *xlStorage) ReadFile(volume, path string, offset int64, buffer []byte, verifier *BitrotVerifier) (int64, error) {
func (s *xlStorage) ReadFile(ctx context.Context, volume string, path string, offset int64, buffer []byte, verifier *BitrotVerifier) (int64, error) {
if offset < 0 {
return 0, errInvalidArgument
}
@ -1518,7 +1518,7 @@ func (s *xlStorage) openFile(volume, path string, mode int) (f *os.File, err err
}
// ReadFileStream - Returns the read stream of the file.
func (s *xlStorage) ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error) {
func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) {
if offset < 0 {
return nil, errInvalidArgument
}
@ -1611,7 +1611,7 @@ func (c closeWrapper) Close() error {
}
// CreateFile - creates the file.
func (s *xlStorage) CreateFile(volume, path string, fileSize int64, r io.Reader) (err error) {
func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSize int64, r io.Reader) (err error) {
if fileSize < -1 {
return errInvalidArgument
}
@ -1720,7 +1720,7 @@ func (s *xlStorage) CreateFile(volume, path string, fileSize int64, r io.Reader)
return nil
}
func (s *xlStorage) WriteAll(volume, path string, reader io.Reader) (err error) {
func (s *xlStorage) WriteAll(ctx context.Context, volume string, path string, reader io.Reader) (err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@ -1742,7 +1742,7 @@ func (s *xlStorage) WriteAll(volume, path string, reader io.Reader) (err error)
// AppendFile - append a byte array at path, if file doesn't exist at
// path this call explicitly creates it.
func (s *xlStorage) AppendFile(volume, path string, buf []byte) (err error) {
func (s *xlStorage) AppendFile(ctx context.Context, volume string, path string, buf []byte) (err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@ -1764,7 +1764,7 @@ func (s *xlStorage) AppendFile(volume, path string, buf []byte) (err error) {
}
// CheckParts check if path has necessary parts available.
func (s *xlStorage) CheckParts(volume, path string, fi FileInfo) error {
func (s *xlStorage) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@ -1809,7 +1809,7 @@ func (s *xlStorage) CheckParts(volume, path string, fi FileInfo) error {
}
// CheckFile check if path has necessary metadata.
func (s *xlStorage) CheckFile(volume, path string) error {
func (s *xlStorage) CheckFile(ctx context.Context, volume string, path string) error {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@ -1913,7 +1913,7 @@ func deleteFile(basePath, deletePath string, recursive bool) error {
}
// DeleteFile - delete a file at path.
func (s *xlStorage) DeleteFile(volume, path string) (err error) {
func (s *xlStorage) DeleteFile(ctx context.Context, volume string, path string) (err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@ -1988,7 +1988,7 @@ func (s *xlStorage) DeleteFileBulk(volume string, paths []string) (errs []error,
}
// RenameData - rename source path to destination path atomically, metadata and data directory.
func (s *xlStorage) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath string) (err error) {
func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath, dataDir, dstVolume, dstPath string) (err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@ -2170,7 +2170,7 @@ func (s *xlStorage) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath s
return errFileCorrupt
}
if err = s.WriteAll(srcVolume, pathJoin(srcPath, xlStorageFormatFile), bytes.NewReader(dstBuf)); err != nil {
if err = s.WriteAll(ctx, srcVolume, pathJoin(srcPath, xlStorageFormatFile), bytes.NewReader(dstBuf)); err != nil {
return err
}
@ -2201,7 +2201,7 @@ func (s *xlStorage) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath s
}
// RenameFile - rename source path to destination path atomically.
func (s *xlStorage) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err error) {
func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) (err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@ -2357,7 +2357,7 @@ func (s *xlStorage) bitrotVerify(partPath string, partSize int64, algo BitrotAlg
}
}
func (s *xlStorage) VerifyFile(volume, path string, fi FileInfo) (err error) {
func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)

129
cmd/xl-storage_test.go
View File

@ -18,6 +18,7 @@ package cmd
import (
"bytes"
"context"
"crypto/rand"
"fmt"
"io"
@ -126,12 +127,12 @@ func newXLStorageTestSetup() (*xlStorageDiskIDCheck, string, error) {
if err != nil {
return nil, "", err
}
err = storage.MakeVol(minioMetaBucket)
err = storage.MakeVol(context.Background(), minioMetaBucket)
if err != nil {
return nil, "", err
}
// Create a sample format.json file
err = storage.WriteAll(minioMetaBucket, formatConfigFile, bytes.NewBufferString(`{"version":"1","format":"xl","id":"592a41c2-b7cc-4130-b883-c4b5cb15965b","xl":{"version":"3","this":"da017d62-70e3-45f1-8a1a-587707e69ad1","sets":[["e07285a6-8c73-4962-89c6-047fb939f803","33b8d431-482d-4376-b63c-626d229f0a29","cff6513a-4439-4dc1-bcaa-56c9e880c352","da017d62-70e3-45f1-8a1a-587707e69ad1","9c9f21d5-1f15-4737-bce6-835faa0d9626","0a59b346-1424-4fc2-9fa2-a2e80541d0c1","7924a3dc-b69a-4971-9a2e-014966d6aebb","4d2b8dd9-4e48-444b-bdca-c89194b26042"]],"distributionAlgo":"CRCMOD"}}`))
err = storage.WriteAll(context.Background(), minioMetaBucket, formatConfigFile, bytes.NewBufferString(`{"version":"1","format":"xl","id":"592a41c2-b7cc-4130-b883-c4b5cb15965b","xl":{"version":"3","this":"da017d62-70e3-45f1-8a1a-587707e69ad1","sets":[["e07285a6-8c73-4962-89c6-047fb939f803","33b8d431-482d-4376-b63c-626d229f0a29","cff6513a-4439-4dc1-bcaa-56c9e880c352","da017d62-70e3-45f1-8a1a-587707e69ad1","9c9f21d5-1f15-4737-bce6-835faa0d9626","0a59b346-1424-4fc2-9fa2-a2e80541d0c1","7924a3dc-b69a-4971-9a2e-014966d6aebb","4d2b8dd9-4e48-444b-bdca-c89194b26042"]],"distributionAlgo":"CRCMOD"}}`))
if err != nil {
return nil, "", err
}
@ -262,16 +263,16 @@ func TestXLStorageReadAll(t *testing.T) {
defer os.RemoveAll(path)
// Create files for the test cases.
if err = xlStorage.MakeVol("exists"); err != nil {
if err = xlStorage.MakeVol(context.Background(), "exists"); err != nil {
t.Fatalf("Unable to create a volume \"exists\", %s", err)
}
if err = xlStorage.AppendFile("exists", "as-directory/as-file", []byte("Hello, World")); err != nil {
if err = xlStorage.AppendFile(context.Background(), "exists", "as-directory/as-file", []byte("Hello, World")); err != nil {
t.Fatalf("Unable to create a file \"as-directory/as-file\", %s", err)
}
if err = xlStorage.AppendFile("exists", "as-file", []byte("Hello, World")); err != nil {
if err = xlStorage.AppendFile(context.Background(), "exists", "as-file", []byte("Hello, World")); err != nil {
t.Fatalf("Unable to create a file \"as-file\", %s", err)
}
if err = xlStorage.AppendFile("exists", "as-file-parent", []byte("Hello, World")); err != nil {
if err = xlStorage.AppendFile(context.Background(), "exists", "as-file-parent", []byte("Hello, World")); err != nil {
t.Fatalf("Unable to create a file \"as-file-parent\", %s", err)
}
@ -328,7 +329,7 @@ func TestXLStorageReadAll(t *testing.T) {
var dataRead []byte
// Run through all the test cases and validate for ReadAll.
for i, testCase := range testCases {
dataRead, err = xlStorage.ReadAll(testCase.volume, testCase.path)
dataRead, err = xlStorage.ReadAll(context.Background(), testCase.volume, testCase.path)
if err != testCase.err {
t.Fatalf("TestXLStorage %d: Expected err \"%s\", got err \"%s\"", i+1, testCase.err, err)
}
@ -434,7 +435,7 @@ func TestXLStorageMakeVol(t *testing.T) {
}
for i, testCase := range testCases {
if err := xlStorage.MakeVol(testCase.volName); err != testCase.expectedErr {
if err := xlStorage.MakeVol(context.Background(), testCase.volName); err != testCase.expectedErr {
t.Fatalf("TestXLStorage %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
}
}
@ -470,7 +471,7 @@ func TestXLStorageMakeVol(t *testing.T) {
t.Fatalf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err)
}
if err := xlStorageNew.MakeVol("test-vol"); err != errDiskAccessDenied {
if err := xlStorageNew.MakeVol(context.Background(), "test-vol"); err != errDiskAccessDenied {
t.Fatalf("expected: %s, got: %s", errDiskAccessDenied, err)
}
}
@ -486,7 +487,7 @@ func TestXLStorageDeleteVol(t *testing.T) {
defer os.RemoveAll(path)
// Setup test environment.
if err = xlStorage.MakeVol("success-vol"); err != nil {
if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil {
t.Fatalf("Unable to create volume, %s", err)
}
@ -530,7 +531,7 @@ func TestXLStorageDeleteVol(t *testing.T) {
}
for i, testCase := range testCases {
if err = xlStorage.DeleteVol(testCase.volName, false); err != testCase.expectedErr {
if err = xlStorage.DeleteVol(context.Background(), testCase.volName, false); err != testCase.expectedErr {
t.Fatalf("TestXLStorage: %d, expected: %s, got: %s", i+1, testCase.expectedErr, err)
}
}
@ -569,7 +570,7 @@ func TestXLStorageDeleteVol(t *testing.T) {
t.Fatalf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err)
}
if err = xlStorageNew.DeleteVol("mybucket", false); err != errDiskAccessDenied {
if err = xlStorageNew.DeleteVol(context.Background(), "mybucket", false); err != errDiskAccessDenied {
t.Fatalf("expected: Permission error, got: %s", err)
}
}
@ -583,7 +584,7 @@ func TestXLStorageDeleteVol(t *testing.T) {
// TestXLStorage for delete on an removed disk.
// should fail with disk not found.
err = xlStorageDeletedStorage.DeleteVol("Del-Vol", false)
err = xlStorageDeletedStorage.DeleteVol(context.Background(), "Del-Vol", false)
if err != errDiskNotFound {
t.Errorf("Expected: \"Disk not found\", got \"%s\"", err)
}
@ -599,7 +600,7 @@ func TestXLStorageStatVol(t *testing.T) {
defer os.RemoveAll(path)
// Setup test environment.
if err = xlStorage.MakeVol("success-vol"); err != nil {
if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil {
t.Fatalf("Unable to create volume, %s", err)
}
@ -626,7 +627,7 @@ func TestXLStorageStatVol(t *testing.T) {
for i, testCase := range testCases {
var volInfo VolInfo
volInfo, err = xlStorage.StatVol(testCase.volName)
volInfo, err = xlStorage.StatVol(context.Background(), testCase.volName)
if err != testCase.expectedErr {
t.Fatalf("TestXLStorage case : %d, Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
}
@ -648,7 +649,7 @@ func TestXLStorageStatVol(t *testing.T) {
// TestXLStorage for delete on an removed disk.
// should fail with disk not found.
_, err = xlStorageDeletedStorage.StatVol("Stat vol")
_, err = xlStorageDeletedStorage.StatVol(context.Background(), "Stat vol")
if err != errDiskNotFound {
t.Errorf("Expected: \"Disk not found\", got \"%s\"", err)
}
@ -664,18 +665,18 @@ func TestXLStorageListVols(t *testing.T) {
var volInfos []VolInfo
// TestXLStorage empty list vols.
if volInfos, err = xlStorage.ListVols(); err != nil {
if volInfos, err = xlStorage.ListVols(context.Background()); err != nil {
t.Fatalf("expected: <nil>, got: %s", err)
} else if len(volInfos) != 1 {
t.Fatalf("expected: one entry, got: %s", volInfos)
}
// TestXLStorage non-empty list vols.
if err = xlStorage.MakeVol("success-vol"); err != nil {
if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil {
t.Fatalf("Unable to create volume, %s", err)
}
volInfos, err = xlStorage.ListVols()
volInfos, err = xlStorage.ListVols(context.Background())
if err != nil {
t.Fatalf("expected: <nil>, got: %s", err)
}
@ -696,7 +697,7 @@ func TestXLStorageListVols(t *testing.T) {
// removing the path and simulating disk failure
os.RemoveAll(path)
// should fail with errDiskNotFound.
if _, err = xlStorage.ListVols(); err != errDiskNotFound {
if _, err = xlStorage.ListVols(context.Background()); err != errDiskNotFound {
t.Errorf("Expected to fail with \"%s\", but instead failed with \"%s\"", errDiskNotFound, err)
}
}
@ -718,13 +719,13 @@ func TestXLStorageXlStorageListDir(t *testing.T) {
// removing the disk, used to recreate disk not found error.
os.RemoveAll(diskPath)
// Setup test environment.
if err = xlStorage.MakeVol("success-vol"); err != nil {
if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil {
t.Fatalf("Unable to create volume, %s", err)
}
if err = xlStorage.AppendFile("success-vol", "abc/def/ghi/success-file", []byte("Hello, world")); err != nil {
if err = xlStorage.AppendFile(context.Background(), "success-vol", "abc/def/ghi/success-file", []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err)
}
if err = xlStorage.AppendFile("success-vol", "abc/xyz/ghi/success-file", []byte("Hello, world")); err != nil {
if err = xlStorage.AppendFile(context.Background(), "success-vol", "abc/xyz/ghi/success-file", []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err)
}
@ -783,7 +784,7 @@ func TestXLStorageXlStorageListDir(t *testing.T) {
for i, testCase := range testCases {
var dirList []string
dirList, err = xlStorage.ListDir(testCase.srcVol, testCase.srcPath, -1)
dirList, err = xlStorage.ListDir(context.Background(), testCase.srcVol, testCase.srcPath, -1)
if err != testCase.expectedErr {
t.Fatalf("TestXLStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
}
@ -816,14 +817,14 @@ func TestXLStorageXlStorageListDir(t *testing.T) {
t.Fatalf("Unable to initialize xlStorage, %s", err)
}
if err = xlStorageNew.DeleteFile("mybucket", "myobject"); err != errFileAccessDenied {
if err = xlStorageNew.DeleteFile(context.Background(), "mybucket", "myobject"); err != errFileAccessDenied {
t.Errorf("expected: %s, got: %s", errFileAccessDenied, err)
}
}
// TestXLStorage for delete on an removed disk.
// should fail with disk not found.
err = xlStorageDeletedStorage.DeleteFile("del-vol", "my-file")
err = xlStorageDeletedStorage.DeleteFile(context.Background(), "del-vol", "my-file")
if err != errDiskNotFound {
t.Errorf("Expected: \"Disk not found\", got \"%s\"", err)
}
@ -846,17 +847,17 @@ func TestXLStorageDeleteFile(t *testing.T) {
// removing the disk, used to recreate disk not found error.
os.RemoveAll(diskPath)
// Setup test environment.
if err = xlStorage.MakeVol("success-vol"); err != nil {
if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil {
t.Fatalf("Unable to create volume, %s", err)
}
if err = xlStorage.AppendFile("success-vol", "success-file", []byte("Hello, world")); err != nil {
if err = xlStorage.AppendFile(context.Background(), "success-vol", "success-file", []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err)
}
if err = xlStorage.MakeVol("no-permissions"); err != nil {
if err = xlStorage.MakeVol(context.Background(), "no-permissions"); err != nil {
t.Fatalf("Unable to create volume, %s", err.Error())
}
if err = xlStorage.AppendFile("no-permissions", "dir/file", []byte("Hello, world")); err != nil {
if err = xlStorage.AppendFile(context.Background(), "no-permissions", "dir/file", []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err.Error())
}
// Parent directory must have write permissions, this is read + execute.
@ -915,7 +916,7 @@ func TestXLStorageDeleteFile(t *testing.T) {
}
for i, testCase := range testCases {
if err = xlStorage.DeleteFile(testCase.srcVol, testCase.srcPath); err != testCase.expectedErr {
if err = xlStorage.DeleteFile(context.Background(), testCase.srcVol, testCase.srcPath); err != testCase.expectedErr {
t.Errorf("TestXLStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
}
}
@ -940,14 +941,14 @@ func TestXLStorageDeleteFile(t *testing.T) {
t.Fatalf("Unable to initialize xlStorage, %s", err)
}
if err = xlStorageNew.DeleteFile("mybucket", "myobject"); err != errFileAccessDenied {
if err = xlStorageNew.DeleteFile(context.Background(), "mybucket", "myobject"); err != errFileAccessDenied {
t.Errorf("expected: %s, got: %s", errFileAccessDenied, err)
}
}
// TestXLStorage for delete on an removed disk.
// should fail with disk not found.
err = xlStorageDeletedStorage.DeleteFile("del-vol", "my-file")
err = xlStorageDeletedStorage.DeleteFile(context.Background(), "del-vol", "my-file")
if err != errDiskNotFound {
t.Errorf("Expected: \"Disk not found\", got \"%s\"", err)
}
@ -964,7 +965,7 @@ func TestXLStorageReadFile(t *testing.T) {
volume := "success-vol"
// Setup test environment.
if err = xlStorage.MakeVol(volume); err != nil {
if err = xlStorage.MakeVol(context.Background(), volume); err != nil {
t.Fatalf("Unable to create volume, %s", err)
}
@ -1048,7 +1049,7 @@ func TestXLStorageReadFile(t *testing.T) {
v := NewBitrotVerifier(SHA256, getSHA256Sum([]byte("hello, world")))
// Create test files for further reading.
for i, appendFile := range appendFiles {
err = xlStorage.AppendFile(volume, appendFile.fileName, []byte("hello, world"))
err = xlStorage.AppendFile(context.Background(), volume, appendFile.fileName, []byte("hello, world"))
if err != appendFile.expectedErr {
t.Fatalf("Creating file failed: %d %#v, expected: %s, got: %s", i+1, appendFile, appendFile.expectedErr, err)
}
@ -1057,7 +1058,7 @@ func TestXLStorageReadFile(t *testing.T) {
{
buf := make([]byte, 5)
// Test for negative offset.
if _, err = xlStorage.ReadFile(volume, "myobject", -1, buf, v); err == nil {
if _, err = xlStorage.ReadFile(context.Background(), volume, "myobject", -1, buf, v); err == nil {
t.Fatalf("expected: error, got: <nil>")
}
}
@ -1067,7 +1068,7 @@ func TestXLStorageReadFile(t *testing.T) {
var n int64
// Common read buffer.
var buf = make([]byte, testCase.bufSize)
n, err = xlStorage.ReadFile(testCase.volume, testCase.fileName, testCase.offset, buf, v)
n, err = xlStorage.ReadFile(context.Background(), testCase.volume, testCase.fileName, testCase.offset, buf, v)
if err != nil && testCase.expectedErr != nil {
// Validate if the type string of the errors are an exact match.
if err.Error() != testCase.expectedErr.Error() {
@ -1140,7 +1141,7 @@ func TestXLStorageReadFile(t *testing.T) {
// Common read buffer.
var buf = make([]byte, 10)
if _, err = xlStoragePermStorage.ReadFile("mybucket", "myobject", 0, buf, v); err != errFileAccessDenied {
if _, err = xlStoragePermStorage.ReadFile(context.Background(), "mybucket", "myobject", 0, buf, v); err != errFileAccessDenied {
t.Errorf("expected: %s, got: %s", errFileAccessDenied, err)
}
}
@ -1182,7 +1183,7 @@ func TestXLStorageReadFileWithVerify(t *testing.T) {
os.RemoveAll(path)
t.Fatalf("Unable to create xlStorage test setup, %s", err)
}
if err = xlStorage.MakeVol(volume); err != nil {
if err = xlStorage.MakeVol(context.Background(), volume); err != nil {
os.RemoveAll(path)
t.Fatalf("Unable to create volume %s: %v", volume, err)
}
@ -1191,7 +1192,7 @@ func TestXLStorageReadFileWithVerify(t *testing.T) {
os.RemoveAll(path)
t.Fatalf("Unable to create generate random data: %v", err)
}
if err = xlStorage.AppendFile(volume, object, data); err != nil {
if err = xlStorage.AppendFile(context.Background(), volume, object, data); err != nil {
os.RemoveAll(path)
t.Fatalf("Unable to create object: %v", err)
}
@ -1204,7 +1205,7 @@ func TestXLStorageReadFileWithVerify(t *testing.T) {
}
buffer := make([]byte, test.length)
n, err := xlStorage.ReadFile(volume, test.file, int64(test.offset), buffer, NewBitrotVerifier(test.algorithm, h.Sum(nil)))
n, err := xlStorage.ReadFile(context.Background(), volume, test.file, int64(test.offset), buffer, NewBitrotVerifier(test.algorithm, h.Sum(nil)))
switch {
case err == nil && test.expError != nil:
@ -1227,7 +1228,7 @@ func TestXLStorageFormatFileChange(t *testing.T) {
}
defer os.RemoveAll(path)
if err = xlStorage.MakeVol(volume); err != nil {
if err = xlStorage.MakeVol(context.Background(), volume); err != nil {
t.Fatalf("MakeVol failed with %s", err)
}
@ -1236,7 +1237,7 @@ func TestXLStorageFormatFileChange(t *testing.T) {
t.Fatalf("ioutil.WriteFile failed with %s", err)
}
err = xlStorage.MakeVol(volume)
err = xlStorage.MakeVol(context.Background(), volume)
if err != errVolumeExists {
t.Fatalf("MakeVol expected to fail with errDiskNotFound but failed with %s", err)
}
@ -1252,7 +1253,7 @@ func TestXLStorageAppendFile(t *testing.T) {
defer os.RemoveAll(path)
// Setup test environment.
if err = xlStorage.MakeVol("success-vol"); err != nil {
if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil {
t.Fatalf("Unable to create volume, %s", err)
}
@ -1282,7 +1283,7 @@ func TestXLStorageAppendFile(t *testing.T) {
}
for i, testCase := range testCases {
if err = xlStorage.AppendFile("success-vol", testCase.fileName, []byte("hello, world")); err != testCase.expectedErr {
if err = xlStorage.AppendFile(context.Background(), "success-vol", testCase.fileName, []byte("hello, world")); err != testCase.expectedErr {
t.Errorf("Case: %d, expected: %s, got: %s", i+1, testCase.expectedErr, err)
}
}
@ -1308,14 +1309,14 @@ func TestXLStorageAppendFile(t *testing.T) {
t.Fatalf("Unable to initialize xlStorage, %s", err)
}
if err = xlStoragePermStorage.AppendFile("mybucket", "myobject", []byte("hello, world")); err != errFileAccessDenied {
if err = xlStoragePermStorage.AppendFile(context.Background(), "mybucket", "myobject", []byte("hello, world")); err != errFileAccessDenied {
t.Fatalf("expected: Permission error, got: %s", err)
}
}
// TestXLStorage case with invalid volume name.
// A valid volume name should be atleast of size 3.
err = xlStorage.AppendFile("bn", "yes", []byte("hello, world"))
err = xlStorage.AppendFile(context.Background(), "bn", "yes", []byte("hello, world"))
if err != errVolumeNotFound {
t.Fatalf("expected: \"Invalid argument error\", got: \"%s\"", err)
}
@ -1331,32 +1332,32 @@ func TestXLStorageRenameFile(t *testing.T) {
defer os.RemoveAll(path)
// Setup test environment.
if err := xlStorage.MakeVol("src-vol"); err != nil {
if err := xlStorage.MakeVol(context.Background(), "src-vol"); err != nil {
t.Fatalf("Unable to create volume, %s", err)
}
if err := xlStorage.MakeVol("dest-vol"); err != nil {
if err := xlStorage.MakeVol(context.Background(), "dest-vol"); err != nil {
t.Fatalf("Unable to create volume, %s", err)
}
if err := xlStorage.AppendFile("src-vol", "file1", []byte("Hello, world")); err != nil {
if err := xlStorage.AppendFile(context.Background(), "src-vol", "file1", []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err)
}
if err := xlStorage.AppendFile("src-vol", "file2", []byte("Hello, world")); err != nil {
if err := xlStorage.AppendFile(context.Background(), "src-vol", "file2", []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err)
}
if err := xlStorage.AppendFile("src-vol", "file3", []byte("Hello, world")); err != nil {
if err := xlStorage.AppendFile(context.Background(), "src-vol", "file3", []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err)
}
if err := xlStorage.AppendFile("src-vol", "file4", []byte("Hello, world")); err != nil {
if err := xlStorage.AppendFile(context.Background(), "src-vol", "file4", []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err)
}
if err := xlStorage.AppendFile("src-vol", "file5", []byte("Hello, world")); err != nil {
if err := xlStorage.AppendFile(context.Background(), "src-vol", "file5", []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err)
}
if err := xlStorage.AppendFile("src-vol", "path/to/file1", []byte("Hello, world")); err != nil {
if err := xlStorage.AppendFile(context.Background(), "src-vol", "path/to/file1", []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err)
}
@ -1533,7 +1534,7 @@ func TestXLStorageRenameFile(t *testing.T) {
}
for i, testCase := range testCases {
if err := xlStorage.RenameFile(testCase.srcVol, testCase.srcPath, testCase.destVol, testCase.destPath); err != testCase.expectedErr {
if err := xlStorage.RenameFile(context.Background(), testCase.srcVol, testCase.srcPath, testCase.destVol, testCase.destPath); err != testCase.expectedErr {
t.Fatalf("TestXLStorage %d: Expected the error to be : \"%v\", got: \"%v\".", i+1, testCase.expectedErr, err)
}
}
@ -1549,15 +1550,15 @@ func TestXLStorageCheckFile(t *testing.T) {
defer os.RemoveAll(path)
// Setup test environment.
if err := xlStorage.MakeVol("success-vol"); err != nil {
if err := xlStorage.MakeVol(context.Background(), "success-vol"); err != nil {
t.Fatalf("Unable to create volume, %s", err)
}
if err := xlStorage.AppendFile("success-vol", pathJoin("success-file", xlStorageFormatFile), []byte("Hello, world")); err != nil {
if err := xlStorage.AppendFile(context.Background(), "success-vol", pathJoin("success-file", xlStorageFormatFile), []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err)
}
if err := xlStorage.AppendFile("success-vol", pathJoin("path/to/success-file", xlStorageFormatFile), []byte("Hello, world")); err != nil {
if err := xlStorage.AppendFile(context.Background(), "success-vol", pathJoin("path/to/success-file", xlStorageFormatFile), []byte("Hello, world")); err != nil {
t.Fatalf("Unable to create file, %s", err)
}
@ -1611,7 +1612,7 @@ func TestXLStorageCheckFile(t *testing.T) {
}
for i, testCase := range testCases {
if err := xlStorage.CheckFile(testCase.srcVol, testCase.srcPath); err != testCase.expectedErr {
if err := xlStorage.CheckFile(context.Background(), testCase.srcVol, testCase.srcPath); err != testCase.expectedErr {
t.Fatalf("TestXLStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
}
}
@ -1634,7 +1635,7 @@ func TestXLStorageVerifyFile(t *testing.T) {
volName := "testvol"
fileName := "testfile"
if err := xlStorage.MakeVol(volName); err != nil {
if err := xlStorage.MakeVol(context.Background(), volName); err != nil {
t.Fatal(err)
}
@ -1648,7 +1649,7 @@ func TestXLStorageVerifyFile(t *testing.T) {
h := algo.New()
h.Write(data)
hashBytes := h.Sum(nil)
if err := xlStorage.WriteAll(volName, fileName, bytes.NewBuffer(data)); err != nil {
if err := xlStorage.WriteAll(context.Background(), volName, fileName, bytes.NewBuffer(data)); err != nil {
t.Fatal(err)
}
if err := xlStorage.storage.bitrotVerify(pathJoin(path, volName, fileName), size, algo, hashBytes, 0); err != nil {
@ -1656,7 +1657,7 @@ func TestXLStorageVerifyFile(t *testing.T) {
}
// 2) Whole-file bitrot check on corrupted file
if err := xlStorage.AppendFile(volName, fileName, []byte("a")); err != nil {
if err := xlStorage.AppendFile(context.Background(), volName, fileName, []byte("a")); err != nil {
t.Fatal(err)
}
@ -1670,7 +1671,7 @@ func TestXLStorageVerifyFile(t *testing.T) {
t.Fatal("expected to fail bitrot check")
}
if err := xlStorage.DeleteFile(volName, fileName); err != nil {
if err := xlStorage.DeleteFile(context.Background(), volName, fileName); err != nil {
t.Fatal(err)
}

9
cmd/xl-storage_unix_test.go
View File

@ -19,6 +19,7 @@
package cmd
import (
"context"
"io/ioutil"
"os"
"path"
@ -56,7 +57,7 @@ func TestIsValidUmaskVol(t *testing.T) {
// Attempt to create a volume to verify the permissions later.
// MakeVol creates 0777.
if err = disk.MakeVol(testCase.volName); err != nil {
if err = disk.MakeVol(context.Background(), testCase.volName); err != nil {
t.Fatalf("Creating a volume failed with %s expected to pass.", err)
}
defer os.RemoveAll(tmpPath)
@ -98,7 +99,7 @@ func TestIsValidUmaskFile(t *testing.T) {
// Attempt to create a volume to verify the permissions later.
// MakeVol creates directory with 0777 perms.
if err = disk.MakeVol(testCase.volName); err != nil {
if err = disk.MakeVol(context.Background(), testCase.volName); err != nil {
t.Fatalf("Creating a volume failed with %s expected to pass.", err)
}
@ -106,12 +107,12 @@ func TestIsValidUmaskFile(t *testing.T) {
// Attempt to create a file to verify the permissions later.
// AppendFile creates file with 0666 perms.
if err = disk.AppendFile(testCase.volName, pathJoin("hello-world.txt", xlStorageFormatFile), []byte("Hello World")); err != nil {
if err = disk.AppendFile(context.Background(), testCase.volName, pathJoin("hello-world.txt", xlStorageFormatFile), []byte("Hello World")); err != nil {
t.Fatalf("Create a file `test` failed with %s expected to pass.", err)
}
// CheckFile - stat the file.
if err := disk.CheckFile(testCase.volName, "hello-world.txt"); err != nil {
if err := disk.CheckFile(context.Background(), testCase.volName, "hello-world.txt"); err != nil {
t.Fatalf("Stat failed with %s expected to pass.", err)
}
}

13
cmd/xl-storage_windows_test.go
View File

@ -20,6 +20,7 @@ package cmd
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"os"
@ -54,20 +55,20 @@ func TestUNCPaths(t *testing.T) {
}
// Create volume to use in conjunction with other StorageAPI's file API(s)
err = fs.MakeVol("voldir")
err = fs.MakeVol(context.Background(), "voldir")
if err != nil {
t.Fatal(err)
}
for i, test := range testCases {
t.Run(fmt.Sprint(i), func(t *testing.T) {
err = fs.AppendFile("voldir", test.objName, []byte("hello"))
err = fs.AppendFile(context.Background(), "voldir", test.objName, []byte("hello"))
if err != nil && test.pass {
t.Error(err)
} else if err == nil && !test.pass {
t.Error(err)
}
fs.DeleteFile("voldir", test.objName)
fs.DeleteFile(context.Background(), "voldir", test.objName)
})
}
}
@ -89,19 +90,19 @@ func TestUNCPathENOTDIR(t *testing.T) {
}
// Create volume to use in conjunction with other StorageAPI's file API(s)
err = fs.MakeVol("voldir")
err = fs.MakeVol(context.Background(), "voldir")
if err != nil {
t.Fatal(err)
}
err = fs.AppendFile("voldir", "/file", []byte("hello"))
err = fs.AppendFile(context.Background(), "voldir", "/file", []byte("hello"))
if err != nil {
t.Fatal(err)
}
// Try to create a file that includes a file in its path components.
// In *nix, this returns syscall.ENOTDIR while in windows we receive the following error.
err = fs.AppendFile("voldir", "/file/obj1", []byte("hello"))
err = fs.AppendFile(context.Background(), "voldir", "/file/obj1", []byte("hello"))
if err != errFileAccessDenied {
t.Errorf("expected: %s, got: %s", errFileAccessDenied, err)
}