From 2d58a8d861d9470b3b7635f1f6f8f2229f366958 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Fri, 4 Sep 2020 09:45:06 -0700 Subject: [PATCH] Add storage layer contexts (#10321) Add context to all (non-trivial) calls to the storage layer. Contexts are propagated through the REST client. - `context.TODO()` is left in place for the places where it needs to be added to the caller. - `endWalkCh` could probably be removed from the walkers, but no changes so far. The "dangerous" part is that now a caller disconnecting *will* propagate down, so a "delete" operation will now be interrupted. In some cases we might want to disconnect this functionality so the operation completes if it has started, leaving the system in a cleaner state. --- cmd/bitrot-streaming.go | 5 +- cmd/bitrot-whole.go | 5 +- cmd/bitrot_test.go | 3 +- cmd/bootstrap-peer-server.go | 4 +- cmd/erasure-bucket.go | 14 ++-- cmd/erasure-common.go | 2 +- cmd/erasure-decode_test.go | 2 +- cmd/erasure-encode_test.go | 8 +- cmd/erasure-healing-common.go | 4 +- cmd/erasure-healing-common_test.go | 2 +- cmd/erasure-healing.go | 14 ++-- cmd/erasure-healing_test.go | 32 +++---- cmd/erasure-metadata-utils.go | 2 +- cmd/erasure-metadata.go | 4 +- cmd/erasure-multipart.go | 12 +-- cmd/erasure-object.go | 16 ++-- cmd/erasure-sets.go | 10 +-- cmd/erasure.go | 4 +- cmd/erasure_test.go | 2 +- cmd/format-erasure.go | 12 +-- cmd/global-heal.go | 2 +- cmd/lock-rest-client.go | 4 +- cmd/naughty-disk_test.go | 104 +++++++++++------------ cmd/object-api-common.go | 6 +- cmd/peer-rest-client.go | 4 +- cmd/rest/client.go | 10 +-- cmd/storage-interface.go | 56 ++++++------- cmd/storage-rest-client.go | 130 ++++++++++++++--------------- cmd/storage-rest-server.go | 52 ++++++------ cmd/storage-rest_test.go | 61 +++++++------- cmd/tree-walk_test.go | 16 ++-- cmd/xl-storage-disk-id-check.go | 104 +++++++++++------------ cmd/xl-storage.go | 76 ++++++++--------- cmd/xl-storage_test.go | 129 ++++++++++++++-------------- cmd/xl-storage_unix_test.go | 9 +- cmd/xl-storage_windows_test.go | 13 +-- 36 files changed, 466 insertions(+), 467 deletions(-) diff --git a/cmd/bitrot-streaming.go b/cmd/bitrot-streaming.go index 254bcdf65..7c0426000 100644 --- a/cmd/bitrot-streaming.go +++ b/cmd/bitrot-streaming.go @@ -18,6 +18,7 @@ package cmd import ( "bytes" + "context" "encoding/hex" "fmt" "hash" @@ -80,7 +81,7 @@ func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length i bitrotSumsTotalSize := ceilFrac(length, shardSize) * int64(h.Size()) // Size used for storing bitrot checksums. totalFileSize = bitrotSumsTotalSize + length } - err := disk.CreateFile(volume, filePath, totalFileSize, r) + err := disk.CreateFile(context.TODO(), volume, filePath, totalFileSize, r) r.CloseWithError(err) close(bw.canClose) }() @@ -118,7 +119,7 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) { // For the first ReadAt() call we need to open the stream for reading. b.currOffset = offset streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset - b.rc, err = b.disk.ReadFileStream(b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset) + b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset) if err != nil { return 0, err } diff --git a/cmd/bitrot-whole.go b/cmd/bitrot-whole.go index 8fd6eca5a..d30265c36 100644 --- a/cmd/bitrot-whole.go +++ b/cmd/bitrot-whole.go @@ -17,6 +17,7 @@ package cmd import ( + "context" "fmt" "hash" "io" @@ -34,7 +35,7 @@ type wholeBitrotWriter struct { } func (b *wholeBitrotWriter) Write(p []byte) (int, error) { - err := b.disk.AppendFile(b.volume, b.filePath, p) + err := b.disk.AppendFile(context.TODO(), b.volume, b.filePath, p) if err != nil { logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err)) return 0, err @@ -69,7 +70,7 @@ type wholeBitrotReader struct { func (b *wholeBitrotReader) ReadAt(buf []byte, offset int64) (n int, err error) { if b.buf == nil { b.buf = make([]byte, b.tillOffset-offset) - if _, err := b.disk.ReadFile(b.volume, b.filePath, offset, b.buf, b.verifier); err != nil { + if _, err := b.disk.ReadFile(context.TODO(), b.volume, b.filePath, offset, b.buf, b.verifier); err != nil { logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err)) return 0, err } diff --git a/cmd/bitrot_test.go b/cmd/bitrot_test.go index c1205fe02..0990c4919 100644 --- a/cmd/bitrot_test.go +++ b/cmd/bitrot_test.go @@ -17,6 +17,7 @@ package cmd import ( + "context" "io" "io/ioutil" "log" @@ -39,7 +40,7 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) { t.Fatal(err) } - disk.MakeVol(volume) + disk.MakeVol(context.Background(), volume) writer := newBitrotWriter(disk, volume, filePath, 35, bitrotAlgo, 10) diff --git a/cmd/bootstrap-peer-server.go b/cmd/bootstrap-peer-server.go index a6c8eae37..9954cd50e 100644 --- a/cmd/bootstrap-peer-server.go +++ b/cmd/bootstrap-peer-server.go @@ -131,7 +131,7 @@ func (client *bootstrapRESTClient) callWithContext(ctx context.Context, method s values = make(url.Values) } - respBody, err = client.restClient.CallWithContext(ctx, method, values, body, length) + respBody, err = client.restClient.Call(ctx, method, values, body, length) if err == nil { return respBody, nil } @@ -234,7 +234,7 @@ func newBootstrapRESTClient(endpoint Endpoint) *bootstrapRESTClient { ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout) // Instantiate a new rest client for healthcheck // to avoid recursive healthCheckFn() - respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).CallWithContext(ctx, bootstrapRESTMethodHealth, nil, nil, -1) + respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).Call(ctx, bootstrapRESTMethodHealth, nil, nil, -1) xhttp.DrainBody(respBody) cancel() var ne *rest.NetworkError diff --git a/cmd/erasure-bucket.go b/cmd/erasure-bucket.go index 696c38dac..d6485b611 100644 --- a/cmd/erasure-bucket.go +++ b/cmd/erasure-bucket.go @@ -49,7 +49,7 @@ func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket stri index := index g.Go(func() error { if storageDisks[index] != nil { - if err := storageDisks[index].MakeVol(bucket); err != nil { + if err := storageDisks[index].MakeVol(ctx, bucket); err != nil { if err != errVolumeExists { logger.LogIf(ctx, err) } @@ -75,7 +75,7 @@ func undoDeleteBucket(storageDisks []StorageAPI, bucket string) { } index := index g.Go(func() error { - _ = storageDisks[index].MakeVol(bucket) + _ = storageDisks[index].MakeVol(context.Background(), bucket) return nil }, index) } @@ -92,7 +92,7 @@ func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string) ( bucketErrs = append(bucketErrs, errDiskNotFound) continue } - volInfo, serr := disk.StatVol(bucketName) + volInfo, serr := disk.StatVol(ctx, bucketName) if serr == nil { return BucketInfo(volInfo), nil } @@ -129,7 +129,7 @@ func (er erasureObjects) listBuckets(ctx context.Context) (bucketsInfo []BucketI continue } var volsInfo []VolInfo - volsInfo, err = disk.ListVols() + volsInfo, err = disk.ListVols(ctx) if err == nil { // NOTE: The assumption here is that volumes across all disks in // readQuorum have consistent view i.e they all have same number @@ -181,10 +181,10 @@ func deleteDanglingBucket(ctx context.Context, storageDisks []StorageAPI, dErrs for index, err := range dErrs { if err == errVolumeNotEmpty { // Attempt to delete bucket again. - if derr := storageDisks[index].DeleteVol(bucket, false); derr == errVolumeNotEmpty { + if derr := storageDisks[index].DeleteVol(ctx, bucket, false); derr == errVolumeNotEmpty { _ = cleanupDir(ctx, storageDisks[index], bucket, "") - _ = storageDisks[index].DeleteVol(bucket, false) + _ = storageDisks[index].DeleteVol(ctx, bucket, false) // Cleanup all the previously incomplete multiparts. _ = cleanupDir(ctx, storageDisks[index], minioMetaMultipartBucket, bucket) @@ -204,7 +204,7 @@ func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, forceD index := index g.Go(func() error { if storageDisks[index] != nil { - if err := storageDisks[index].DeleteVol(bucket, forceDelete); err != nil { + if err := storageDisks[index].DeleteVol(ctx, bucket, forceDelete); err != nil { return err } err := cleanupDir(ctx, storageDisks[index], minioMetaMultipartBucket, bucket) diff --git a/cmd/erasure-common.go b/cmd/erasure-common.go index 639770e48..4a05f00e3 100644 --- a/cmd/erasure-common.go +++ b/cmd/erasure-common.go @@ -77,7 +77,7 @@ func (er erasureObjects) isObject(ctx context.Context, bucket, prefix string) (o return errDiskNotFound } // Check if 'prefix' is an object on this 'disk', else continue the check the next disk - return storageDisks[index].CheckFile(bucket, prefix) + return storageDisks[index].CheckFile(ctx, bucket, prefix) }, index) } diff --git a/cmd/erasure-decode_test.go b/cmd/erasure-decode_test.go index 17beb2ea1..3af1bdbce 100644 --- a/cmd/erasure-decode_test.go +++ b/cmd/erasure-decode_test.go @@ -28,7 +28,7 @@ import ( humanize "github.com/dustin/go-humanize" ) -func (a badDisk) ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) { +func (a badDisk) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) { return 0, errFaultyDisk } diff --git a/cmd/erasure-encode_test.go b/cmd/erasure-encode_test.go index ab923ec1b..bb78b4dab 100644 --- a/cmd/erasure-encode_test.go +++ b/cmd/erasure-encode_test.go @@ -32,11 +32,11 @@ func (a badDisk) String() string { return "bad-disk" } -func (a badDisk) AppendFile(volume string, path string, buf []byte) error { +func (a badDisk) AppendFile(ctx context.Context, volume string, path string, buf []byte) error { return errFaultyDisk } -func (a badDisk) ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error) { +func (a badDisk) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) { return nil, errFaultyDisk } @@ -44,7 +44,7 @@ func (a badDisk) UpdateBloomFilter(ctx context.Context, oldest, current uint64) return nil, errFaultyDisk } -func (a badDisk) CreateFile(volume, path string, size int64, reader io.Reader) error { +func (a badDisk) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error { return errFaultyDisk } @@ -195,7 +195,7 @@ func benchmarkErasureEncode(data, parity, dataDown, parityDown int, size int64, if disk == OfflineDisk { continue } - disk.DeleteFile("testbucket", "object") + disk.DeleteFile(context.Background(), "testbucket", "object") writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize()) } _, err := erasure.Encode(context.Background(), bytes.NewReader(content), writers, buffer, erasure.dataBlocks+1) diff --git a/cmd/erasure-healing-common.go b/cmd/erasure-healing-common.go index 61cacc19d..6059f9a6c 100644 --- a/cmd/erasure-healing-common.go +++ b/cmd/erasure-healing-common.go @@ -175,9 +175,9 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad // disk has a valid xl.meta but may not have all the // parts. This is considered an outdated disk, since // it needs healing too. - dataErrs[i] = onlineDisk.VerifyFile(bucket, object, partsMetadata[i]) + dataErrs[i] = onlineDisk.VerifyFile(ctx, bucket, object, partsMetadata[i]) case madmin.HealNormalScan: - dataErrs[i] = onlineDisk.CheckParts(bucket, object, partsMetadata[i]) + dataErrs[i] = onlineDisk.CheckParts(ctx, bucket, object, partsMetadata[i]) } if dataErrs[i] == nil { diff --git a/cmd/erasure-healing-common_test.go b/cmd/erasure-healing-common_test.go index 8c4c720ef..d1f9ea2f4 100644 --- a/cmd/erasure-healing-common_test.go +++ b/cmd/erasure-healing-common_test.go @@ -210,7 +210,7 @@ func TestListOnlineDisks(t *testing.T) { // and check if that disk // appears in outDatedDisks. tamperedIndex = index - dErr := erasureDisks[index].DeleteFile(bucket, pathJoin(object, fi.DataDir, "part.1")) + dErr := erasureDisks[index].DeleteFile(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1")) if dErr != nil { t.Fatalf("Test %d: Failed to delete %s - %v", i+1, filepath.Join(object, "part.1"), dErr) diff --git a/cmd/erasure-healing.go b/cmd/erasure-healing.go index 596487bfd..a83194642 100644 --- a/cmd/erasure-healing.go +++ b/cmd/erasure-healing.go @@ -78,7 +78,7 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints afterState[index] = madmin.DriveStateOffline return errDiskNotFound } - if _, serr := storageDisks[index].StatVol(bucket); serr != nil { + if _, serr := storageDisks[index].StatVol(ctx, bucket); serr != nil { if serr == errDiskNotFound { beforeState[index] = madmin.DriveStateOffline afterState[index] = madmin.DriveStateOffline @@ -136,7 +136,7 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints index := index g.Go(func() error { if beforeState[index] == madmin.DriveStateMissing { - makeErr := storageDisks[index].MakeVol(bucket) + makeErr := storageDisks[index].MakeVol(ctx, bucket) if makeErr == nil { afterState[index] = madmin.DriveStateOk } @@ -171,7 +171,7 @@ func listAllBuckets(storageDisks []StorageAPI, healBuckets map[string]VolInfo) ( continue } var volsInfo []VolInfo - volsInfo, err = disk.ListVols() + volsInfo, err = disk.ListVols(context.TODO()) if err != nil { if IsErrIgnored(err, bucketMetadataOpIgnoredErrs...) { continue @@ -472,7 +472,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s } // Attempt a rename now from healed data to final location. - if err = disk.RenameData(minioMetaTmpBucket, tmpID, partsMetadata[i].DataDir, bucket, object); err != nil { + if err = disk.RenameData(ctx, minioMetaTmpBucket, tmpID, partsMetadata[i].DataDir, bucket, object); err != nil { if err != errIsNotRegular && err != errFileNotFound { logger.LogIf(ctx, err) } @@ -525,7 +525,7 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin wg.Add(1) go func(index int, disk StorageAPI) { defer wg.Done() - _ = disk.DeleteFile(bucket, object) + _ = disk.DeleteFile(ctx, bucket, object) }(index, disk) } wg.Wait() @@ -557,7 +557,7 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin for i, err := range errs { if err == errVolumeNotFound || err == errFileNotFound { // Bucket or prefix/directory not found - merr := storageDisks[i].MakeVol(pathJoin(bucket, object)) + merr := storageDisks[i].MakeVol(ctx, pathJoin(bucket, object)) switch merr { case nil, errVolumeExists: hr.After.Drives[i].State = madmin.DriveStateOk @@ -642,7 +642,7 @@ func statAllDirs(ctx context.Context, storageDisks []StorageAPI, bucket, prefix } index := index g.Go(func() error { - entries, err := storageDisks[index].ListDir(bucket, prefix, 1) + entries, err := storageDisks[index].ListDir(ctx, bucket, prefix, 1) if err != nil { return err } diff --git a/cmd/erasure-healing_test.go b/cmd/erasure-healing_test.go index a4f1addea..cc3bfd251 100644 --- a/cmd/erasure-healing_test.go +++ b/cmd/erasure-healing_test.go @@ -66,7 +66,7 @@ func TestHealing(t *testing.T) { } disk := er.getDisks()[0] - fileInfoPreHeal, err := disk.ReadVersion(bucket, object, "") + fileInfoPreHeal, err := disk.ReadVersion(context.Background(), bucket, object, "") if err != nil { t.Fatal(err) } @@ -83,7 +83,7 @@ func TestHealing(t *testing.T) { t.Fatal(err) } - fileInfoPostHeal, err := disk.ReadVersion(bucket, object, "") + fileInfoPostHeal, err := disk.ReadVersion(context.Background(), bucket, object, "") if err != nil { t.Fatal(err) } @@ -102,7 +102,7 @@ func TestHealing(t *testing.T) { // gone down when an object was replaced by a new object. fileInfoOutDated := fileInfoPreHeal fileInfoOutDated.ModTime = time.Now() - err = disk.WriteMetadata(bucket, object, fileInfoOutDated) + err = disk.WriteMetadata(context.Background(), bucket, object, fileInfoOutDated) if err != nil { t.Fatal(err) } @@ -112,7 +112,7 @@ func TestHealing(t *testing.T) { t.Fatal(err) } - fileInfoPostHeal, err = disk.ReadVersion(bucket, object, "") + fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "") if err != nil { t.Fatal(err) } @@ -134,7 +134,7 @@ func TestHealing(t *testing.T) { t.Fatal(err) } // Stat the bucket to make sure that it was created. - _, err = er.getDisks()[0].StatVol(bucket) + _, err = er.getDisks()[0].StatVol(context.Background(), bucket) if err != nil { t.Fatal(err) } @@ -200,7 +200,7 @@ func TestHealObjectCorrupted(t *testing.T) { er := z.zones[0].sets[0] erasureDisks := er.getDisks() firstDisk := erasureDisks[0] - err = firstDisk.DeleteFile(bucket, pathJoin(object, xlStorageFormatFile)) + err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile)) if err != nil { t.Fatalf("Failed to delete a file - %v", err) } @@ -216,16 +216,16 @@ func TestHealObjectCorrupted(t *testing.T) { t.Fatalf("Failed to getLatestFileInfo - %v", err) } - if err = firstDisk.CheckFile(bucket, object); err != nil { + if err = firstDisk.CheckFile(context.Background(), bucket, object); err != nil { t.Errorf("Expected er.meta file to be present but stat failed - %v", err) } - err = firstDisk.DeleteFile(bucket, pathJoin(object, fi.DataDir, "part.1")) + err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1")) if err != nil { t.Errorf("Failure during deleting part.1 - %v", err) } - err = firstDisk.WriteAll(bucket, pathJoin(object, fi.DataDir, "part.1"), bytes.NewReader([]byte{})) + err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bytes.NewReader([]byte{})) if err != nil { t.Errorf("Failure during creating part.1 - %v", err) } @@ -245,13 +245,13 @@ func TestHealObjectCorrupted(t *testing.T) { t.Fatalf("FileInfo not equal after healing") } - err = firstDisk.DeleteFile(bucket, pathJoin(object, fi.DataDir, "part.1")) + err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1")) if err != nil { t.Errorf("Failure during deleting part.1 - %v", err) } bdata := bytes.Repeat([]byte("b"), int(nfi.Size)) - err = firstDisk.WriteAll(bucket, pathJoin(object, fi.DataDir, "part.1"), bytes.NewReader(bdata)) + err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bytes.NewReader(bdata)) if err != nil { t.Errorf("Failure during creating part.1 - %v", err) } @@ -274,7 +274,7 @@ func TestHealObjectCorrupted(t *testing.T) { // Test 4: checks if HealObject returns an error when xl.meta is not found // in more than read quorum number of disks, to create a corrupted situation. for i := 0; i <= len(er.getDisks())/2; i++ { - er.getDisks()[i].DeleteFile(bucket, pathJoin(object, xlStorageFormatFile)) + er.getDisks()[i].DeleteFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile)) } // Try healing now, expect to receive errFileNotFound. @@ -350,7 +350,7 @@ func TestHealObjectErasure(t *testing.T) { t.Fatalf("Failed to complete multipart upload - %v", err) } - err = firstDisk.DeleteFile(bucket, pathJoin(object, xlStorageFormatFile)) + err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile)) if err != nil { t.Fatalf("Failed to delete a file - %v", err) } @@ -360,7 +360,7 @@ func TestHealObjectErasure(t *testing.T) { t.Fatalf("Failed to heal object - %v", err) } - if err = firstDisk.CheckFile(bucket, object); err != nil { + if err = firstDisk.CheckFile(context.Background(), bucket, object); err != nil { t.Errorf("Expected er.meta file to be present but stat failed - %v", err) } @@ -421,7 +421,7 @@ func TestHealEmptyDirectoryErasure(t *testing.T) { z := obj.(*erasureZones) er := z.zones[0].sets[0] firstDisk := er.getDisks()[0] - err = firstDisk.DeleteFile(bucket, object) + err = firstDisk.DeleteFile(context.Background(), bucket, object) if err != nil { t.Fatalf("Failed to delete a file - %v", err) } @@ -433,7 +433,7 @@ func TestHealEmptyDirectoryErasure(t *testing.T) { } // Check if the empty directory is restored in the first disk - _, err = firstDisk.StatVol(pathJoin(bucket, object)) + _, err = firstDisk.StatVol(context.Background(), pathJoin(bucket, object)) if err != nil { t.Fatalf("Expected object to be present but stat failed - %v", err) } diff --git a/cmd/erasure-metadata-utils.go b/cmd/erasure-metadata-utils.go index ca78b9678..6f761e007 100644 --- a/cmd/erasure-metadata-utils.go +++ b/cmd/erasure-metadata-utils.go @@ -126,7 +126,7 @@ func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, ve if disks[index] == nil { return errDiskNotFound } - metadataArray[index], err = disks[index].ReadVersion(bucket, object, versionID) + metadataArray[index], err = disks[index].ReadVersion(ctx, bucket, object, versionID) if err != nil { if err != errFileNotFound && err != errVolumeNotFound && err != errFileVersionNotFound { logger.GetReqInfo(ctx).AppendTags("disk", disks[index].String()) diff --git a/cmd/erasure-metadata.go b/cmd/erasure-metadata.go index 7755fc164..5f7a36b1c 100644 --- a/cmd/erasure-metadata.go +++ b/cmd/erasure-metadata.go @@ -273,7 +273,7 @@ func renameFileInfo(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry if disks[index] == nil { return errDiskNotFound } - if err := disks[index].RenameData(srcBucket, srcEntry, "", dstBucket, dstEntry); err != nil { + if err := disks[index].RenameData(ctx, srcBucket, srcEntry, "", dstBucket, dstEntry); err != nil { if !IsErrIgnored(err, ignoredErr...) { return err } @@ -304,7 +304,7 @@ func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, bucket, prefix } // Pick one FileInfo for a disk at index. files[index].Erasure.Index = index + 1 - return disks[index].WriteMetadata(bucket, prefix, files[index]) + return disks[index].WriteMetadata(ctx, bucket, prefix, files[index]) }, index) } diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go index 09bb6f3c5..45162fe94 100644 --- a/cmd/erasure-multipart.go +++ b/cmd/erasure-multipart.go @@ -63,7 +63,7 @@ func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir stri // Ignoring failure to remove parts that weren't present in CompleteMultipartUpload // requests. xl.meta is the authoritative source of truth on which parts constitute // the object. The presence of parts that don't belong in the object doesn't affect correctness. - _ = storageDisks[index].DeleteFile(minioMetaMultipartBucket, curpartPath) + _ = storageDisks[index].DeleteFile(context.TODO(), minioMetaMultipartBucket, curpartPath) return nil }, index) } @@ -99,18 +99,18 @@ func (er erasureObjects) cleanupStaleMultipartUploads(ctx context.Context, clean // Remove the old multipart uploads on the given disk. func (er erasureObjects) cleanupStaleMultipartUploadsOnDisk(ctx context.Context, disk StorageAPI, expiry time.Duration) { now := time.Now() - shaDirs, err := disk.ListDir(minioMetaMultipartBucket, "", -1) + shaDirs, err := disk.ListDir(ctx, minioMetaMultipartBucket, "", -1) if err != nil { return } for _, shaDir := range shaDirs { - uploadIDDirs, err := disk.ListDir(minioMetaMultipartBucket, shaDir, -1) + uploadIDDirs, err := disk.ListDir(ctx, minioMetaMultipartBucket, shaDir, -1) if err != nil { continue } for _, uploadIDDir := range uploadIDDirs { uploadIDPath := pathJoin(shaDir, uploadIDDir) - fi, err := disk.ReadVersion(minioMetaMultipartBucket, uploadIDPath, "") + fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, uploadIDPath, "") if err != nil { continue } @@ -139,7 +139,7 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec if disk == nil { continue } - uploadIDs, err = disk.ListDir(minioMetaMultipartBucket, er.getMultipartSHADir(bucket, object), -1) + uploadIDs, err = disk.ListDir(ctx, minioMetaMultipartBucket, er.getMultipartSHADir(bucket, object), -1) if err != nil { if err == errDiskNotFound { continue @@ -172,7 +172,7 @@ retry: if populatedUploadIds.Contains(uploadID) { continue } - fi, err := disk.ReadVersion(minioMetaMultipartBucket, pathJoin(er.getUploadIDDir(bucket, object, uploadID)), "") + fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, pathJoin(er.getUploadIDDir(bucket, object, uploadID)), "") if err != nil { if err == errDiskNotFound || err == errFileNotFound { goto retry diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 75d248040..676c02091 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -49,7 +49,7 @@ func (er erasureObjects) putObjectDir(ctx context.Context, bucket, object string } index := index g.Go(func() error { - err := storageDisks[index].MakeVol(pathJoin(bucket, object)) + err := storageDisks[index].MakeVol(ctx, pathJoin(bucket, object)) if err != nil && err != errVolumeExists { return err } @@ -362,7 +362,7 @@ func (er erasureObjects) getObjectInfoDir(ctx context.Context, bucket, object st index := index g.Go(func() error { // Check if 'prefix' is an object on this 'disk'. - entries, err := storageDisks[index].ListDir(bucket, object, 1) + entries, err := storageDisks[index].ListDir(ctx, bucket, object, 1) if err != nil { return err } @@ -459,7 +459,7 @@ func undoRename(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry str index := index g.Go(func() error { if errs[index] == nil { - _ = disks[index].RenameFile(dstBucket, dstEntry, srcBucket, srcEntry) + _ = disks[index].RenameFile(context.TODO(), dstBucket, dstEntry, srcBucket, srcEntry) } return nil }, index) @@ -479,7 +479,7 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, da if disks[index] == nil { return errDiskNotFound } - if err := disks[index].RenameData(srcBucket, srcEntry, dataDir, dstBucket, dstEntry); err != nil { + if err := disks[index].RenameData(ctx, srcBucket, srcEntry, dataDir, dstBucket, dstEntry); err != nil { if !IsErrIgnored(err, ignoredErr...) { return err } @@ -504,7 +504,7 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, da ug.Go(func() error { // Undo all the partial rename operations. if errs[index] == nil { - _ = disks[index].RenameData(dstBucket, dstEntry, dataDir, srcBucket, srcEntry) + _ = disks[index].RenameData(context.Background(), dstBucket, dstEntry, dataDir, srcBucket, srcEntry) } return nil }, index) @@ -532,7 +532,7 @@ func rename(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBuc if disks[index] == nil { return errDiskNotFound } - if err := disks[index].RenameFile(srcBucket, srcEntry, dstBucket, dstEntry); err != nil { + if err := disks[index].RenameFile(ctx, srcBucket, srcEntry, dstBucket, dstEntry); err != nil { if !IsErrIgnored(err, ignoredErr...) { return err } @@ -776,7 +776,7 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object if disks[index] == nil { return errDiskNotFound } - err := disks[index].DeleteVersion(bucket, object, fi) + err := disks[index].DeleteVersion(ctx, bucket, object, fi) if err != nil && err != errVolumeNotFound { return err } @@ -899,7 +899,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec wg.Add(1) go func(index int, disk StorageAPI) { defer wg.Done() - delObjErrs[index] = disk.DeleteVersions(bucket, versions) + delObjErrs[index] = disk.DeleteVersions(ctx, bucket, versions) }(index, disk) } diff --git a/cmd/erasure-sets.go b/cmd/erasure-sets.go index 1be10ace5..e5f6e717e 100644 --- a/cmd/erasure-sets.go +++ b/cmd/erasure-sets.go @@ -140,7 +140,7 @@ func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) { // Close the internal connection to avoid connection leaks. disk.Close() if errors.Is(err, errUnformattedDisk) { - info, derr := disk.DiskInfo() + info, derr := disk.DiskInfo(context.TODO()) if derr != nil && info.RootDisk { return nil, nil, fmt.Errorf("Disk: %s returned %w but its a root disk refusing to use it", disk, derr) // make sure to '%w' to wrap the error @@ -981,7 +981,7 @@ func (s *erasureSets) startMergeWalksVersionsN(ctx context.Context, bucket, pref // Disk can be offline continue } - entryCh, err := disk.WalkVersions(bucket, prefix, marker, recursive, endWalkCh) + entryCh, err := disk.WalkVersions(GlobalContext, bucket, prefix, marker, recursive, endWalkCh) if err != nil { logger.LogIf(ctx, err) // Disk walk returned error, ignore it. @@ -1012,7 +1012,7 @@ func (s *erasureSets) startMergeWalksN(ctx context.Context, bucket, prefix, mark // Disk can be offline continue } - entryCh, err := disk.Walk(bucket, prefix, marker, recursive, endWalkCh) + entryCh, err := disk.Walk(GlobalContext, bucket, prefix, marker, recursive, endWalkCh) if err != nil { // Disk walk returned error, ignore it. continue @@ -1042,7 +1042,7 @@ func (s *erasureSets) startSplunkMergeWalksN(ctx context.Context, bucket, prefix // Disk can be offline continue } - entryCh, err := disk.WalkSplunk(bucket, prefix, marker, endWalkCh) + entryCh, err := disk.WalkSplunk(GlobalContext, bucket, prefix, marker, endWalkCh) if err != nil { // Disk walk returned error, ignore it. continue @@ -1300,7 +1300,7 @@ func getHealDiskInfos(storageDisks []StorageAPI, errs []error) ([]DiskInfo, []er return errDiskNotFound } var err error - infos[index], err = storageDisks[index].DiskInfo() + infos[index], err = storageDisks[index].DiskInfo(context.TODO()) return err }, index) } diff --git a/cmd/erasure.go b/cmd/erasure.go index 755ebcd52..a59d9ad42 100644 --- a/cmd/erasure.go +++ b/cmd/erasure.go @@ -139,7 +139,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di // Storage disk is empty, perhaps ignored disk or not available. return errDiskNotFound } - info, err := disks[index].DiskInfo() + info, err := disks[index].DiskInfo(context.TODO()) if err != nil { if !IsErr(err, baseErrs...) { reqInfo := (&logger.ReqInfo{}).AppendTags("disk", disks[index].String()) @@ -282,7 +282,7 @@ func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []Buc if d == nil || !d.IsOnline() { continue } - di, err := d.DiskInfo() + di, err := d.DiskInfo(ctx) if err != nil { logger.LogIf(ctx, err) continue diff --git a/cmd/erasure_test.go b/cmd/erasure_test.go index 71a88d574..924d72089 100644 --- a/cmd/erasure_test.go +++ b/cmd/erasure_test.go @@ -134,7 +134,7 @@ func newErasureTestSetup(dataBlocks int, parityBlocks int, blockSize int64) (*er if err != nil { return nil, err } - err = disks[i].MakeVol("testbucket") + err = disks[i].MakeVol(context.Background(), "testbucket") if err != nil { return nil, err } diff --git a/cmd/format-erasure.go b/cmd/format-erasure.go index bccebb4ef..924f6ae43 100644 --- a/cmd/format-erasure.go +++ b/cmd/format-erasure.go @@ -355,15 +355,15 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3) error { tmpFormat := mustGetUUID() // Purge any existing temporary file, okay to ignore errors here. - defer disk.DeleteFile(minioMetaBucket, tmpFormat) + defer disk.DeleteFile(context.TODO(), minioMetaBucket, tmpFormat) // write to unique file. - if err = disk.WriteAll(minioMetaBucket, tmpFormat, bytes.NewReader(formatBytes)); err != nil { + if err = disk.WriteAll(context.TODO(), minioMetaBucket, tmpFormat, bytes.NewReader(formatBytes)); err != nil { return err } // Rename file `uuid.json` --> `format.json`. - if err = disk.RenameFile(minioMetaBucket, tmpFormat, minioMetaBucket, formatConfigFile); err != nil { + if err = disk.RenameFile(context.TODO(), minioMetaBucket, tmpFormat, minioMetaBucket, formatConfigFile); err != nil { return err } @@ -392,13 +392,13 @@ func isHiddenDirectories(vols ...VolInfo) bool { // loadFormatErasure - loads format.json from disk. func loadFormatErasure(disk StorageAPI) (format *formatErasureV3, err error) { - buf, err := disk.ReadAll(minioMetaBucket, formatConfigFile) + buf, err := disk.ReadAll(context.TODO(), minioMetaBucket, formatConfigFile) if err != nil { // 'file not found' and 'volume not found' as // same. 'volume not found' usually means its a fresh disk. if err == errFileNotFound || err == errVolumeNotFound { var vols []VolInfo - vols, err = disk.ListVols() + vols, err = disk.ListVols(context.TODO()) if err != nil { return nil, err } @@ -862,7 +862,7 @@ func makeFormatErasureMetaVolumes(disk StorageAPI) error { return errDiskNotFound } // Attempt to create MinIO internal buckets. - return disk.MakeVolBulk(minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, dataUsageBucket) + return disk.MakeVolBulk(context.TODO(), minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, dataUsageBucket) } // Get all UUIDs which are present in reference format should diff --git a/cmd/global-heal.go b/cmd/global-heal.go index 7909c2e7a..a0e04720b 100644 --- a/cmd/global-heal.go +++ b/cmd/global-heal.go @@ -131,7 +131,7 @@ func healErasureSet(ctx context.Context, setIndex int, xlObj *erasureObjects, se continue } - entryCh, err := disk.WalkVersions(bucket.Name, "", "", true, ctx.Done()) + entryCh, err := disk.WalkVersions(ctx, bucket.Name, "", "", true, ctx.Done()) if err != nil { // Disk walk returned error, ignore it. continue diff --git a/cmd/lock-rest-client.go b/cmd/lock-rest-client.go index 1c3d2d174..366b23d8b 100644 --- a/cmd/lock-rest-client.go +++ b/cmd/lock-rest-client.go @@ -63,7 +63,7 @@ func (client *lockRESTClient) callWithContext(ctx context.Context, method string values = make(url.Values) } - respBody, err = client.restClient.Call(method, values, body, length) + respBody, err = client.restClient.Call(ctx, method, values, body, length) if err == nil { return respBody, nil } @@ -158,7 +158,7 @@ func newlockRESTClient(endpoint Endpoint) *lockRESTClient { ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout) // Instantiate a new rest client for healthcheck // to avoid recursive healthCheckFn() - respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).CallWithContext(ctx, lockRESTMethodHealth, nil, nil, -1) + respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).Call(ctx, lockRESTMethodHealth, nil, nil, -1) xhttp.DrainBody(respBody) cancel() var ne *rest.NetworkError diff --git a/cmd/naughty-disk_test.go b/cmd/naughty-disk_test.go index 98601250b..8c85b4a1e 100644 --- a/cmd/naughty-disk_test.go +++ b/cmd/naughty-disk_test.go @@ -94,139 +94,139 @@ func (d *naughtyDisk) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageC return d.disk.CrawlAndGetDataUsage(ctx, cache) } -func (d *naughtyDisk) DiskInfo() (info DiskInfo, err error) { +func (d *naughtyDisk) DiskInfo(ctx context.Context) (info DiskInfo, err error) { if err := d.calcError(); err != nil { return info, err } - return d.disk.DiskInfo() + return d.disk.DiskInfo(ctx) } -func (d *naughtyDisk) MakeVolBulk(volumes ...string) (err error) { +func (d *naughtyDisk) MakeVolBulk(ctx context.Context, volumes ...string) (err error) { if err := d.calcError(); err != nil { return err } - return d.disk.MakeVolBulk(volumes...) + return d.disk.MakeVolBulk(ctx, volumes...) } -func (d *naughtyDisk) MakeVol(volume string) (err error) { +func (d *naughtyDisk) MakeVol(ctx context.Context, volume string) (err error) { if err := d.calcError(); err != nil { return err } - return d.disk.MakeVol(volume) + return d.disk.MakeVol(ctx, volume) } -func (d *naughtyDisk) ListVols() (vols []VolInfo, err error) { +func (d *naughtyDisk) ListVols(ctx context.Context) (vols []VolInfo, err error) { if err := d.calcError(); err != nil { return nil, err } - return d.disk.ListVols() + return d.disk.ListVols(ctx) } -func (d *naughtyDisk) StatVol(volume string) (volInfo VolInfo, err error) { +func (d *naughtyDisk) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) { if err := d.calcError(); err != nil { return VolInfo{}, err } - return d.disk.StatVol(volume) + return d.disk.StatVol(ctx, volume) } -func (d *naughtyDisk) DeleteVol(volume string, forceDelete bool) (err error) { +func (d *naughtyDisk) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) { if err := d.calcError(); err != nil { return err } - return d.disk.DeleteVol(volume, forceDelete) + return d.disk.DeleteVol(ctx, volume, forceDelete) } -func (d *naughtyDisk) WalkSplunk(volume, path, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error) { +func (d *naughtyDisk) WalkSplunk(ctx context.Context, volume, dirPath, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error) { if err := d.calcError(); err != nil { return nil, err } - return d.disk.WalkSplunk(volume, path, marker, endWalkCh) + return d.disk.WalkSplunk(ctx, volume, dirPath, marker, endWalkCh) } -func (d *naughtyDisk) WalkVersions(volume, path, marker string, recursive bool, endWalkVersionsCh <-chan struct{}) (chan FileInfoVersions, error) { +func (d *naughtyDisk) WalkVersions(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error) { if err := d.calcError(); err != nil { return nil, err } - return d.disk.WalkVersions(volume, path, marker, recursive, endWalkVersionsCh) + return d.disk.WalkVersions(ctx, volume, dirPath, marker, recursive, endWalkCh) } -func (d *naughtyDisk) Walk(volume, path, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) { +func (d *naughtyDisk) Walk(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) { if err := d.calcError(); err != nil { return nil, err } - return d.disk.Walk(volume, path, marker, recursive, endWalkCh) + return d.disk.Walk(ctx, volume, dirPath, marker, recursive, endWalkCh) } -func (d *naughtyDisk) ListDir(volume, path string, count int) (entries []string, err error) { +func (d *naughtyDisk) ListDir(ctx context.Context, volume, dirPath string, count int) (entries []string, err error) { if err := d.calcError(); err != nil { return []string{}, err } - return d.disk.ListDir(volume, path, count) + return d.disk.ListDir(ctx, volume, dirPath, count) } -func (d *naughtyDisk) ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) { +func (d *naughtyDisk) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) { if err := d.calcError(); err != nil { return 0, err } - return d.disk.ReadFile(volume, path, offset, buf, verifier) + return d.disk.ReadFile(ctx, volume, path, offset, buf, verifier) } -func (d *naughtyDisk) ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error) { +func (d *naughtyDisk) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) { if err := d.calcError(); err != nil { return nil, err } - return d.disk.ReadFileStream(volume, path, offset, length) + return d.disk.ReadFileStream(ctx, volume, path, offset, length) } -func (d *naughtyDisk) CreateFile(volume, path string, size int64, reader io.Reader) error { +func (d *naughtyDisk) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error { if err := d.calcError(); err != nil { return err } - return d.disk.CreateFile(volume, path, size, reader) + return d.disk.CreateFile(ctx, volume, path, size, reader) } -func (d *naughtyDisk) AppendFile(volume, path string, buf []byte) error { +func (d *naughtyDisk) AppendFile(ctx context.Context, volume string, path string, buf []byte) error { if err := d.calcError(); err != nil { return err } - return d.disk.AppendFile(volume, path, buf) + return d.disk.AppendFile(ctx, volume, path, buf) } -func (d *naughtyDisk) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath string) error { +func (d *naughtyDisk) RenameData(ctx context.Context, srcVolume, srcPath, dataDir, dstVolume, dstPath string) error { if err := d.calcError(); err != nil { return err } - return d.disk.RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath) + return d.disk.RenameData(ctx, srcVolume, srcPath, dataDir, dstVolume, dstPath) } -func (d *naughtyDisk) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error { +func (d *naughtyDisk) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error { if err := d.calcError(); err != nil { return err } - return d.disk.RenameFile(srcVolume, srcPath, dstVolume, dstPath) + return d.disk.RenameFile(ctx, srcVolume, srcPath, dstVolume, dstPath) } -func (d *naughtyDisk) CheckParts(volume string, path string, fi FileInfo) (err error) { +func (d *naughtyDisk) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (err error) { if err := d.calcError(); err != nil { return err } - return d.disk.CheckParts(volume, path, fi) + return d.disk.CheckParts(ctx, volume, path, fi) } -func (d *naughtyDisk) CheckFile(volume string, path string) (err error) { +func (d *naughtyDisk) CheckFile(ctx context.Context, volume string, path string) (err error) { if err := d.calcError(); err != nil { return err } - return d.disk.CheckFile(volume, path) + return d.disk.CheckFile(ctx, volume, path) } -func (d *naughtyDisk) DeleteFile(volume string, path string) (err error) { +func (d *naughtyDisk) DeleteFile(ctx context.Context, volume string, path string) (err error) { if err := d.calcError(); err != nil { return err } - return d.disk.DeleteFile(volume, path) + return d.disk.DeleteFile(ctx, volume, path) } -func (d *naughtyDisk) DeleteVersions(volume string, versions []FileInfo) []error { +func (d *naughtyDisk) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) []error { if err := d.calcError(); err != nil { errs := make([]error, len(versions)) for i := range errs { @@ -234,47 +234,47 @@ func (d *naughtyDisk) DeleteVersions(volume string, versions []FileInfo) []error } return errs } - return d.disk.DeleteVersions(volume, versions) + return d.disk.DeleteVersions(ctx, volume, versions) } -func (d *naughtyDisk) WriteMetadata(volume, path string, fi FileInfo) (err error) { +func (d *naughtyDisk) WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) (err error) { if err := d.calcError(); err != nil { return err } - return d.disk.WriteMetadata(volume, path, fi) + return d.disk.WriteMetadata(ctx, volume, path, fi) } -func (d *naughtyDisk) DeleteVersion(volume string, path string, fi FileInfo) (err error) { +func (d *naughtyDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo) (err error) { if err := d.calcError(); err != nil { return err } - return d.disk.DeleteVersion(volume, path, fi) + return d.disk.DeleteVersion(ctx, volume, path, fi) } -func (d *naughtyDisk) ReadVersion(volume string, path string, versionID string) (fi FileInfo, err error) { +func (d *naughtyDisk) ReadVersion(ctx context.Context, volume, path, versionID string) (fi FileInfo, err error) { if err := d.calcError(); err != nil { return FileInfo{}, err } - return d.disk.ReadVersion(volume, path, versionID) + return d.disk.ReadVersion(ctx, volume, path, versionID) } -func (d *naughtyDisk) WriteAll(volume string, path string, reader io.Reader) (err error) { +func (d *naughtyDisk) WriteAll(ctx context.Context, volume string, path string, reader io.Reader) (err error) { if err := d.calcError(); err != nil { return err } - return d.disk.WriteAll(volume, path, reader) + return d.disk.WriteAll(ctx, volume, path, reader) } -func (d *naughtyDisk) ReadAll(volume string, path string) (buf []byte, err error) { +func (d *naughtyDisk) ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) { if err := d.calcError(); err != nil { return nil, err } - return d.disk.ReadAll(volume, path) + return d.disk.ReadAll(ctx, volume, path) } -func (d *naughtyDisk) VerifyFile(volume, path string, fi FileInfo) error { +func (d *naughtyDisk) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error { if err := d.calcError(); err != nil { return err } - return d.disk.VerifyFile(volume, path, fi) + return d.disk.VerifyFile(ctx, volume, path, fi) } diff --git a/cmd/object-api-common.go b/cmd/object-api-common.go index 295f810f5..551e20ec2 100644 --- a/cmd/object-api-common.go +++ b/cmd/object-api-common.go @@ -99,7 +99,7 @@ func cleanupDir(ctx context.Context, storage StorageAPI, volume, dirPath string) delFunc = func(entryPath string) error { if !HasSuffix(entryPath, SlashSeparator) { // Delete the file entry. - err := storage.DeleteFile(volume, entryPath) + err := storage.DeleteFile(ctx, volume, entryPath) if err != errDiskNotFound && err != errUnformattedDisk { logger.LogIf(ctx, err) } @@ -107,7 +107,7 @@ func cleanupDir(ctx context.Context, storage StorageAPI, volume, dirPath string) } // If it's a directory, list and call delFunc() for each entry. - entries, err := storage.ListDir(volume, entryPath, -1) + entries, err := storage.ListDir(ctx, volume, entryPath, -1) // If entryPath prefix never existed, safe to ignore. if err == errFileNotFound { return nil @@ -120,7 +120,7 @@ func cleanupDir(ctx context.Context, storage StorageAPI, volume, dirPath string) // Entry path is empty, just delete it. if len(entries) == 0 { - err = storage.DeleteFile(volume, entryPath) + err = storage.DeleteFile(ctx, volume, entryPath) if err != errDiskNotFound && err != errUnformattedDisk { logger.LogIf(ctx, err) } diff --git a/cmd/peer-rest-client.go b/cmd/peer-rest-client.go index 745c253e6..236ffc988 100644 --- a/cmd/peer-rest-client.go +++ b/cmd/peer-rest-client.go @@ -62,7 +62,7 @@ func (client *peerRESTClient) callWithContext(ctx context.Context, method string values = make(url.Values) } - respBody, err = client.restClient.CallWithContext(ctx, method, values, body, length) + respBody, err = client.restClient.Call(ctx, method, values, body, length) if err == nil { return respBody, nil } @@ -882,7 +882,7 @@ func newPeerRESTClient(peer *xnet.Host) *peerRESTClient { ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout) // Instantiate a new rest client for healthcheck // to avoid recursive healthCheckFn() - respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).CallWithContext(ctx, peerRESTMethodHealth, nil, nil, -1) + respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).Call(ctx, peerRESTMethodHealth, nil, nil, -1) xhttp.DrainBody(respBody) cancel() var ne *rest.NetworkError diff --git a/cmd/rest/client.go b/cmd/rest/client.go index dbbf03d6a..8dc871c4c 100644 --- a/cmd/rest/client.go +++ b/cmd/rest/client.go @@ -95,8 +95,8 @@ func (e restError) Timeout() bool { return true } -// CallWithContext - make a REST call with context. -func (c *Client) CallWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (reply io.ReadCloser, err error) { +// Call - make a REST call with context. +func (c *Client) Call(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (reply io.ReadCloser, err error) { if !c.IsOnline() { return nil, &NetworkError{Err: &url.Error{Op: method, URL: c.url.String(), Err: restError("remote server offline")}} } @@ -153,12 +153,6 @@ func (c *Client) CallWithContext(ctx context.Context, method string, values url. return resp.Body, nil } -// Call - make a REST call. -func (c *Client) Call(method string, values url.Values, body io.Reader, length int64) (reply io.ReadCloser, err error) { - ctx := context.Background() - return c.CallWithContext(ctx, method, values, body, length) -} - // Close closes all idle connections of the underlying http client func (c *Client) Close() { atomic.StoreInt32(&c.connected, closed) diff --git a/cmd/storage-interface.go b/cmd/storage-interface.go index 1d0f3e4ef..b47a927ce 100644 --- a/cmd/storage-interface.go +++ b/cmd/storage-interface.go @@ -35,47 +35,47 @@ type StorageAPI interface { GetDiskID() (string, error) SetDiskID(id string) - DiskInfo() (info DiskInfo, err error) + DiskInfo(ctx context.Context) (info DiskInfo, err error) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache) (dataUsageCache, error) // Volume operations. - MakeVol(volume string) (err error) - MakeVolBulk(volumes ...string) (err error) - ListVols() (vols []VolInfo, err error) - StatVol(volume string) (vol VolInfo, err error) - DeleteVol(volume string, forceDelete bool) (err error) + MakeVol(ctx context.Context, volume string) (err error) + MakeVolBulk(ctx context.Context, volumes ...string) (err error) + ListVols(ctx context.Context) (vols []VolInfo, err error) + StatVol(ctx context.Context, volume string) (vol VolInfo, err error) + DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) // WalkVersions in sorted order directly on disk. - WalkVersions(volume, dirPath string, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error) + WalkVersions(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error) // Walk in sorted order directly on disk. - Walk(volume, dirPath string, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) + Walk(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) // Walk in sorted order directly on disk. - WalkSplunk(volume, dirPath string, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error) + WalkSplunk(ctx context.Context, volume, dirPath, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error) // Metadata operations - DeleteVersion(volume, path string, fi FileInfo) error - DeleteVersions(volume string, versions []FileInfo) []error - WriteMetadata(volume, path string, fi FileInfo) error - ReadVersion(volume, path, versionID string) (FileInfo, error) - RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath string) error + DeleteVersion(ctx context.Context, volume, path string, fi FileInfo) error + DeleteVersions(ctx context.Context, volume string, versions []FileInfo) []error + WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) error + ReadVersion(ctx context.Context, volume, path, versionID string) (FileInfo, error) + RenameData(ctx context.Context, srcVolume, srcPath, dataDir, dstVolume, dstPath string) error // File operations. - ListDir(volume, dirPath string, count int) ([]string, error) - ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) - AppendFile(volume string, path string, buf []byte) (err error) - CreateFile(volume, path string, size int64, reader io.Reader) error - ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error) - RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error - CheckParts(volume string, path string, fi FileInfo) error - CheckFile(volume string, path string) (err error) - DeleteFile(volume string, path string) (err error) - VerifyFile(volume, path string, fi FileInfo) error + ListDir(ctx context.Context, volume, dirPath string, count int) ([]string, error) + ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) + AppendFile(ctx context.Context, volume string, path string, buf []byte) (err error) + CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error + ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) + RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error + CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error + CheckFile(ctx context.Context, volume string, path string) (err error) + DeleteFile(ctx context.Context, volume string, path string) (err error) + VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error // Write all data, syncs the data to disk. - WriteAll(volume string, path string, reader io.Reader) (err error) + WriteAll(ctx context.Context, volume string, path string, reader io.Reader) (err error) // Read all. - ReadAll(volume string, path string) (buf []byte, err error) + ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) } // storageReader is an io.Reader view of a disk @@ -86,7 +86,7 @@ type storageReader struct { } func (r *storageReader) Read(p []byte) (n int, err error) { - nn, err := r.storage.ReadFile(r.volume, r.path, r.offset, p, nil) + nn, err := r.storage.ReadFile(context.TODO(), r.volume, r.path, r.offset, p, nil) r.offset += nn n = int(nn) @@ -103,7 +103,7 @@ type storageWriter struct { } func (w *storageWriter) Write(p []byte) (n int, err error) { - err = w.storage.AppendFile(w.volume, w.path, p) + err = w.storage.AppendFile(context.TODO(), w.volume, w.path, p) if err == nil { n = len(p) } diff --git a/cmd/storage-rest-client.go b/cmd/storage-rest-client.go index ee089e6a2..8ea0d91e6 100644 --- a/cmd/storage-rest-client.go +++ b/cmd/storage-rest-client.go @@ -119,7 +119,7 @@ type storageRESTClient struct { // Wrapper to restClient.Call to handle network errors, in case of network error the connection is makred disconnected // permanently. The only way to restore the storage connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints() // after verifying format.json -func (client *storageRESTClient) call(method string, values url.Values, body io.Reader, length int64) (io.ReadCloser, error) { +func (client *storageRESTClient) call(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (io.ReadCloser, error) { if !client.IsOnline() { return nil, errDiskNotFound } @@ -127,7 +127,7 @@ func (client *storageRESTClient) call(method string, values url.Values, body io. values = make(url.Values) } values.Set(storageRESTDiskID, client.diskID) - respBody, err := client.restClient.Call(method, values, body, length) + respBody, err := client.restClient.Call(ctx, method, values, body, length) if err == nil { return respBody, nil } @@ -157,9 +157,7 @@ func (client *storageRESTClient) Hostname() string { func (client *storageRESTClient) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache) (dataUsageCache, error) { b := cache.serialize() - respBody, err := client.call(storageRESTMethodCrawlAndGetDataUsage, - url.Values{}, - bytes.NewBuffer(b), int64(len(b))) + respBody, err := client.call(ctx, storageRESTMethodCrawlAndGetDataUsage, url.Values{}, bytes.NewBuffer(b), int64(len(b))) defer http.DrainBody(respBody) if err != nil { return cache, err @@ -185,8 +183,8 @@ func (client *storageRESTClient) SetDiskID(id string) { } // DiskInfo - fetch disk information for a remote disk. -func (client *storageRESTClient) DiskInfo() (info DiskInfo, err error) { - respBody, err := client.call(storageRESTMethodDiskInfo, nil, nil, -1) +func (client *storageRESTClient) DiskInfo(ctx context.Context) (info DiskInfo, err error) { + respBody, err := client.call(ctx, storageRESTMethodDiskInfo, nil, nil, -1) if err != nil { return } @@ -202,81 +200,81 @@ func (client *storageRESTClient) DiskInfo() (info DiskInfo, err error) { } // MakeVolBulk - create multiple volumes in a bulk operation. -func (client *storageRESTClient) MakeVolBulk(volumes ...string) (err error) { +func (client *storageRESTClient) MakeVolBulk(ctx context.Context, volumes ...string) (err error) { values := make(url.Values) values.Set(storageRESTVolumes, strings.Join(volumes, ",")) - respBody, err := client.call(storageRESTMethodMakeVolBulk, values, nil, -1) + respBody, err := client.call(ctx, storageRESTMethodMakeVolBulk, values, nil, -1) defer http.DrainBody(respBody) return err } // MakeVol - create a volume on a remote disk. -func (client *storageRESTClient) MakeVol(volume string) (err error) { +func (client *storageRESTClient) MakeVol(ctx context.Context, volume string) (err error) { values := make(url.Values) values.Set(storageRESTVolume, volume) - respBody, err := client.call(storageRESTMethodMakeVol, values, nil, -1) + respBody, err := client.call(ctx, storageRESTMethodMakeVol, values, nil, -1) defer http.DrainBody(respBody) return err } // ListVols - List all volumes on a remote disk. -func (client *storageRESTClient) ListVols() (volinfo []VolInfo, err error) { - respBody, err := client.call(storageRESTMethodListVols, nil, nil, -1) +func (client *storageRESTClient) ListVols(ctx context.Context) (vols []VolInfo, err error) { + respBody, err := client.call(ctx, storageRESTMethodListVols, nil, nil, -1) if err != nil { return } defer http.DrainBody(respBody) - err = gob.NewDecoder(respBody).Decode(&volinfo) - return volinfo, err + err = gob.NewDecoder(respBody).Decode(&vols) + return vols, err } // StatVol - get volume info over the network. -func (client *storageRESTClient) StatVol(volume string) (volInfo VolInfo, err error) { +func (client *storageRESTClient) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) { values := make(url.Values) values.Set(storageRESTVolume, volume) - respBody, err := client.call(storageRESTMethodStatVol, values, nil, -1) + respBody, err := client.call(ctx, storageRESTMethodStatVol, values, nil, -1) if err != nil { return } defer http.DrainBody(respBody) - err = gob.NewDecoder(respBody).Decode(&volInfo) - return volInfo, err + err = gob.NewDecoder(respBody).Decode(&vol) + return vol, err } // DeleteVol - Deletes a volume over the network. -func (client *storageRESTClient) DeleteVol(volume string, forceDelete bool) (err error) { +func (client *storageRESTClient) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) { values := make(url.Values) values.Set(storageRESTVolume, volume) if forceDelete { values.Set(storageRESTForceDelete, "true") } - respBody, err := client.call(storageRESTMethodDeleteVol, values, nil, -1) + respBody, err := client.call(ctx, storageRESTMethodDeleteVol, values, nil, -1) defer http.DrainBody(respBody) return err } // AppendFile - append to a file. -func (client *storageRESTClient) AppendFile(volume, path string, buffer []byte) error { +func (client *storageRESTClient) AppendFile(ctx context.Context, volume string, path string, buf []byte) error { values := make(url.Values) values.Set(storageRESTVolume, volume) values.Set(storageRESTFilePath, path) - reader := bytes.NewReader(buffer) - respBody, err := client.call(storageRESTMethodAppendFile, values, reader, -1) + reader := bytes.NewReader(buf) + respBody, err := client.call(ctx, storageRESTMethodAppendFile, values, reader, -1) defer http.DrainBody(respBody) return err } -func (client *storageRESTClient) CreateFile(volume, path string, length int64, r io.Reader) error { +func (client *storageRESTClient) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error { values := make(url.Values) values.Set(storageRESTVolume, volume) values.Set(storageRESTFilePath, path) - values.Set(storageRESTLength, strconv.Itoa(int(length))) - respBody, err := client.call(storageRESTMethodCreateFile, values, ioutil.NopCloser(r), length) + values.Set(storageRESTLength, strconv.Itoa(int(size))) + respBody, err := client.call(ctx, storageRESTMethodCreateFile, values, ioutil.NopCloser(reader), size) defer http.DrainBody(respBody) return err } -func (client *storageRESTClient) WriteMetadata(volume, path string, fi FileInfo) error { +func (client *storageRESTClient) WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) error { values := make(url.Values) values.Set(storageRESTVolume, volume) values.Set(storageRESTFilePath, path) @@ -286,12 +284,12 @@ func (client *storageRESTClient) WriteMetadata(volume, path string, fi FileInfo) return err } - respBody, err := client.call(storageRESTMethodWriteMetadata, values, &reader, -1) + respBody, err := client.call(ctx, storageRESTMethodWriteMetadata, values, &reader, -1) defer http.DrainBody(respBody) return err } -func (client *storageRESTClient) DeleteVersion(volume, path string, fi FileInfo) error { +func (client *storageRESTClient) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo) error { values := make(url.Values) values.Set(storageRESTVolume, volume) values.Set(storageRESTFilePath, path) @@ -301,33 +299,33 @@ func (client *storageRESTClient) DeleteVersion(volume, path string, fi FileInfo) return err } - respBody, err := client.call(storageRESTMethodDeleteVersion, values, &buffer, -1) + respBody, err := client.call(ctx, storageRESTMethodDeleteVersion, values, &buffer, -1) defer http.DrainBody(respBody) return err } // WriteAll - write all data to a file. -func (client *storageRESTClient) WriteAll(volume, path string, reader io.Reader) error { +func (client *storageRESTClient) WriteAll(ctx context.Context, volume string, path string, reader io.Reader) error { values := make(url.Values) values.Set(storageRESTVolume, volume) values.Set(storageRESTFilePath, path) - respBody, err := client.call(storageRESTMethodWriteAll, values, reader, -1) + respBody, err := client.call(ctx, storageRESTMethodWriteAll, values, reader, -1) defer http.DrainBody(respBody) return err } // CheckFile - stat a file metadata. -func (client *storageRESTClient) CheckFile(volume, path string) error { +func (client *storageRESTClient) CheckFile(ctx context.Context, volume string, path string) error { values := make(url.Values) values.Set(storageRESTVolume, volume) values.Set(storageRESTFilePath, path) - respBody, err := client.call(storageRESTMethodCheckFile, values, nil, -1) + respBody, err := client.call(ctx, storageRESTMethodCheckFile, values, nil, -1) defer http.DrainBody(respBody) return err } // CheckParts - stat all file parts. -func (client *storageRESTClient) CheckParts(volume, path string, fi FileInfo) error { +func (client *storageRESTClient) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error { values := make(url.Values) values.Set(storageRESTVolume, volume) values.Set(storageRESTFilePath, path) @@ -338,32 +336,32 @@ func (client *storageRESTClient) CheckParts(volume, path string, fi FileInfo) er return err } - respBody, err := client.call(storageRESTMethodCheckParts, values, &reader, -1) + respBody, err := client.call(ctx, storageRESTMethodCheckParts, values, &reader, -1) defer http.DrainBody(respBody) return err } // RenameData - rename source path to destination path atomically, metadata and data file. -func (client *storageRESTClient) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath string) (err error) { +func (client *storageRESTClient) RenameData(ctx context.Context, srcVolume, srcPath, dataDir, dstVolume, dstPath string) (err error) { values := make(url.Values) values.Set(storageRESTSrcVolume, srcVolume) values.Set(storageRESTSrcPath, srcPath) values.Set(storageRESTDataDir, dataDir) values.Set(storageRESTDstVolume, dstVolume) values.Set(storageRESTDstPath, dstPath) - respBody, err := client.call(storageRESTMethodRenameData, values, nil, -1) + respBody, err := client.call(ctx, storageRESTMethodRenameData, values, nil, -1) defer http.DrainBody(respBody) return err } -func (client *storageRESTClient) ReadVersion(volume, path, versionID string) (fi FileInfo, err error) { +func (client *storageRESTClient) ReadVersion(ctx context.Context, volume, path, versionID string) (fi FileInfo, err error) { values := make(url.Values) values.Set(storageRESTVolume, volume) values.Set(storageRESTFilePath, path) values.Set(storageRESTVersionID, versionID) - respBody, err := client.call(storageRESTMethodReadVersion, values, nil, -1) + respBody, err := client.call(ctx, storageRESTMethodReadVersion, values, nil, -1) if err != nil { return fi, err } @@ -374,11 +372,11 @@ func (client *storageRESTClient) ReadVersion(volume, path, versionID string) (fi } // ReadAll - reads all contents of a file. -func (client *storageRESTClient) ReadAll(volume, path string) ([]byte, error) { +func (client *storageRESTClient) ReadAll(ctx context.Context, volume string, path string) ([]byte, error) { values := make(url.Values) values.Set(storageRESTVolume, volume) values.Set(storageRESTFilePath, path) - respBody, err := client.call(storageRESTMethodReadAll, values, nil, -1) + respBody, err := client.call(ctx, storageRESTMethodReadAll, values, nil, -1) if err != nil { return nil, err } @@ -387,13 +385,13 @@ func (client *storageRESTClient) ReadAll(volume, path string) ([]byte, error) { } // ReadFileStream - returns a reader for the requested file. -func (client *storageRESTClient) ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error) { +func (client *storageRESTClient) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) { values := make(url.Values) values.Set(storageRESTVolume, volume) values.Set(storageRESTFilePath, path) values.Set(storageRESTOffset, strconv.Itoa(int(offset))) values.Set(storageRESTLength, strconv.Itoa(int(length))) - respBody, err := client.call(storageRESTMethodReadFileStream, values, nil, -1) + respBody, err := client.call(ctx, storageRESTMethodReadFileStream, values, nil, -1) if err != nil { return nil, err } @@ -401,12 +399,12 @@ func (client *storageRESTClient) ReadFileStream(volume, path string, offset, len } // ReadFile - reads section of a file. -func (client *storageRESTClient) ReadFile(volume, path string, offset int64, buffer []byte, verifier *BitrotVerifier) (int64, error) { +func (client *storageRESTClient) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (int64, error) { values := make(url.Values) values.Set(storageRESTVolume, volume) values.Set(storageRESTFilePath, path) values.Set(storageRESTOffset, strconv.Itoa(int(offset))) - values.Set(storageRESTLength, strconv.Itoa(len(buffer))) + values.Set(storageRESTLength, strconv.Itoa(len(buf))) if verifier != nil { values.Set(storageRESTBitrotAlgo, verifier.algorithm.String()) values.Set(storageRESTBitrotHash, hex.EncodeToString(verifier.sum)) @@ -414,21 +412,21 @@ func (client *storageRESTClient) ReadFile(volume, path string, offset int64, buf values.Set(storageRESTBitrotAlgo, "") values.Set(storageRESTBitrotHash, "") } - respBody, err := client.call(storageRESTMethodReadFile, values, nil, -1) + respBody, err := client.call(ctx, storageRESTMethodReadFile, values, nil, -1) if err != nil { return 0, err } defer http.DrainBody(respBody) - n, err := io.ReadFull(respBody, buffer) + n, err := io.ReadFull(respBody, buf) return int64(n), err } -func (client *storageRESTClient) WalkSplunk(volume, dirPath, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error) { +func (client *storageRESTClient) WalkSplunk(ctx context.Context, volume, dirPath, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error) { values := make(url.Values) values.Set(storageRESTVolume, volume) values.Set(storageRESTDirPath, dirPath) values.Set(storageRESTMarkerPath, marker) - respBody, err := client.call(storageRESTMethodWalkSplunk, values, nil, -1) + respBody, err := client.call(ctx, storageRESTMethodWalkSplunk, values, nil, -1) if err != nil { return nil, err } @@ -457,13 +455,13 @@ func (client *storageRESTClient) WalkSplunk(volume, dirPath, marker string, endW return ch, nil } -func (client *storageRESTClient) WalkVersions(volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error) { +func (client *storageRESTClient) WalkVersions(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error) { values := make(url.Values) values.Set(storageRESTVolume, volume) values.Set(storageRESTDirPath, dirPath) values.Set(storageRESTMarkerPath, marker) values.Set(storageRESTRecursive, strconv.FormatBool(recursive)) - respBody, err := client.call(storageRESTMethodWalkVersions, values, nil, -1) + respBody, err := client.call(ctx, storageRESTMethodWalkVersions, values, nil, -1) if err != nil { return nil, err } @@ -494,13 +492,13 @@ func (client *storageRESTClient) WalkVersions(volume, dirPath, marker string, re return ch, nil } -func (client *storageRESTClient) Walk(volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) { +func (client *storageRESTClient) Walk(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) { values := make(url.Values) values.Set(storageRESTVolume, volume) values.Set(storageRESTDirPath, dirPath) values.Set(storageRESTMarkerPath, marker) values.Set(storageRESTRecursive, strconv.FormatBool(recursive)) - respBody, err := client.call(storageRESTMethodWalk, values, nil, -1) + respBody, err := client.call(ctx, storageRESTMethodWalk, values, nil, -1) if err != nil { return nil, err } @@ -530,12 +528,12 @@ func (client *storageRESTClient) Walk(volume, dirPath, marker string, recursive } // ListDir - lists a directory. -func (client *storageRESTClient) ListDir(volume, dirPath string, count int) (entries []string, err error) { +func (client *storageRESTClient) ListDir(ctx context.Context, volume, dirPath string, count int) (entries []string, err error) { values := make(url.Values) values.Set(storageRESTVolume, volume) values.Set(storageRESTDirPath, dirPath) values.Set(storageRESTCount, strconv.Itoa(count)) - respBody, err := client.call(storageRESTMethodListDir, values, nil, -1) + respBody, err := client.call(ctx, storageRESTMethodListDir, values, nil, -1) if err != nil { return nil, err } @@ -545,17 +543,17 @@ func (client *storageRESTClient) ListDir(volume, dirPath string, count int) (ent } // DeleteFile - deletes a file. -func (client *storageRESTClient) DeleteFile(volume, path string) error { +func (client *storageRESTClient) DeleteFile(ctx context.Context, volume string, path string) error { values := make(url.Values) values.Set(storageRESTVolume, volume) values.Set(storageRESTFilePath, path) - respBody, err := client.call(storageRESTMethodDeleteFile, values, nil, -1) + respBody, err := client.call(ctx, storageRESTMethodDeleteFile, values, nil, -1) defer http.DrainBody(respBody) return err } // DeleteVersions - deletes list of specified versions if present -func (client *storageRESTClient) DeleteVersions(volume string, versions []FileInfo) (errs []error) { +func (client *storageRESTClient) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) (errs []error) { if len(versions) == 0 { return errs } @@ -572,7 +570,7 @@ func (client *storageRESTClient) DeleteVersions(volume string, versions []FileIn errs = make([]error, len(versions)) - respBody, err := client.call(storageRESTMethodDeleteVersions, values, &buffer, -1) + respBody, err := client.call(ctx, storageRESTMethodDeleteVersions, values, &buffer, -1) defer http.DrainBody(respBody) if err != nil { for i := range errs { @@ -605,18 +603,18 @@ func (client *storageRESTClient) DeleteVersions(volume string, versions []FileIn } // RenameFile - renames a file. -func (client *storageRESTClient) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err error) { +func (client *storageRESTClient) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) (err error) { values := make(url.Values) values.Set(storageRESTSrcVolume, srcVolume) values.Set(storageRESTSrcPath, srcPath) values.Set(storageRESTDstVolume, dstVolume) values.Set(storageRESTDstPath, dstPath) - respBody, err := client.call(storageRESTMethodRenameFile, values, nil, -1) + respBody, err := client.call(ctx, storageRESTMethodRenameFile, values, nil, -1) defer http.DrainBody(respBody) return err } -func (client *storageRESTClient) VerifyFile(volume, path string, fi FileInfo) error { +func (client *storageRESTClient) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error { values := make(url.Values) values.Set(storageRESTVolume, volume) values.Set(storageRESTFilePath, path) @@ -626,7 +624,7 @@ func (client *storageRESTClient) VerifyFile(volume, path string, fi FileInfo) er return err } - respBody, err := client.call(storageRESTMethodVerifyFile, values, &reader, -1) + respBody, err := client.call(ctx, storageRESTMethodVerifyFile, values, &reader, -1) defer http.DrainBody(respBody) if err != nil { return err @@ -674,7 +672,7 @@ func newStorageRESTClient(endpoint Endpoint) *storageRESTClient { ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout) // Instantiate a new rest client for healthcheck // to avoid recursive healthCheckFn() - respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).CallWithContext(ctx, storageRESTMethodHealth, nil, nil, -1) + respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).Call(ctx, storageRESTMethodHealth, nil, nil, -1) xhttp.DrainBody(respBody) cancel() return !errors.Is(err, context.DeadlineExceeded) && toStorageErr(err) != errDiskNotFound diff --git a/cmd/storage-rest-server.go b/cmd/storage-rest-server.go index 2de6d08fc..b4603a7af 100644 --- a/cmd/storage-rest-server.go +++ b/cmd/storage-rest-server.go @@ -138,7 +138,7 @@ func (s *storageRESTServer) DiskInfoHandler(w http.ResponseWriter, r *http.Reque if !s.IsValid(w, r) { return } - info, err := s.storage.DiskInfo() + info, err := s.storage.DiskInfo(r.Context()) if err != nil { info.Error = err.Error() } @@ -179,7 +179,7 @@ func (s *storageRESTServer) MakeVolHandler(w http.ResponseWriter, r *http.Reques } vars := mux.Vars(r) volume := vars[storageRESTVolume] - err := s.storage.MakeVol(volume) + err := s.storage.MakeVol(r.Context(), volume) if err != nil { s.writeErrorResponse(w, err) } @@ -192,7 +192,7 @@ func (s *storageRESTServer) MakeVolBulkHandler(w http.ResponseWriter, r *http.Re } vars := mux.Vars(r) volumes := strings.Split(vars[storageRESTVolumes], ",") - err := s.storage.MakeVolBulk(volumes...) + err := s.storage.MakeVolBulk(r.Context(), volumes...) if err != nil { s.writeErrorResponse(w, err) } @@ -203,7 +203,7 @@ func (s *storageRESTServer) ListVolsHandler(w http.ResponseWriter, r *http.Reque if !s.IsValid(w, r) { return } - infos, err := s.storage.ListVols() + infos, err := s.storage.ListVols(r.Context()) if err != nil { s.writeErrorResponse(w, err) return @@ -219,7 +219,7 @@ func (s *storageRESTServer) StatVolHandler(w http.ResponseWriter, r *http.Reques } vars := mux.Vars(r) volume := vars[storageRESTVolume] - info, err := s.storage.StatVol(volume) + info, err := s.storage.StatVol(r.Context(), volume) if err != nil { s.writeErrorResponse(w, err) return @@ -236,7 +236,7 @@ func (s *storageRESTServer) DeleteVolHandler(w http.ResponseWriter, r *http.Requ vars := mux.Vars(r) volume := vars[storageRESTVolume] forceDelete := vars[storageRESTForceDelete] == "true" - err := s.storage.DeleteVol(volume, forceDelete) + err := s.storage.DeleteVol(r.Context(), volume, forceDelete) if err != nil { s.writeErrorResponse(w, err) } @@ -257,7 +257,7 @@ func (s *storageRESTServer) AppendFileHandler(w http.ResponseWriter, r *http.Req s.writeErrorResponse(w, err) return } - err = s.storage.AppendFile(volume, filePath, buf) + err = s.storage.AppendFile(r.Context(), volume, filePath, buf) if err != nil { s.writeErrorResponse(w, err) } @@ -278,7 +278,7 @@ func (s *storageRESTServer) CreateFileHandler(w http.ResponseWriter, r *http.Req s.writeErrorResponse(w, err) return } - err = s.storage.CreateFile(volume, filePath, int64(fileSize), r.Body) + err = s.storage.CreateFile(r.Context(), volume, filePath, int64(fileSize), r.Body) if err != nil { s.writeErrorResponse(w, err) } @@ -304,7 +304,7 @@ func (s *storageRESTServer) DeleteVersionHandler(w http.ResponseWriter, r *http. return } - err := s.storage.DeleteVersion(volume, filePath, fi) + err := s.storage.DeleteVersion(r.Context(), volume, filePath, fi) if err != nil { s.writeErrorResponse(w, err) } @@ -320,7 +320,7 @@ func (s *storageRESTServer) ReadVersionHandler(w http.ResponseWriter, r *http.Re filePath := vars[storageRESTFilePath] versionID := vars[storageRESTVersionID] - fi, err := s.storage.ReadVersion(volume, filePath, versionID) + fi, err := s.storage.ReadVersion(r.Context(), volume, filePath, versionID) if err != nil { s.writeErrorResponse(w, err) return @@ -351,7 +351,7 @@ func (s *storageRESTServer) WriteMetadataHandler(w http.ResponseWriter, r *http. return } - err = s.storage.WriteMetadata(volume, filePath, fi) + err = s.storage.WriteMetadata(r.Context(), volume, filePath, fi) if err != nil { s.writeErrorResponse(w, err) } @@ -371,7 +371,7 @@ func (s *storageRESTServer) WriteAllHandler(w http.ResponseWriter, r *http.Reque return } - err := s.storage.WriteAll(volume, filePath, io.LimitReader(r.Body, r.ContentLength)) + err := s.storage.WriteAll(r.Context(), volume, filePath, io.LimitReader(r.Body, r.ContentLength)) if err != nil { s.writeErrorResponse(w, err) } @@ -397,7 +397,7 @@ func (s *storageRESTServer) CheckPartsHandler(w http.ResponseWriter, r *http.Req return } - if err := s.storage.CheckParts(volume, filePath, fi); err != nil { + if err := s.storage.CheckParts(r.Context(), volume, filePath, fi); err != nil { s.writeErrorResponse(w, err) } } @@ -411,7 +411,7 @@ func (s *storageRESTServer) CheckFileHandler(w http.ResponseWriter, r *http.Requ volume := vars[storageRESTVolume] filePath := vars[storageRESTFilePath] - if err := s.storage.CheckFile(volume, filePath); err != nil { + if err := s.storage.CheckFile(r.Context(), volume, filePath); err != nil { s.writeErrorResponse(w, err) } } @@ -425,7 +425,7 @@ func (s *storageRESTServer) ReadAllHandler(w http.ResponseWriter, r *http.Reques volume := vars[storageRESTVolume] filePath := vars[storageRESTFilePath] - buf, err := s.storage.ReadAll(volume, filePath) + buf, err := s.storage.ReadAll(r.Context(), volume, filePath) if err != nil { s.writeErrorResponse(w, err) return @@ -469,7 +469,7 @@ func (s *storageRESTServer) ReadFileHandler(w http.ResponseWriter, r *http.Reque verifier = NewBitrotVerifier(BitrotAlgorithmFromString(vars[storageRESTBitrotAlgo]), hash) } buf := make([]byte, length) - _, err = s.storage.ReadFile(volume, filePath, int64(offset), buf, verifier) + _, err = s.storage.ReadFile(r.Context(), volume, filePath, int64(offset), buf, verifier) if err != nil { s.writeErrorResponse(w, err) return @@ -498,7 +498,7 @@ func (s *storageRESTServer) ReadFileStreamHandler(w http.ResponseWriter, r *http return } - rc, err := s.storage.ReadFileStream(volume, filePath, int64(offset), int64(length)) + rc, err := s.storage.ReadFileStream(r.Context(), volume, filePath, int64(offset), int64(length)) if err != nil { s.writeErrorResponse(w, err) return @@ -524,7 +524,7 @@ func (s *storageRESTServer) WalkSplunkHandler(w http.ResponseWriter, r *http.Req setEventStreamHeaders(w) encoder := gob.NewEncoder(w) - fch, err := s.storage.WalkSplunk(volume, dirPath, markerPath, r.Context().Done()) + fch, err := s.storage.WalkSplunk(r.Context(), volume, dirPath, markerPath, r.Context().Done()) if err != nil { s.writeErrorResponse(w, err) return @@ -552,7 +552,7 @@ func (s *storageRESTServer) WalkVersionsHandler(w http.ResponseWriter, r *http.R setEventStreamHeaders(w) encoder := gob.NewEncoder(w) - fch, err := s.storage.WalkVersions(volume, dirPath, markerPath, recursive, r.Context().Done()) + fch, err := s.storage.WalkVersions(r.Context(), volume, dirPath, markerPath, recursive, r.Context().Done()) if err != nil { s.writeErrorResponse(w, err) return @@ -580,7 +580,7 @@ func (s *storageRESTServer) WalkHandler(w http.ResponseWriter, r *http.Request) setEventStreamHeaders(w) encoder := gob.NewEncoder(w) - fch, err := s.storage.Walk(volume, dirPath, markerPath, recursive, r.Context().Done()) + fch, err := s.storage.Walk(r.Context(), volume, dirPath, markerPath, recursive, r.Context().Done()) if err != nil { s.writeErrorResponse(w, err) return @@ -604,7 +604,7 @@ func (s *storageRESTServer) ListDirHandler(w http.ResponseWriter, r *http.Reques return } - entries, err := s.storage.ListDir(volume, dirPath, count) + entries, err := s.storage.ListDir(r.Context(), volume, dirPath, count) if err != nil { s.writeErrorResponse(w, err) return @@ -622,7 +622,7 @@ func (s *storageRESTServer) DeleteFileHandler(w http.ResponseWriter, r *http.Req volume := vars[storageRESTVolume] filePath := vars[storageRESTFilePath] - err := s.storage.DeleteFile(volume, filePath) + err := s.storage.DeleteFile(r.Context(), volume, filePath) if err != nil { s.writeErrorResponse(w, err) } @@ -663,7 +663,7 @@ func (s *storageRESTServer) DeleteVersionsHandler(w http.ResponseWriter, r *http setEventStreamHeaders(w) encoder := gob.NewEncoder(w) done := keepHTTPResponseAlive(w) - errs := s.storage.DeleteVersions(volume, versions) + errs := s.storage.DeleteVersions(r.Context(), volume, versions) done(nil) for idx := range versions { if errs[idx] != nil { @@ -685,7 +685,7 @@ func (s *storageRESTServer) RenameDataHandler(w http.ResponseWriter, r *http.Req dataDir := vars[storageRESTDataDir] dstVolume := vars[storageRESTDstVolume] dstFilePath := vars[storageRESTDstPath] - err := s.storage.RenameData(srcVolume, srcFilePath, dataDir, dstVolume, dstFilePath) + err := s.storage.RenameData(r.Context(), srcVolume, srcFilePath, dataDir, dstVolume, dstFilePath) if err != nil { s.writeErrorResponse(w, err) } @@ -701,7 +701,7 @@ func (s *storageRESTServer) RenameFileHandler(w http.ResponseWriter, r *http.Req srcFilePath := vars[storageRESTSrcPath] dstVolume := vars[storageRESTDstVolume] dstFilePath := vars[storageRESTDstPath] - err := s.storage.RenameFile(srcVolume, srcFilePath, dstVolume, dstFilePath) + err := s.storage.RenameFile(r.Context(), srcVolume, srcFilePath, dstVolume, dstFilePath) if err != nil { s.writeErrorResponse(w, err) } @@ -809,7 +809,7 @@ func (s *storageRESTServer) VerifyFileHandler(w http.ResponseWriter, r *http.Req setEventStreamHeaders(w) encoder := gob.NewEncoder(w) done := keepHTTPResponseAlive(w) - err = s.storage.VerifyFile(volume, filePath, fi) + err = s.storage.VerifyFile(r.Context(), volume, filePath, fi) done(nil) vresp := &VerifyFileResp{} if err != nil { diff --git a/cmd/storage-rest_test.go b/cmd/storage-rest_test.go index 086d1de28..376587302 100644 --- a/cmd/storage-rest_test.go +++ b/cmd/storage-rest_test.go @@ -17,6 +17,7 @@ package cmd import ( + "context" "io/ioutil" "net/http/httptest" "os" @@ -43,7 +44,7 @@ func testStorageAPIDiskInfo(t *testing.T, storage StorageAPI) { } for i, testCase := range testCases { - _, err := storage.DiskInfo() + _, err := storage.DiskInfo(context.Background()) expectErr := (err != nil) if expectErr != testCase.expectErr { @@ -66,7 +67,7 @@ func testStorageAPIMakeVol(t *testing.T, storage StorageAPI) { } for i, testCase := range testCases { - err := storage.MakeVol(testCase.volumeName) + err := storage.MakeVol(context.Background(), testCase.volumeName) expectErr := (err != nil) if expectErr != testCase.expectErr { @@ -87,13 +88,13 @@ func testStorageAPIListVols(t *testing.T, storage StorageAPI) { for i, testCase := range testCases { for _, volumeName := range testCase.volumeNames { - err := storage.MakeVol(volumeName) + err := storage.MakeVol(context.Background(), volumeName) if err != nil { t.Fatalf("unexpected error %v", err) } } - result, err := storage.ListVols() + result, err := storage.ListVols(context.Background()) expectErr := (err != nil) if expectErr != testCase.expectErr { @@ -109,7 +110,7 @@ func testStorageAPIListVols(t *testing.T, storage StorageAPI) { } func testStorageAPIStatVol(t *testing.T, storage StorageAPI) { - err := storage.MakeVol("foo") + err := storage.MakeVol(context.Background(), "foo") if err != nil { t.Fatalf("unexpected error %v", err) } @@ -124,7 +125,7 @@ func testStorageAPIStatVol(t *testing.T, storage StorageAPI) { } for i, testCase := range testCases { - result, err := storage.StatVol(testCase.volumeName) + result, err := storage.StatVol(context.Background(), testCase.volumeName) expectErr := (err != nil) if expectErr != testCase.expectErr { @@ -140,7 +141,7 @@ func testStorageAPIStatVol(t *testing.T, storage StorageAPI) { } func testStorageAPIDeleteVol(t *testing.T, storage StorageAPI) { - err := storage.MakeVol("foo") + err := storage.MakeVol(context.Background(), "foo") if err != nil { t.Fatalf("unexpected error %v", err) } @@ -155,7 +156,7 @@ func testStorageAPIDeleteVol(t *testing.T, storage StorageAPI) { } for i, testCase := range testCases { - err := storage.DeleteVol(testCase.volumeName, false) + err := storage.DeleteVol(context.Background(), testCase.volumeName, false) expectErr := (err != nil) if expectErr != testCase.expectErr { @@ -165,11 +166,11 @@ func testStorageAPIDeleteVol(t *testing.T, storage StorageAPI) { } func testStorageAPICheckFile(t *testing.T, storage StorageAPI) { - err := storage.MakeVol("foo") + err := storage.MakeVol(context.Background(), "foo") if err != nil { t.Fatalf("unexpected error %v", err) } - err = storage.AppendFile("foo", pathJoin("myobject", xlStorageFormatFile), []byte("foo")) + err = storage.AppendFile(context.Background(), "foo", pathJoin("myobject", xlStorageFormatFile), []byte("foo")) if err != nil { t.Fatalf("unexpected error %v", err) } @@ -185,7 +186,7 @@ func testStorageAPICheckFile(t *testing.T, storage StorageAPI) { } for i, testCase := range testCases { - err := storage.CheckFile(testCase.volumeName, testCase.objectName) + err := storage.CheckFile(context.Background(), testCase.volumeName, testCase.objectName) expectErr := (err != nil) if expectErr != testCase.expectErr { @@ -195,11 +196,11 @@ func testStorageAPICheckFile(t *testing.T, storage StorageAPI) { } func testStorageAPIListDir(t *testing.T, storage StorageAPI) { - err := storage.MakeVol("foo") + err := storage.MakeVol(context.Background(), "foo") if err != nil { t.Fatalf("unexpected error %v", err) } - err = storage.AppendFile("foo", "path/to/myobject", []byte("foo")) + err = storage.AppendFile(context.Background(), "foo", "path/to/myobject", []byte("foo")) if err != nil { t.Fatalf("unexpected error %v", err) } @@ -216,7 +217,7 @@ func testStorageAPIListDir(t *testing.T, storage StorageAPI) { } for i, testCase := range testCases { - result, err := storage.ListDir(testCase.volumeName, testCase.prefix, -1) + result, err := storage.ListDir(context.Background(), testCase.volumeName, testCase.prefix, -1) expectErr := (err != nil) if expectErr != testCase.expectErr { @@ -232,11 +233,11 @@ func testStorageAPIListDir(t *testing.T, storage StorageAPI) { } func testStorageAPIReadAll(t *testing.T, storage StorageAPI) { - err := storage.MakeVol("foo") + err := storage.MakeVol(context.Background(), "foo") if err != nil { t.Fatalf("unexpected error %v", err) } - err = storage.AppendFile("foo", "myobject", []byte("foo")) + err = storage.AppendFile(context.Background(), "foo", "myobject", []byte("foo")) if err != nil { t.Fatalf("unexpected error %v", err) } @@ -253,7 +254,7 @@ func testStorageAPIReadAll(t *testing.T, storage StorageAPI) { } for i, testCase := range testCases { - result, err := storage.ReadAll(testCase.volumeName, testCase.objectName) + result, err := storage.ReadAll(context.Background(), testCase.volumeName, testCase.objectName) expectErr := (err != nil) if expectErr != testCase.expectErr { @@ -269,11 +270,11 @@ func testStorageAPIReadAll(t *testing.T, storage StorageAPI) { } func testStorageAPIReadFile(t *testing.T, storage StorageAPI) { - err := storage.MakeVol("foo") + err := storage.MakeVol(context.Background(), "foo") if err != nil { t.Fatalf("unexpected error %v", err) } - err = storage.AppendFile("foo", "myobject", []byte("foo")) + err = storage.AppendFile(context.Background(), "foo", "myobject", []byte("foo")) if err != nil { t.Fatalf("unexpected error %v", err) } @@ -294,7 +295,7 @@ func testStorageAPIReadFile(t *testing.T, storage StorageAPI) { result := make([]byte, 100) for i, testCase := range testCases { result = result[testCase.offset:3] - _, err := storage.ReadFile(testCase.volumeName, testCase.objectName, testCase.offset, result, nil) + _, err := storage.ReadFile(context.Background(), testCase.volumeName, testCase.objectName, testCase.offset, result, nil) expectErr := (err != nil) if expectErr != testCase.expectErr { @@ -310,7 +311,7 @@ func testStorageAPIReadFile(t *testing.T, storage StorageAPI) { } func testStorageAPIAppendFile(t *testing.T, storage StorageAPI) { - err := storage.MakeVol("foo") + err := storage.MakeVol(context.Background(), "foo") if err != nil { t.Fatalf("unexpected error %v", err) } @@ -328,7 +329,7 @@ func testStorageAPIAppendFile(t *testing.T, storage StorageAPI) { } for i, testCase := range testCases { - err := storage.AppendFile(testCase.volumeName, testCase.objectName, testCase.data) + err := storage.AppendFile(context.Background(), testCase.volumeName, testCase.objectName, testCase.data) expectErr := (err != nil) if expectErr != testCase.expectErr { @@ -338,12 +339,12 @@ func testStorageAPIAppendFile(t *testing.T, storage StorageAPI) { } func testStorageAPIDeleteFile(t *testing.T, storage StorageAPI) { - err := storage.MakeVol("foo") + err := storage.MakeVol(context.Background(), "foo") if err != nil { t.Fatalf("unexpected error %v", err) } - err = storage.AppendFile("foo", "myobject", []byte("foo")) + err = storage.AppendFile(context.Background(), "foo", "myobject", []byte("foo")) if err != nil { t.Fatalf("unexpected error %v", err) } @@ -361,7 +362,7 @@ func testStorageAPIDeleteFile(t *testing.T, storage StorageAPI) { } for i, testCase := range testCases { - err := storage.DeleteFile(testCase.volumeName, testCase.objectName) + err := storage.DeleteFile(context.Background(), testCase.volumeName, testCase.objectName) expectErr := (err != nil) if expectErr != testCase.expectErr { @@ -371,22 +372,22 @@ func testStorageAPIDeleteFile(t *testing.T, storage StorageAPI) { } func testStorageAPIRenameFile(t *testing.T, storage StorageAPI) { - err := storage.MakeVol("foo") + err := storage.MakeVol(context.Background(), "foo") if err != nil { t.Fatalf("unexpected error %v", err) } - err = storage.MakeVol("bar") + err = storage.MakeVol(context.Background(), "bar") if err != nil { t.Fatalf("unexpected error %v", err) } - err = storage.AppendFile("foo", "myobject", []byte("foo")) + err = storage.AppendFile(context.Background(), "foo", "myobject", []byte("foo")) if err != nil { t.Fatalf("unexpected error %v", err) } - err = storage.AppendFile("foo", "otherobject", []byte("foo")) + err = storage.AppendFile(context.Background(), "foo", "otherobject", []byte("foo")) if err != nil { t.Fatalf("unexpected error %v", err) } @@ -405,7 +406,7 @@ func testStorageAPIRenameFile(t *testing.T, storage StorageAPI) { } for i, testCase := range testCases { - err := storage.RenameFile(testCase.volumeName, testCase.objectName, testCase.destVolumeName, testCase.destObjectName) + err := storage.RenameFile(context.Background(), testCase.volumeName, testCase.objectName, testCase.destVolumeName, testCase.destObjectName) expectErr := (err != nil) if expectErr != testCase.expectErr { diff --git a/cmd/tree-walk_test.go b/cmd/tree-walk_test.go index 5cb20eac5..46f47304a 100644 --- a/cmd/tree-walk_test.go +++ b/cmd/tree-walk_test.go @@ -69,14 +69,14 @@ func TestFilterMatchingPrefix(t *testing.T) { // Helper function that creates a volume and files in it. func createNamespace(disk StorageAPI, volume string, files []string) error { // Make a volume. - err := disk.MakeVol(volume) + err := disk.MakeVol(context.Background(), volume) if err != nil { return err } // Create files. for _, file := range files { - err = disk.AppendFile(volume, file, []byte{}) + err = disk.AppendFile(context.Background(), volume, file, []byte{}) if err != nil { return err } @@ -88,7 +88,7 @@ func createNamespace(disk StorageAPI, volume string, files []string) error { // disks - used for doing disk.ListDir() func listDirFactory(ctx context.Context, disk StorageAPI, isLeaf IsLeafFunc) ListDirFunc { return func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string, delayIsLeaf bool) { - entries, err := disk.ListDir(volume, dirPath, -1) + entries, err := disk.ListDir(ctx, volume, dirPath, -1) if err != nil { return false, nil, false } @@ -163,7 +163,7 @@ func TestTreeWalk(t *testing.T) { } isLeafDir := func(bucket, leafPath string) bool { - entries, _ := disk.ListDir(bucket, leafPath, 1) + entries, _ := disk.ListDir(context.Background(), bucket, leafPath, 1) return len(entries) == 0 } @@ -207,7 +207,7 @@ func TestTreeWalkTimeout(t *testing.T) { } isLeafDir := func(bucket, leafPath string) bool { - entries, _ := disk.ListDir(bucket, leafPath, 1) + entries, _ := disk.ListDir(context.Background(), bucket, leafPath, 1) return len(entries) == 0 } @@ -275,7 +275,7 @@ func TestRecursiveTreeWalk(t *testing.T) { } isLeafDir := func(bucket, leafPath string) bool { - entries, _ := disk1.ListDir(bucket, leafPath, 1) + entries, _ := disk1.ListDir(context.Background(), bucket, leafPath, 1) return len(entries) == 0 } @@ -389,7 +389,7 @@ func TestSortedness(t *testing.T) { } isLeafDir := func(bucket, leafPath string) bool { - entries, _ := disk1.ListDir(bucket, leafPath, 1) + entries, _ := disk1.ListDir(context.Background(), bucket, leafPath, 1) return len(entries) == 0 } @@ -469,7 +469,7 @@ func TestTreeWalkIsEnd(t *testing.T) { } isLeafDir := func(bucket, leafPath string) bool { - entries, _ := disk1.ListDir(bucket, leafPath, 1) + entries, _ := disk1.ListDir(context.Background(), bucket, leafPath, 1) return len(entries) == 0 } diff --git a/cmd/xl-storage-disk-id-check.go b/cmd/xl-storage-disk-id-check.go index 7b98a2355..03681f8ce 100644 --- a/cmd/xl-storage-disk-id-check.go +++ b/cmd/xl-storage-disk-id-check.go @@ -84,8 +84,8 @@ func (p *xlStorageDiskIDCheck) checkDiskStale() error { return errDiskNotFound } -func (p *xlStorageDiskIDCheck) DiskInfo() (info DiskInfo, err error) { - info, err = p.storage.DiskInfo() +func (p *xlStorageDiskIDCheck) DiskInfo(ctx context.Context) (info DiskInfo, err error) { + info, err = p.storage.DiskInfo(ctx) if err != nil { return info, err } @@ -99,145 +99,145 @@ func (p *xlStorageDiskIDCheck) DiskInfo() (info DiskInfo, err error) { return info, nil } -func (p *xlStorageDiskIDCheck) MakeVolBulk(volumes ...string) (err error) { +func (p *xlStorageDiskIDCheck) MakeVolBulk(ctx context.Context, volumes ...string) (err error) { if err = p.checkDiskStale(); err != nil { return err } - return p.storage.MakeVolBulk(volumes...) + return p.storage.MakeVolBulk(ctx, volumes...) } -func (p *xlStorageDiskIDCheck) MakeVol(volume string) (err error) { +func (p *xlStorageDiskIDCheck) MakeVol(ctx context.Context, volume string) (err error) { if err = p.checkDiskStale(); err != nil { return err } - return p.storage.MakeVol(volume) + return p.storage.MakeVol(ctx, volume) } -func (p *xlStorageDiskIDCheck) ListVols() ([]VolInfo, error) { +func (p *xlStorageDiskIDCheck) ListVols(ctx context.Context) ([]VolInfo, error) { if err := p.checkDiskStale(); err != nil { return nil, err } - return p.storage.ListVols() + return p.storage.ListVols(ctx) } -func (p *xlStorageDiskIDCheck) StatVol(volume string) (vol VolInfo, err error) { +func (p *xlStorageDiskIDCheck) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) { if err = p.checkDiskStale(); err != nil { return vol, err } - return p.storage.StatVol(volume) + return p.storage.StatVol(ctx, volume) } -func (p *xlStorageDiskIDCheck) DeleteVol(volume string, forceDelete bool) (err error) { +func (p *xlStorageDiskIDCheck) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) { if err = p.checkDiskStale(); err != nil { return err } - return p.storage.DeleteVol(volume, forceDelete) + return p.storage.DeleteVol(ctx, volume, forceDelete) } -func (p *xlStorageDiskIDCheck) WalkVersions(volume, dirPath string, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error) { +func (p *xlStorageDiskIDCheck) WalkVersions(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error) { if err := p.checkDiskStale(); err != nil { return nil, err } - return p.storage.WalkVersions(volume, dirPath, marker, recursive, endWalkCh) + return p.storage.WalkVersions(ctx, volume, dirPath, marker, recursive, endWalkCh) } -func (p *xlStorageDiskIDCheck) Walk(volume, dirPath string, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) { +func (p *xlStorageDiskIDCheck) Walk(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) { if err := p.checkDiskStale(); err != nil { return nil, err } - return p.storage.Walk(volume, dirPath, marker, recursive, endWalkCh) + return p.storage.Walk(ctx, volume, dirPath, marker, recursive, endWalkCh) } -func (p *xlStorageDiskIDCheck) WalkSplunk(volume, dirPath string, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error) { +func (p *xlStorageDiskIDCheck) WalkSplunk(ctx context.Context, volume, dirPath, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error) { if err := p.checkDiskStale(); err != nil { return nil, err } - return p.storage.WalkSplunk(volume, dirPath, marker, endWalkCh) + return p.storage.WalkSplunk(ctx, volume, dirPath, marker, endWalkCh) } -func (p *xlStorageDiskIDCheck) ListDir(volume, dirPath string, count int) ([]string, error) { +func (p *xlStorageDiskIDCheck) ListDir(ctx context.Context, volume, dirPath string, count int) ([]string, error) { if err := p.checkDiskStale(); err != nil { return nil, err } - return p.storage.ListDir(volume, dirPath, count) + return p.storage.ListDir(ctx, volume, dirPath, count) } -func (p *xlStorageDiskIDCheck) ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) { +func (p *xlStorageDiskIDCheck) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) { if err := p.checkDiskStale(); err != nil { return 0, err } - return p.storage.ReadFile(volume, path, offset, buf, verifier) + return p.storage.ReadFile(ctx, volume, path, offset, buf, verifier) } -func (p *xlStorageDiskIDCheck) AppendFile(volume string, path string, buf []byte) (err error) { +func (p *xlStorageDiskIDCheck) AppendFile(ctx context.Context, volume string, path string, buf []byte) (err error) { if err = p.checkDiskStale(); err != nil { return err } - return p.storage.AppendFile(volume, path, buf) + return p.storage.AppendFile(ctx, volume, path, buf) } -func (p *xlStorageDiskIDCheck) CreateFile(volume, path string, size int64, reader io.Reader) error { +func (p *xlStorageDiskIDCheck) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error { if err := p.checkDiskStale(); err != nil { return err } - return p.storage.CreateFile(volume, path, size, reader) + return p.storage.CreateFile(ctx, volume, path, size, reader) } -func (p *xlStorageDiskIDCheck) ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error) { +func (p *xlStorageDiskIDCheck) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) { if err := p.checkDiskStale(); err != nil { return nil, err } - return p.storage.ReadFileStream(volume, path, offset, length) + return p.storage.ReadFileStream(ctx, volume, path, offset, length) } -func (p *xlStorageDiskIDCheck) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error { +func (p *xlStorageDiskIDCheck) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error { if err := p.checkDiskStale(); err != nil { return err } - return p.storage.RenameFile(srcVolume, srcPath, dstVolume, dstPath) + return p.storage.RenameFile(ctx, srcVolume, srcPath, dstVolume, dstPath) } -func (p *xlStorageDiskIDCheck) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath string) error { +func (p *xlStorageDiskIDCheck) RenameData(ctx context.Context, srcVolume, srcPath, dataDir, dstVolume, dstPath string) error { if err := p.checkDiskStale(); err != nil { return err } - return p.storage.RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath) + return p.storage.RenameData(ctx, srcVolume, srcPath, dataDir, dstVolume, dstPath) } -func (p *xlStorageDiskIDCheck) CheckParts(volume string, path string, fi FileInfo) (err error) { +func (p *xlStorageDiskIDCheck) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (err error) { if err = p.checkDiskStale(); err != nil { return err } - return p.storage.CheckParts(volume, path, fi) + return p.storage.CheckParts(ctx, volume, path, fi) } -func (p *xlStorageDiskIDCheck) CheckFile(volume string, path string) (err error) { +func (p *xlStorageDiskIDCheck) CheckFile(ctx context.Context, volume string, path string) (err error) { if err = p.checkDiskStale(); err != nil { return err } - return p.storage.CheckFile(volume, path) + return p.storage.CheckFile(ctx, volume, path) } -func (p *xlStorageDiskIDCheck) DeleteFile(volume string, path string) (err error) { +func (p *xlStorageDiskIDCheck) DeleteFile(ctx context.Context, volume string, path string) (err error) { if err = p.checkDiskStale(); err != nil { return err } - return p.storage.DeleteFile(volume, path) + return p.storage.DeleteFile(ctx, volume, path) } -func (p *xlStorageDiskIDCheck) DeleteVersions(volume string, versions []FileInfo) (errs []error) { +func (p *xlStorageDiskIDCheck) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) (errs []error) { if err := p.checkDiskStale(); err != nil { errs = make([]error, len(versions)) for i := range errs { @@ -245,53 +245,53 @@ func (p *xlStorageDiskIDCheck) DeleteVersions(volume string, versions []FileInfo } return errs } - return p.storage.DeleteVersions(volume, versions) + return p.storage.DeleteVersions(ctx, volume, versions) } -func (p *xlStorageDiskIDCheck) VerifyFile(volume, path string, fi FileInfo) error { +func (p *xlStorageDiskIDCheck) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error { if err := p.checkDiskStale(); err != nil { return err } - return p.storage.VerifyFile(volume, path, fi) + return p.storage.VerifyFile(ctx, volume, path, fi) } -func (p *xlStorageDiskIDCheck) WriteAll(volume string, path string, reader io.Reader) (err error) { +func (p *xlStorageDiskIDCheck) WriteAll(ctx context.Context, volume string, path string, reader io.Reader) (err error) { if err = p.checkDiskStale(); err != nil { return err } - return p.storage.WriteAll(volume, path, reader) + return p.storage.WriteAll(ctx, volume, path, reader) } -func (p *xlStorageDiskIDCheck) DeleteVersion(volume, path string, fi FileInfo) (err error) { +func (p *xlStorageDiskIDCheck) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo) (err error) { if err = p.checkDiskStale(); err != nil { return err } - return p.storage.DeleteVersion(volume, path, fi) + return p.storage.DeleteVersion(ctx, volume, path, fi) } -func (p *xlStorageDiskIDCheck) WriteMetadata(volume, path string, fi FileInfo) (err error) { +func (p *xlStorageDiskIDCheck) WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) (err error) { if err = p.checkDiskStale(); err != nil { return err } - return p.storage.WriteMetadata(volume, path, fi) + return p.storage.WriteMetadata(ctx, volume, path, fi) } -func (p *xlStorageDiskIDCheck) ReadVersion(volume, path, versionID string) (fi FileInfo, err error) { +func (p *xlStorageDiskIDCheck) ReadVersion(ctx context.Context, volume, path, versionID string) (fi FileInfo, err error) { if err = p.checkDiskStale(); err != nil { return fi, err } - return p.storage.ReadVersion(volume, path, versionID) + return p.storage.ReadVersion(ctx, volume, path, versionID) } -func (p *xlStorageDiskIDCheck) ReadAll(volume string, path string) (buf []byte, err error) { +func (p *xlStorageDiskIDCheck) ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) { if err = p.checkDiskStale(); err != nil { return nil, err } - return p.storage.ReadAll(volume, path) + return p.storage.ReadAll(ctx, volume, path) } diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index ac729cbfe..2459d6488 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -417,7 +417,7 @@ type DiskInfo struct { // DiskInfo provides current information about disk space usage, // total free inodes and underlying filesystem. -func (s *xlStorage) DiskInfo() (info DiskInfo, err error) { +func (s *xlStorage) DiskInfo(context.Context) (info DiskInfo, err error) { atomic.AddInt32(&s.activeIOCount, 1) defer func() { atomic.AddInt32(&s.activeIOCount, -1) @@ -548,9 +548,9 @@ func (s *xlStorage) SetDiskID(id string) { // storage rest server for remote disks. } -func (s *xlStorage) MakeVolBulk(volumes ...string) (err error) { +func (s *xlStorage) MakeVolBulk(ctx context.Context, volumes ...string) (err error) { for _, volume := range volumes { - if err = s.MakeVol(volume); err != nil { + if err = s.MakeVol(ctx, volume); err != nil { if os.IsPermission(err) { return errVolumeAccessDenied } @@ -560,7 +560,7 @@ func (s *xlStorage) MakeVolBulk(volumes ...string) (err error) { } // Make a volume entry. -func (s *xlStorage) MakeVol(volume string) (err error) { +func (s *xlStorage) MakeVol(ctx context.Context, volume string) (err error) { if !isValidVolname(volume) { return errInvalidArgument } @@ -594,7 +594,7 @@ func (s *xlStorage) MakeVol(volume string) (err error) { } // ListVols - list volumes. -func (s *xlStorage) ListVols() (volsInfo []VolInfo, err error) { +func (s *xlStorage) ListVols(context.Context) (volsInfo []VolInfo, err error) { atomic.AddInt32(&s.activeIOCount, 1) defer func() { atomic.AddInt32(&s.activeIOCount, -1) @@ -654,7 +654,7 @@ func listVols(dirPath string) ([]VolInfo, error) { } // StatVol - get volume info. -func (s *xlStorage) StatVol(volume string) (volInfo VolInfo, err error) { +func (s *xlStorage) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) { atomic.AddInt32(&s.activeIOCount, 1) defer func() { atomic.AddInt32(&s.activeIOCount, -1) @@ -686,7 +686,7 @@ func (s *xlStorage) StatVol(volume string) (volInfo VolInfo, err error) { } // DeleteVol - delete a volume. -func (s *xlStorage) DeleteVol(volume string, forceDelete bool) (err error) { +func (s *xlStorage) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) { atomic.AddInt32(&s.activeIOCount, 1) defer func() { atomic.AddInt32(&s.activeIOCount, -1) @@ -806,7 +806,7 @@ func (s *xlStorage) isLeafDir(volume, leafPath string) bool { // sorted order, additionally along with metadata about each of those entries. // Implemented specifically for Splunk backend structure and List call with // delimiter as "guidSplunk" -func (s *xlStorage) WalkSplunk(volume, dirPath, marker string, endWalkCh <-chan struct{}) (ch chan FileInfo, err error) { +func (s *xlStorage) WalkSplunk(ctx context.Context, volume, dirPath, marker string, endWalkCh <-chan struct{}) (ch chan FileInfo, err error) { // Verify if volume is valid and it exists. volumeDir, err := s.getVolDir(volume) if err != nil { @@ -877,7 +877,7 @@ func (s *xlStorage) WalkSplunk(volume, dirPath, marker string, endWalkCh <-chan // WalkVersions - is a sorted walker which returns file entries in lexically sorted order, // additionally along with metadata version info about each of those entries. -func (s *xlStorage) WalkVersions(volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (ch chan FileInfoVersions, err error) { +func (s *xlStorage) WalkVersions(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (ch chan FileInfoVersions, err error) { atomic.AddInt32(&s.activeIOCount, 1) defer func() { atomic.AddInt32(&s.activeIOCount, -1) @@ -912,7 +912,7 @@ func (s *xlStorage) WalkVersions(volume, dirPath, marker string, recursive bool, go func() { defer close(ch) listDir := func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string, delayIsLeaf bool) { - entries, err := s.ListDir(volume, dirPath, -1) + entries, err := s.ListDir(ctx, volume, dirPath, -1) if err != nil { return false, nil, false } @@ -962,7 +962,7 @@ func (s *xlStorage) WalkVersions(volume, dirPath, marker string, recursive bool, // Walk - is a sorted walker which returns file entries in lexically // sorted order, additionally along with metadata about each of those entries. -func (s *xlStorage) Walk(volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (ch chan FileInfo, err error) { +func (s *xlStorage) Walk(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (ch chan FileInfo, err error) { atomic.AddInt32(&s.activeIOCount, 1) defer func() { atomic.AddInt32(&s.activeIOCount, -1) @@ -997,7 +997,7 @@ func (s *xlStorage) Walk(volume, dirPath, marker string, recursive bool, endWalk go func() { defer close(ch) listDir := func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string, delayIsLeaf bool) { - entries, err := s.ListDir(volume, dirPath, -1) + entries, err := s.ListDir(ctx, volume, dirPath, -1) if err != nil { return false, nil, false } @@ -1046,7 +1046,7 @@ func (s *xlStorage) Walk(volume, dirPath, marker string, recursive bool, endWalk // ListDir - return all the entries at the given directory path. // If an entry is a directory it will be returned with a trailing SlashSeparator. -func (s *xlStorage) ListDir(volume, dirPath string, count int) (entries []string, err error) { +func (s *xlStorage) ListDir(ctx context.Context, volume, dirPath string, count int) (entries []string, err error) { atomic.AddInt32(&s.activeIOCount, 1) defer func() { atomic.AddInt32(&s.activeIOCount, -1) @@ -1082,10 +1082,10 @@ func (s *xlStorage) ListDir(volume, dirPath string, count int) (entries []string // DeleteVersions deletes slice of versions, it can be same object // or multiple objects. -func (s *xlStorage) DeleteVersions(volume string, versions []FileInfo) []error { +func (s *xlStorage) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) []error { errs := make([]error, len(versions)) for i, version := range versions { - if err := s.DeleteVersion(volume, version.Name, version); err != nil { + if err := s.DeleteVersion(ctx, volume, version.Name, version); err != nil { errs[i] = err } } @@ -1094,12 +1094,12 @@ func (s *xlStorage) DeleteVersions(volume string, versions []FileInfo) []error { } // DeleteVersion - deletes FileInfo metadata for path at `xl.meta` -func (s *xlStorage) DeleteVersion(volume, path string, fi FileInfo) error { +func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo) error { if HasSuffix(path, SlashSeparator) { - return s.DeleteFile(volume, path) + return s.DeleteFile(ctx, volume, path) } - buf, err := s.ReadAll(volume, pathJoin(path, xlStorageFormatFile)) + buf, err := s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile)) if err != nil { return err } @@ -1152,7 +1152,7 @@ func (s *xlStorage) DeleteVersion(volume, path string, fi FileInfo) error { } if !lastVersion { - return s.WriteAll(volume, pathJoin(path, xlStorageFormatFile), bytes.NewReader(buf)) + return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), bytes.NewReader(buf)) } // Delete the meta file, if there are no more versions the @@ -1166,8 +1166,8 @@ func (s *xlStorage) DeleteVersion(volume, path string, fi FileInfo) error { } // WriteMetadata - writes FileInfo metadata for path at `xl.meta` -func (s *xlStorage) WriteMetadata(volume, path string, fi FileInfo) error { - buf, err := s.ReadAll(volume, pathJoin(path, xlStorageFormatFile)) +func (s *xlStorage) WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) error { + buf, err := s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile)) if err != nil && err != errFileNotFound { return err } @@ -1200,7 +1200,7 @@ func (s *xlStorage) WriteMetadata(volume, path string, fi FileInfo) error { } } - return s.WriteAll(volume, pathJoin(path, xlStorageFormatFile), bytes.NewReader(buf)) + return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), bytes.NewReader(buf)) } func (s *xlStorage) renameLegacyMetadata(volume, path string) error { @@ -1268,14 +1268,14 @@ func (s *xlStorage) renameLegacyMetadata(volume, path string) error { } // ReadVersion - reads metadata and returns FileInfo at path `xl.meta` -func (s *xlStorage) ReadVersion(volume, path, versionID string) (fi FileInfo, err error) { - buf, err := s.ReadAll(volume, pathJoin(path, xlStorageFormatFile)) +func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID string) (fi FileInfo, err error) { + buf, err := s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile)) if err != nil { if err == errFileNotFound { if err = s.renameLegacyMetadata(volume, path); err != nil { return fi, err } - buf, err = s.ReadAll(volume, pathJoin(path, xlStorageFormatFile)) + buf, err = s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile)) if err != nil { return fi, err } @@ -1300,7 +1300,7 @@ func (s *xlStorage) ReadVersion(volume, path, versionID string) (fi FileInfo, er // as an error to be reported. // This API is meant to be used on files which have small memory footprint, do // not use this on large files as it would cause server to crash. -func (s *xlStorage) ReadAll(volume, path string) (buf []byte, err error) { +func (s *xlStorage) ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) { atomic.AddInt32(&s.activeIOCount, 1) defer func() { atomic.AddInt32(&s.activeIOCount, -1) @@ -1363,7 +1363,7 @@ func (s *xlStorage) ReadAll(volume, path string) (buf []byte, err error) { // // Additionally ReadFile also starts reading from an offset. ReadFile // semantics are same as io.ReadFull. -func (s *xlStorage) ReadFile(volume, path string, offset int64, buffer []byte, verifier *BitrotVerifier) (int64, error) { +func (s *xlStorage) ReadFile(ctx context.Context, volume string, path string, offset int64, buffer []byte, verifier *BitrotVerifier) (int64, error) { if offset < 0 { return 0, errInvalidArgument } @@ -1518,7 +1518,7 @@ func (s *xlStorage) openFile(volume, path string, mode int) (f *os.File, err err } // ReadFileStream - Returns the read stream of the file. -func (s *xlStorage) ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error) { +func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) { if offset < 0 { return nil, errInvalidArgument } @@ -1611,7 +1611,7 @@ func (c closeWrapper) Close() error { } // CreateFile - creates the file. -func (s *xlStorage) CreateFile(volume, path string, fileSize int64, r io.Reader) (err error) { +func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSize int64, r io.Reader) (err error) { if fileSize < -1 { return errInvalidArgument } @@ -1720,7 +1720,7 @@ func (s *xlStorage) CreateFile(volume, path string, fileSize int64, r io.Reader) return nil } -func (s *xlStorage) WriteAll(volume, path string, reader io.Reader) (err error) { +func (s *xlStorage) WriteAll(ctx context.Context, volume string, path string, reader io.Reader) (err error) { atomic.AddInt32(&s.activeIOCount, 1) defer func() { atomic.AddInt32(&s.activeIOCount, -1) @@ -1742,7 +1742,7 @@ func (s *xlStorage) WriteAll(volume, path string, reader io.Reader) (err error) // AppendFile - append a byte array at path, if file doesn't exist at // path this call explicitly creates it. -func (s *xlStorage) AppendFile(volume, path string, buf []byte) (err error) { +func (s *xlStorage) AppendFile(ctx context.Context, volume string, path string, buf []byte) (err error) { atomic.AddInt32(&s.activeIOCount, 1) defer func() { atomic.AddInt32(&s.activeIOCount, -1) @@ -1764,7 +1764,7 @@ func (s *xlStorage) AppendFile(volume, path string, buf []byte) (err error) { } // CheckParts check if path has necessary parts available. -func (s *xlStorage) CheckParts(volume, path string, fi FileInfo) error { +func (s *xlStorage) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error { atomic.AddInt32(&s.activeIOCount, 1) defer func() { atomic.AddInt32(&s.activeIOCount, -1) @@ -1809,7 +1809,7 @@ func (s *xlStorage) CheckParts(volume, path string, fi FileInfo) error { } // CheckFile check if path has necessary metadata. -func (s *xlStorage) CheckFile(volume, path string) error { +func (s *xlStorage) CheckFile(ctx context.Context, volume string, path string) error { atomic.AddInt32(&s.activeIOCount, 1) defer func() { atomic.AddInt32(&s.activeIOCount, -1) @@ -1913,7 +1913,7 @@ func deleteFile(basePath, deletePath string, recursive bool) error { } // DeleteFile - delete a file at path. -func (s *xlStorage) DeleteFile(volume, path string) (err error) { +func (s *xlStorage) DeleteFile(ctx context.Context, volume string, path string) (err error) { atomic.AddInt32(&s.activeIOCount, 1) defer func() { atomic.AddInt32(&s.activeIOCount, -1) @@ -1988,7 +1988,7 @@ func (s *xlStorage) DeleteFileBulk(volume string, paths []string) (errs []error, } // RenameData - rename source path to destination path atomically, metadata and data directory. -func (s *xlStorage) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath string) (err error) { +func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath, dataDir, dstVolume, dstPath string) (err error) { atomic.AddInt32(&s.activeIOCount, 1) defer func() { atomic.AddInt32(&s.activeIOCount, -1) @@ -2170,7 +2170,7 @@ func (s *xlStorage) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath s return errFileCorrupt } - if err = s.WriteAll(srcVolume, pathJoin(srcPath, xlStorageFormatFile), bytes.NewReader(dstBuf)); err != nil { + if err = s.WriteAll(ctx, srcVolume, pathJoin(srcPath, xlStorageFormatFile), bytes.NewReader(dstBuf)); err != nil { return err } @@ -2201,7 +2201,7 @@ func (s *xlStorage) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath s } // RenameFile - rename source path to destination path atomically. -func (s *xlStorage) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err error) { +func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) (err error) { atomic.AddInt32(&s.activeIOCount, 1) defer func() { atomic.AddInt32(&s.activeIOCount, -1) @@ -2357,7 +2357,7 @@ func (s *xlStorage) bitrotVerify(partPath string, partSize int64, algo BitrotAlg } } -func (s *xlStorage) VerifyFile(volume, path string, fi FileInfo) (err error) { +func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (err error) { atomic.AddInt32(&s.activeIOCount, 1) defer func() { atomic.AddInt32(&s.activeIOCount, -1) diff --git a/cmd/xl-storage_test.go b/cmd/xl-storage_test.go index 918234c97..85259926f 100644 --- a/cmd/xl-storage_test.go +++ b/cmd/xl-storage_test.go @@ -18,6 +18,7 @@ package cmd import ( "bytes" + "context" "crypto/rand" "fmt" "io" @@ -126,12 +127,12 @@ func newXLStorageTestSetup() (*xlStorageDiskIDCheck, string, error) { if err != nil { return nil, "", err } - err = storage.MakeVol(minioMetaBucket) + err = storage.MakeVol(context.Background(), minioMetaBucket) if err != nil { return nil, "", err } // Create a sample format.json file - err = storage.WriteAll(minioMetaBucket, formatConfigFile, bytes.NewBufferString(`{"version":"1","format":"xl","id":"592a41c2-b7cc-4130-b883-c4b5cb15965b","xl":{"version":"3","this":"da017d62-70e3-45f1-8a1a-587707e69ad1","sets":[["e07285a6-8c73-4962-89c6-047fb939f803","33b8d431-482d-4376-b63c-626d229f0a29","cff6513a-4439-4dc1-bcaa-56c9e880c352","da017d62-70e3-45f1-8a1a-587707e69ad1","9c9f21d5-1f15-4737-bce6-835faa0d9626","0a59b346-1424-4fc2-9fa2-a2e80541d0c1","7924a3dc-b69a-4971-9a2e-014966d6aebb","4d2b8dd9-4e48-444b-bdca-c89194b26042"]],"distributionAlgo":"CRCMOD"}}`)) + err = storage.WriteAll(context.Background(), minioMetaBucket, formatConfigFile, bytes.NewBufferString(`{"version":"1","format":"xl","id":"592a41c2-b7cc-4130-b883-c4b5cb15965b","xl":{"version":"3","this":"da017d62-70e3-45f1-8a1a-587707e69ad1","sets":[["e07285a6-8c73-4962-89c6-047fb939f803","33b8d431-482d-4376-b63c-626d229f0a29","cff6513a-4439-4dc1-bcaa-56c9e880c352","da017d62-70e3-45f1-8a1a-587707e69ad1","9c9f21d5-1f15-4737-bce6-835faa0d9626","0a59b346-1424-4fc2-9fa2-a2e80541d0c1","7924a3dc-b69a-4971-9a2e-014966d6aebb","4d2b8dd9-4e48-444b-bdca-c89194b26042"]],"distributionAlgo":"CRCMOD"}}`)) if err != nil { return nil, "", err } @@ -262,16 +263,16 @@ func TestXLStorageReadAll(t *testing.T) { defer os.RemoveAll(path) // Create files for the test cases. - if err = xlStorage.MakeVol("exists"); err != nil { + if err = xlStorage.MakeVol(context.Background(), "exists"); err != nil { t.Fatalf("Unable to create a volume \"exists\", %s", err) } - if err = xlStorage.AppendFile("exists", "as-directory/as-file", []byte("Hello, World")); err != nil { + if err = xlStorage.AppendFile(context.Background(), "exists", "as-directory/as-file", []byte("Hello, World")); err != nil { t.Fatalf("Unable to create a file \"as-directory/as-file\", %s", err) } - if err = xlStorage.AppendFile("exists", "as-file", []byte("Hello, World")); err != nil { + if err = xlStorage.AppendFile(context.Background(), "exists", "as-file", []byte("Hello, World")); err != nil { t.Fatalf("Unable to create a file \"as-file\", %s", err) } - if err = xlStorage.AppendFile("exists", "as-file-parent", []byte("Hello, World")); err != nil { + if err = xlStorage.AppendFile(context.Background(), "exists", "as-file-parent", []byte("Hello, World")); err != nil { t.Fatalf("Unable to create a file \"as-file-parent\", %s", err) } @@ -328,7 +329,7 @@ func TestXLStorageReadAll(t *testing.T) { var dataRead []byte // Run through all the test cases and validate for ReadAll. for i, testCase := range testCases { - dataRead, err = xlStorage.ReadAll(testCase.volume, testCase.path) + dataRead, err = xlStorage.ReadAll(context.Background(), testCase.volume, testCase.path) if err != testCase.err { t.Fatalf("TestXLStorage %d: Expected err \"%s\", got err \"%s\"", i+1, testCase.err, err) } @@ -434,7 +435,7 @@ func TestXLStorageMakeVol(t *testing.T) { } for i, testCase := range testCases { - if err := xlStorage.MakeVol(testCase.volName); err != testCase.expectedErr { + if err := xlStorage.MakeVol(context.Background(), testCase.volName); err != testCase.expectedErr { t.Fatalf("TestXLStorage %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) } } @@ -470,7 +471,7 @@ func TestXLStorageMakeVol(t *testing.T) { t.Fatalf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err) } - if err := xlStorageNew.MakeVol("test-vol"); err != errDiskAccessDenied { + if err := xlStorageNew.MakeVol(context.Background(), "test-vol"); err != errDiskAccessDenied { t.Fatalf("expected: %s, got: %s", errDiskAccessDenied, err) } } @@ -486,7 +487,7 @@ func TestXLStorageDeleteVol(t *testing.T) { defer os.RemoveAll(path) // Setup test environment. - if err = xlStorage.MakeVol("success-vol"); err != nil { + if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil { t.Fatalf("Unable to create volume, %s", err) } @@ -530,7 +531,7 @@ func TestXLStorageDeleteVol(t *testing.T) { } for i, testCase := range testCases { - if err = xlStorage.DeleteVol(testCase.volName, false); err != testCase.expectedErr { + if err = xlStorage.DeleteVol(context.Background(), testCase.volName, false); err != testCase.expectedErr { t.Fatalf("TestXLStorage: %d, expected: %s, got: %s", i+1, testCase.expectedErr, err) } } @@ -569,7 +570,7 @@ func TestXLStorageDeleteVol(t *testing.T) { t.Fatalf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err) } - if err = xlStorageNew.DeleteVol("mybucket", false); err != errDiskAccessDenied { + if err = xlStorageNew.DeleteVol(context.Background(), "mybucket", false); err != errDiskAccessDenied { t.Fatalf("expected: Permission error, got: %s", err) } } @@ -583,7 +584,7 @@ func TestXLStorageDeleteVol(t *testing.T) { // TestXLStorage for delete on an removed disk. // should fail with disk not found. - err = xlStorageDeletedStorage.DeleteVol("Del-Vol", false) + err = xlStorageDeletedStorage.DeleteVol(context.Background(), "Del-Vol", false) if err != errDiskNotFound { t.Errorf("Expected: \"Disk not found\", got \"%s\"", err) } @@ -599,7 +600,7 @@ func TestXLStorageStatVol(t *testing.T) { defer os.RemoveAll(path) // Setup test environment. - if err = xlStorage.MakeVol("success-vol"); err != nil { + if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil { t.Fatalf("Unable to create volume, %s", err) } @@ -626,7 +627,7 @@ func TestXLStorageStatVol(t *testing.T) { for i, testCase := range testCases { var volInfo VolInfo - volInfo, err = xlStorage.StatVol(testCase.volName) + volInfo, err = xlStorage.StatVol(context.Background(), testCase.volName) if err != testCase.expectedErr { t.Fatalf("TestXLStorage case : %d, Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) } @@ -648,7 +649,7 @@ func TestXLStorageStatVol(t *testing.T) { // TestXLStorage for delete on an removed disk. // should fail with disk not found. - _, err = xlStorageDeletedStorage.StatVol("Stat vol") + _, err = xlStorageDeletedStorage.StatVol(context.Background(), "Stat vol") if err != errDiskNotFound { t.Errorf("Expected: \"Disk not found\", got \"%s\"", err) } @@ -664,18 +665,18 @@ func TestXLStorageListVols(t *testing.T) { var volInfos []VolInfo // TestXLStorage empty list vols. - if volInfos, err = xlStorage.ListVols(); err != nil { + if volInfos, err = xlStorage.ListVols(context.Background()); err != nil { t.Fatalf("expected: , got: %s", err) } else if len(volInfos) != 1 { t.Fatalf("expected: one entry, got: %s", volInfos) } // TestXLStorage non-empty list vols. - if err = xlStorage.MakeVol("success-vol"); err != nil { + if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil { t.Fatalf("Unable to create volume, %s", err) } - volInfos, err = xlStorage.ListVols() + volInfos, err = xlStorage.ListVols(context.Background()) if err != nil { t.Fatalf("expected: , got: %s", err) } @@ -696,7 +697,7 @@ func TestXLStorageListVols(t *testing.T) { // removing the path and simulating disk failure os.RemoveAll(path) // should fail with errDiskNotFound. - if _, err = xlStorage.ListVols(); err != errDiskNotFound { + if _, err = xlStorage.ListVols(context.Background()); err != errDiskNotFound { t.Errorf("Expected to fail with \"%s\", but instead failed with \"%s\"", errDiskNotFound, err) } } @@ -718,13 +719,13 @@ func TestXLStorageXlStorageListDir(t *testing.T) { // removing the disk, used to recreate disk not found error. os.RemoveAll(diskPath) // Setup test environment. - if err = xlStorage.MakeVol("success-vol"); err != nil { + if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil { t.Fatalf("Unable to create volume, %s", err) } - if err = xlStorage.AppendFile("success-vol", "abc/def/ghi/success-file", []byte("Hello, world")); err != nil { + if err = xlStorage.AppendFile(context.Background(), "success-vol", "abc/def/ghi/success-file", []byte("Hello, world")); err != nil { t.Fatalf("Unable to create file, %s", err) } - if err = xlStorage.AppendFile("success-vol", "abc/xyz/ghi/success-file", []byte("Hello, world")); err != nil { + if err = xlStorage.AppendFile(context.Background(), "success-vol", "abc/xyz/ghi/success-file", []byte("Hello, world")); err != nil { t.Fatalf("Unable to create file, %s", err) } @@ -783,7 +784,7 @@ func TestXLStorageXlStorageListDir(t *testing.T) { for i, testCase := range testCases { var dirList []string - dirList, err = xlStorage.ListDir(testCase.srcVol, testCase.srcPath, -1) + dirList, err = xlStorage.ListDir(context.Background(), testCase.srcVol, testCase.srcPath, -1) if err != testCase.expectedErr { t.Fatalf("TestXLStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) } @@ -816,14 +817,14 @@ func TestXLStorageXlStorageListDir(t *testing.T) { t.Fatalf("Unable to initialize xlStorage, %s", err) } - if err = xlStorageNew.DeleteFile("mybucket", "myobject"); err != errFileAccessDenied { + if err = xlStorageNew.DeleteFile(context.Background(), "mybucket", "myobject"); err != errFileAccessDenied { t.Errorf("expected: %s, got: %s", errFileAccessDenied, err) } } // TestXLStorage for delete on an removed disk. // should fail with disk not found. - err = xlStorageDeletedStorage.DeleteFile("del-vol", "my-file") + err = xlStorageDeletedStorage.DeleteFile(context.Background(), "del-vol", "my-file") if err != errDiskNotFound { t.Errorf("Expected: \"Disk not found\", got \"%s\"", err) } @@ -846,17 +847,17 @@ func TestXLStorageDeleteFile(t *testing.T) { // removing the disk, used to recreate disk not found error. os.RemoveAll(diskPath) // Setup test environment. - if err = xlStorage.MakeVol("success-vol"); err != nil { + if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil { t.Fatalf("Unable to create volume, %s", err) } - if err = xlStorage.AppendFile("success-vol", "success-file", []byte("Hello, world")); err != nil { + if err = xlStorage.AppendFile(context.Background(), "success-vol", "success-file", []byte("Hello, world")); err != nil { t.Fatalf("Unable to create file, %s", err) } - if err = xlStorage.MakeVol("no-permissions"); err != nil { + if err = xlStorage.MakeVol(context.Background(), "no-permissions"); err != nil { t.Fatalf("Unable to create volume, %s", err.Error()) } - if err = xlStorage.AppendFile("no-permissions", "dir/file", []byte("Hello, world")); err != nil { + if err = xlStorage.AppendFile(context.Background(), "no-permissions", "dir/file", []byte("Hello, world")); err != nil { t.Fatalf("Unable to create file, %s", err.Error()) } // Parent directory must have write permissions, this is read + execute. @@ -915,7 +916,7 @@ func TestXLStorageDeleteFile(t *testing.T) { } for i, testCase := range testCases { - if err = xlStorage.DeleteFile(testCase.srcVol, testCase.srcPath); err != testCase.expectedErr { + if err = xlStorage.DeleteFile(context.Background(), testCase.srcVol, testCase.srcPath); err != testCase.expectedErr { t.Errorf("TestXLStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) } } @@ -940,14 +941,14 @@ func TestXLStorageDeleteFile(t *testing.T) { t.Fatalf("Unable to initialize xlStorage, %s", err) } - if err = xlStorageNew.DeleteFile("mybucket", "myobject"); err != errFileAccessDenied { + if err = xlStorageNew.DeleteFile(context.Background(), "mybucket", "myobject"); err != errFileAccessDenied { t.Errorf("expected: %s, got: %s", errFileAccessDenied, err) } } // TestXLStorage for delete on an removed disk. // should fail with disk not found. - err = xlStorageDeletedStorage.DeleteFile("del-vol", "my-file") + err = xlStorageDeletedStorage.DeleteFile(context.Background(), "del-vol", "my-file") if err != errDiskNotFound { t.Errorf("Expected: \"Disk not found\", got \"%s\"", err) } @@ -964,7 +965,7 @@ func TestXLStorageReadFile(t *testing.T) { volume := "success-vol" // Setup test environment. - if err = xlStorage.MakeVol(volume); err != nil { + if err = xlStorage.MakeVol(context.Background(), volume); err != nil { t.Fatalf("Unable to create volume, %s", err) } @@ -1048,7 +1049,7 @@ func TestXLStorageReadFile(t *testing.T) { v := NewBitrotVerifier(SHA256, getSHA256Sum([]byte("hello, world"))) // Create test files for further reading. for i, appendFile := range appendFiles { - err = xlStorage.AppendFile(volume, appendFile.fileName, []byte("hello, world")) + err = xlStorage.AppendFile(context.Background(), volume, appendFile.fileName, []byte("hello, world")) if err != appendFile.expectedErr { t.Fatalf("Creating file failed: %d %#v, expected: %s, got: %s", i+1, appendFile, appendFile.expectedErr, err) } @@ -1057,7 +1058,7 @@ func TestXLStorageReadFile(t *testing.T) { { buf := make([]byte, 5) // Test for negative offset. - if _, err = xlStorage.ReadFile(volume, "myobject", -1, buf, v); err == nil { + if _, err = xlStorage.ReadFile(context.Background(), volume, "myobject", -1, buf, v); err == nil { t.Fatalf("expected: error, got: ") } } @@ -1067,7 +1068,7 @@ func TestXLStorageReadFile(t *testing.T) { var n int64 // Common read buffer. var buf = make([]byte, testCase.bufSize) - n, err = xlStorage.ReadFile(testCase.volume, testCase.fileName, testCase.offset, buf, v) + n, err = xlStorage.ReadFile(context.Background(), testCase.volume, testCase.fileName, testCase.offset, buf, v) if err != nil && testCase.expectedErr != nil { // Validate if the type string of the errors are an exact match. if err.Error() != testCase.expectedErr.Error() { @@ -1140,7 +1141,7 @@ func TestXLStorageReadFile(t *testing.T) { // Common read buffer. var buf = make([]byte, 10) - if _, err = xlStoragePermStorage.ReadFile("mybucket", "myobject", 0, buf, v); err != errFileAccessDenied { + if _, err = xlStoragePermStorage.ReadFile(context.Background(), "mybucket", "myobject", 0, buf, v); err != errFileAccessDenied { t.Errorf("expected: %s, got: %s", errFileAccessDenied, err) } } @@ -1182,7 +1183,7 @@ func TestXLStorageReadFileWithVerify(t *testing.T) { os.RemoveAll(path) t.Fatalf("Unable to create xlStorage test setup, %s", err) } - if err = xlStorage.MakeVol(volume); err != nil { + if err = xlStorage.MakeVol(context.Background(), volume); err != nil { os.RemoveAll(path) t.Fatalf("Unable to create volume %s: %v", volume, err) } @@ -1191,7 +1192,7 @@ func TestXLStorageReadFileWithVerify(t *testing.T) { os.RemoveAll(path) t.Fatalf("Unable to create generate random data: %v", err) } - if err = xlStorage.AppendFile(volume, object, data); err != nil { + if err = xlStorage.AppendFile(context.Background(), volume, object, data); err != nil { os.RemoveAll(path) t.Fatalf("Unable to create object: %v", err) } @@ -1204,7 +1205,7 @@ func TestXLStorageReadFileWithVerify(t *testing.T) { } buffer := make([]byte, test.length) - n, err := xlStorage.ReadFile(volume, test.file, int64(test.offset), buffer, NewBitrotVerifier(test.algorithm, h.Sum(nil))) + n, err := xlStorage.ReadFile(context.Background(), volume, test.file, int64(test.offset), buffer, NewBitrotVerifier(test.algorithm, h.Sum(nil))) switch { case err == nil && test.expError != nil: @@ -1227,7 +1228,7 @@ func TestXLStorageFormatFileChange(t *testing.T) { } defer os.RemoveAll(path) - if err = xlStorage.MakeVol(volume); err != nil { + if err = xlStorage.MakeVol(context.Background(), volume); err != nil { t.Fatalf("MakeVol failed with %s", err) } @@ -1236,7 +1237,7 @@ func TestXLStorageFormatFileChange(t *testing.T) { t.Fatalf("ioutil.WriteFile failed with %s", err) } - err = xlStorage.MakeVol(volume) + err = xlStorage.MakeVol(context.Background(), volume) if err != errVolumeExists { t.Fatalf("MakeVol expected to fail with errDiskNotFound but failed with %s", err) } @@ -1252,7 +1253,7 @@ func TestXLStorageAppendFile(t *testing.T) { defer os.RemoveAll(path) // Setup test environment. - if err = xlStorage.MakeVol("success-vol"); err != nil { + if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil { t.Fatalf("Unable to create volume, %s", err) } @@ -1282,7 +1283,7 @@ func TestXLStorageAppendFile(t *testing.T) { } for i, testCase := range testCases { - if err = xlStorage.AppendFile("success-vol", testCase.fileName, []byte("hello, world")); err != testCase.expectedErr { + if err = xlStorage.AppendFile(context.Background(), "success-vol", testCase.fileName, []byte("hello, world")); err != testCase.expectedErr { t.Errorf("Case: %d, expected: %s, got: %s", i+1, testCase.expectedErr, err) } } @@ -1308,14 +1309,14 @@ func TestXLStorageAppendFile(t *testing.T) { t.Fatalf("Unable to initialize xlStorage, %s", err) } - if err = xlStoragePermStorage.AppendFile("mybucket", "myobject", []byte("hello, world")); err != errFileAccessDenied { + if err = xlStoragePermStorage.AppendFile(context.Background(), "mybucket", "myobject", []byte("hello, world")); err != errFileAccessDenied { t.Fatalf("expected: Permission error, got: %s", err) } } // TestXLStorage case with invalid volume name. // A valid volume name should be atleast of size 3. - err = xlStorage.AppendFile("bn", "yes", []byte("hello, world")) + err = xlStorage.AppendFile(context.Background(), "bn", "yes", []byte("hello, world")) if err != errVolumeNotFound { t.Fatalf("expected: \"Invalid argument error\", got: \"%s\"", err) } @@ -1331,32 +1332,32 @@ func TestXLStorageRenameFile(t *testing.T) { defer os.RemoveAll(path) // Setup test environment. - if err := xlStorage.MakeVol("src-vol"); err != nil { + if err := xlStorage.MakeVol(context.Background(), "src-vol"); err != nil { t.Fatalf("Unable to create volume, %s", err) } - if err := xlStorage.MakeVol("dest-vol"); err != nil { + if err := xlStorage.MakeVol(context.Background(), "dest-vol"); err != nil { t.Fatalf("Unable to create volume, %s", err) } - if err := xlStorage.AppendFile("src-vol", "file1", []byte("Hello, world")); err != nil { + if err := xlStorage.AppendFile(context.Background(), "src-vol", "file1", []byte("Hello, world")); err != nil { t.Fatalf("Unable to create file, %s", err) } - if err := xlStorage.AppendFile("src-vol", "file2", []byte("Hello, world")); err != nil { + if err := xlStorage.AppendFile(context.Background(), "src-vol", "file2", []byte("Hello, world")); err != nil { t.Fatalf("Unable to create file, %s", err) } - if err := xlStorage.AppendFile("src-vol", "file3", []byte("Hello, world")); err != nil { + if err := xlStorage.AppendFile(context.Background(), "src-vol", "file3", []byte("Hello, world")); err != nil { t.Fatalf("Unable to create file, %s", err) } - if err := xlStorage.AppendFile("src-vol", "file4", []byte("Hello, world")); err != nil { + if err := xlStorage.AppendFile(context.Background(), "src-vol", "file4", []byte("Hello, world")); err != nil { t.Fatalf("Unable to create file, %s", err) } - if err := xlStorage.AppendFile("src-vol", "file5", []byte("Hello, world")); err != nil { + if err := xlStorage.AppendFile(context.Background(), "src-vol", "file5", []byte("Hello, world")); err != nil { t.Fatalf("Unable to create file, %s", err) } - if err := xlStorage.AppendFile("src-vol", "path/to/file1", []byte("Hello, world")); err != nil { + if err := xlStorage.AppendFile(context.Background(), "src-vol", "path/to/file1", []byte("Hello, world")); err != nil { t.Fatalf("Unable to create file, %s", err) } @@ -1533,7 +1534,7 @@ func TestXLStorageRenameFile(t *testing.T) { } for i, testCase := range testCases { - if err := xlStorage.RenameFile(testCase.srcVol, testCase.srcPath, testCase.destVol, testCase.destPath); err != testCase.expectedErr { + if err := xlStorage.RenameFile(context.Background(), testCase.srcVol, testCase.srcPath, testCase.destVol, testCase.destPath); err != testCase.expectedErr { t.Fatalf("TestXLStorage %d: Expected the error to be : \"%v\", got: \"%v\".", i+1, testCase.expectedErr, err) } } @@ -1549,15 +1550,15 @@ func TestXLStorageCheckFile(t *testing.T) { defer os.RemoveAll(path) // Setup test environment. - if err := xlStorage.MakeVol("success-vol"); err != nil { + if err := xlStorage.MakeVol(context.Background(), "success-vol"); err != nil { t.Fatalf("Unable to create volume, %s", err) } - if err := xlStorage.AppendFile("success-vol", pathJoin("success-file", xlStorageFormatFile), []byte("Hello, world")); err != nil { + if err := xlStorage.AppendFile(context.Background(), "success-vol", pathJoin("success-file", xlStorageFormatFile), []byte("Hello, world")); err != nil { t.Fatalf("Unable to create file, %s", err) } - if err := xlStorage.AppendFile("success-vol", pathJoin("path/to/success-file", xlStorageFormatFile), []byte("Hello, world")); err != nil { + if err := xlStorage.AppendFile(context.Background(), "success-vol", pathJoin("path/to/success-file", xlStorageFormatFile), []byte("Hello, world")); err != nil { t.Fatalf("Unable to create file, %s", err) } @@ -1611,7 +1612,7 @@ func TestXLStorageCheckFile(t *testing.T) { } for i, testCase := range testCases { - if err := xlStorage.CheckFile(testCase.srcVol, testCase.srcPath); err != testCase.expectedErr { + if err := xlStorage.CheckFile(context.Background(), testCase.srcVol, testCase.srcPath); err != testCase.expectedErr { t.Fatalf("TestXLStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) } } @@ -1634,7 +1635,7 @@ func TestXLStorageVerifyFile(t *testing.T) { volName := "testvol" fileName := "testfile" - if err := xlStorage.MakeVol(volName); err != nil { + if err := xlStorage.MakeVol(context.Background(), volName); err != nil { t.Fatal(err) } @@ -1648,7 +1649,7 @@ func TestXLStorageVerifyFile(t *testing.T) { h := algo.New() h.Write(data) hashBytes := h.Sum(nil) - if err := xlStorage.WriteAll(volName, fileName, bytes.NewBuffer(data)); err != nil { + if err := xlStorage.WriteAll(context.Background(), volName, fileName, bytes.NewBuffer(data)); err != nil { t.Fatal(err) } if err := xlStorage.storage.bitrotVerify(pathJoin(path, volName, fileName), size, algo, hashBytes, 0); err != nil { @@ -1656,7 +1657,7 @@ func TestXLStorageVerifyFile(t *testing.T) { } // 2) Whole-file bitrot check on corrupted file - if err := xlStorage.AppendFile(volName, fileName, []byte("a")); err != nil { + if err := xlStorage.AppendFile(context.Background(), volName, fileName, []byte("a")); err != nil { t.Fatal(err) } @@ -1670,7 +1671,7 @@ func TestXLStorageVerifyFile(t *testing.T) { t.Fatal("expected to fail bitrot check") } - if err := xlStorage.DeleteFile(volName, fileName); err != nil { + if err := xlStorage.DeleteFile(context.Background(), volName, fileName); err != nil { t.Fatal(err) } diff --git a/cmd/xl-storage_unix_test.go b/cmd/xl-storage_unix_test.go index c6ca52fca..5eb781a6f 100644 --- a/cmd/xl-storage_unix_test.go +++ b/cmd/xl-storage_unix_test.go @@ -19,6 +19,7 @@ package cmd import ( + "context" "io/ioutil" "os" "path" @@ -56,7 +57,7 @@ func TestIsValidUmaskVol(t *testing.T) { // Attempt to create a volume to verify the permissions later. // MakeVol creates 0777. - if err = disk.MakeVol(testCase.volName); err != nil { + if err = disk.MakeVol(context.Background(), testCase.volName); err != nil { t.Fatalf("Creating a volume failed with %s expected to pass.", err) } defer os.RemoveAll(tmpPath) @@ -98,7 +99,7 @@ func TestIsValidUmaskFile(t *testing.T) { // Attempt to create a volume to verify the permissions later. // MakeVol creates directory with 0777 perms. - if err = disk.MakeVol(testCase.volName); err != nil { + if err = disk.MakeVol(context.Background(), testCase.volName); err != nil { t.Fatalf("Creating a volume failed with %s expected to pass.", err) } @@ -106,12 +107,12 @@ func TestIsValidUmaskFile(t *testing.T) { // Attempt to create a file to verify the permissions later. // AppendFile creates file with 0666 perms. - if err = disk.AppendFile(testCase.volName, pathJoin("hello-world.txt", xlStorageFormatFile), []byte("Hello World")); err != nil { + if err = disk.AppendFile(context.Background(), testCase.volName, pathJoin("hello-world.txt", xlStorageFormatFile), []byte("Hello World")); err != nil { t.Fatalf("Create a file `test` failed with %s expected to pass.", err) } // CheckFile - stat the file. - if err := disk.CheckFile(testCase.volName, "hello-world.txt"); err != nil { + if err := disk.CheckFile(context.Background(), testCase.volName, "hello-world.txt"); err != nil { t.Fatalf("Stat failed with %s expected to pass.", err) } } diff --git a/cmd/xl-storage_windows_test.go b/cmd/xl-storage_windows_test.go index a3a30b150..633ce32fd 100644 --- a/cmd/xl-storage_windows_test.go +++ b/cmd/xl-storage_windows_test.go @@ -20,6 +20,7 @@ package cmd import ( "bytes" + "context" "fmt" "io/ioutil" "os" @@ -54,20 +55,20 @@ func TestUNCPaths(t *testing.T) { } // Create volume to use in conjunction with other StorageAPI's file API(s) - err = fs.MakeVol("voldir") + err = fs.MakeVol(context.Background(), "voldir") if err != nil { t.Fatal(err) } for i, test := range testCases { t.Run(fmt.Sprint(i), func(t *testing.T) { - err = fs.AppendFile("voldir", test.objName, []byte("hello")) + err = fs.AppendFile(context.Background(), "voldir", test.objName, []byte("hello")) if err != nil && test.pass { t.Error(err) } else if err == nil && !test.pass { t.Error(err) } - fs.DeleteFile("voldir", test.objName) + fs.DeleteFile(context.Background(), "voldir", test.objName) }) } } @@ -89,19 +90,19 @@ func TestUNCPathENOTDIR(t *testing.T) { } // Create volume to use in conjunction with other StorageAPI's file API(s) - err = fs.MakeVol("voldir") + err = fs.MakeVol(context.Background(), "voldir") if err != nil { t.Fatal(err) } - err = fs.AppendFile("voldir", "/file", []byte("hello")) + err = fs.AppendFile(context.Background(), "voldir", "/file", []byte("hello")) if err != nil { t.Fatal(err) } // Try to create a file that includes a file in its path components. // In *nix, this returns syscall.ENOTDIR while in windows we receive the following error. - err = fs.AppendFile("voldir", "/file/obj1", []byte("hello")) + err = fs.AppendFile(context.Background(), "voldir", "/file/obj1", []byte("hello")) if err != errFileAccessDenied { t.Errorf("expected: %s, got: %s", errFileAccessDenied, err) }