MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 12:52:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add storage layer contexts (#10321)

Browse Source 
Add context to all (non-trivial) calls to the storage layer. 

Contexts are propagated through the REST client.

- `context.TODO()` is left in place for the places where it needs to be added to the caller.
- `endWalkCh` could probably be removed from the walkers, but no changes so far.

The "dangerous" part is that now a caller disconnecting *will* propagate down,  so a 
"delete" operation will now be interrupted. In some cases we might want to disconnect 
this functionality so the operation completes if it has started, leaving the system in a cleaner state.
This commit is contained in:
Klaus Post
2020-09-04 09:45:06 -07:00
committed by GitHub
parent 0037951b6e
commit 2d58a8d861
36 changed files with 466 additions and 467 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
cmd/xl-storage.go
View File

@@ -417,7 +417,7 @@ type DiskInfo struct {
// DiskInfo provides current information about disk space usage,
// total free inodes and underlying filesystem.
func (s *xlStorage) DiskInfo() (info DiskInfo, err error) {
func (s *xlStorage) DiskInfo(context.Context) (info DiskInfo, err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@@ -548,9 +548,9 @@ func (s *xlStorage) SetDiskID(id string) {
// storage rest server for remote disks.
}
func (s *xlStorage) MakeVolBulk(volumes ...string) (err error) {
func (s *xlStorage) MakeVolBulk(ctx context.Context, volumes ...string) (err error) {
for _, volume := range volumes {
if err = s.MakeVol(volume); err != nil {
if err = s.MakeVol(ctx, volume); err != nil {
if os.IsPermission(err) {
return errVolumeAccessDenied
}
@@ -560,7 +560,7 @@ func (s *xlStorage) MakeVolBulk(volumes ...string) (err error) {
}
// Make a volume entry.
func (s *xlStorage) MakeVol(volume string) (err error) {
func (s *xlStorage) MakeVol(ctx context.Context, volume string) (err error) {
if !isValidVolname(volume) {
return errInvalidArgument
}
@@ -594,7 +594,7 @@ func (s *xlStorage) MakeVol(volume string) (err error) {
}
// ListVols - list volumes.
func (s *xlStorage) ListVols() (volsInfo []VolInfo, err error) {
func (s *xlStorage) ListVols(context.Context) (volsInfo []VolInfo, err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@@ -654,7 +654,7 @@ func listVols(dirPath string) ([]VolInfo, error) {
}
// StatVol - get volume info.
func (s *xlStorage) StatVol(volume string) (volInfo VolInfo, err error) {
func (s *xlStorage) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@@ -686,7 +686,7 @@ func (s *xlStorage) StatVol(volume string) (volInfo VolInfo, err error) {
}
// DeleteVol - delete a volume.
func (s *xlStorage) DeleteVol(volume string, forceDelete bool) (err error) {
func (s *xlStorage) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@@ -806,7 +806,7 @@ func (s *xlStorage) isLeafDir(volume, leafPath string) bool {
// sorted order, additionally along with metadata about each of those entries.
// Implemented specifically for Splunk backend structure and List call with
// delimiter as "guidSplunk"
func (s *xlStorage) WalkSplunk(volume, dirPath, marker string, endWalkCh <-chan struct{}) (ch chan FileInfo, err error) {
func (s *xlStorage) WalkSplunk(ctx context.Context, volume, dirPath, marker string, endWalkCh <-chan struct{}) (ch chan FileInfo, err error) {
// Verify if volume is valid and it exists.
volumeDir, err := s.getVolDir(volume)
if err != nil {
@@ -877,7 +877,7 @@ func (s *xlStorage) WalkSplunk(volume, dirPath, marker string, endWalkCh <-chan
// WalkVersions - is a sorted walker which returns file entries in lexically sorted order,
// additionally along with metadata version info about each of those entries.
func (s *xlStorage) WalkVersions(volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (ch chan FileInfoVersions, err error) {
func (s *xlStorage) WalkVersions(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (ch chan FileInfoVersions, err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@@ -912,7 +912,7 @@ func (s *xlStorage) WalkVersions(volume, dirPath, marker string, recursive bool,
go func() {
defer close(ch)
listDir := func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string, delayIsLeaf bool) {
entries, err := s.ListDir(volume, dirPath, -1)
entries, err := s.ListDir(ctx, volume, dirPath, -1)
if err != nil {
return false, nil, false
}
@@ -962,7 +962,7 @@ func (s *xlStorage) WalkVersions(volume, dirPath, marker string, recursive bool,
// Walk - is a sorted walker which returns file entries in lexically
// sorted order, additionally along with metadata about each of those entries.
func (s *xlStorage) Walk(volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (ch chan FileInfo, err error) {
func (s *xlStorage) Walk(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (ch chan FileInfo, err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@@ -997,7 +997,7 @@ func (s *xlStorage) Walk(volume, dirPath, marker string, recursive bool, endWalk
go func() {
defer close(ch)
listDir := func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string, delayIsLeaf bool) {
entries, err := s.ListDir(volume, dirPath, -1)
entries, err := s.ListDir(ctx, volume, dirPath, -1)
if err != nil {
return false, nil, false
}
@@ -1046,7 +1046,7 @@ func (s *xlStorage) Walk(volume, dirPath, marker string, recursive bool, endWalk
// ListDir - return all the entries at the given directory path.
// If an entry is a directory it will be returned with a trailing SlashSeparator.
func (s *xlStorage) ListDir(volume, dirPath string, count int) (entries []string, err error) {
func (s *xlStorage) ListDir(ctx context.Context, volume, dirPath string, count int) (entries []string, err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@@ -1082,10 +1082,10 @@ func (s *xlStorage) ListDir(volume, dirPath string, count int) (entries []string
// DeleteVersions deletes slice of versions, it can be same object
// or multiple objects.
func (s *xlStorage) DeleteVersions(volume string, versions []FileInfo) []error {
func (s *xlStorage) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) []error {
errs := make([]error, len(versions))
for i, version := range versions {
if err := s.DeleteVersion(volume, version.Name, version); err != nil {
if err := s.DeleteVersion(ctx, volume, version.Name, version); err != nil {
errs[i] = err
}
}
@@ -1094,12 +1094,12 @@ func (s *xlStorage) DeleteVersions(volume string, versions []FileInfo) []error {
}
// DeleteVersion - deletes FileInfo metadata for path at `xl.meta`
func (s *xlStorage) DeleteVersion(volume, path string, fi FileInfo) error {
func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo) error {
if HasSuffix(path, SlashSeparator) {
return s.DeleteFile(volume, path)
return s.DeleteFile(ctx, volume, path)
}
buf, err := s.ReadAll(volume, pathJoin(path, xlStorageFormatFile))
buf, err := s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile))
if err != nil {
return err
}
@@ -1152,7 +1152,7 @@ func (s *xlStorage) DeleteVersion(volume, path string, fi FileInfo) error {
}
if !lastVersion {
return s.WriteAll(volume, pathJoin(path, xlStorageFormatFile), bytes.NewReader(buf))
return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), bytes.NewReader(buf))
}
// Delete the meta file, if there are no more versions the
@@ -1166,8 +1166,8 @@ func (s *xlStorage) DeleteVersion(volume, path string, fi FileInfo) error {
}
// WriteMetadata - writes FileInfo metadata for path at `xl.meta`
func (s *xlStorage) WriteMetadata(volume, path string, fi FileInfo) error {
buf, err := s.ReadAll(volume, pathJoin(path, xlStorageFormatFile))
func (s *xlStorage) WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) error {
buf, err := s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile))
if err != nil && err != errFileNotFound {
return err
}
@@ -1200,7 +1200,7 @@ func (s *xlStorage) WriteMetadata(volume, path string, fi FileInfo) error {
}
}
return s.WriteAll(volume, pathJoin(path, xlStorageFormatFile), bytes.NewReader(buf))
return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), bytes.NewReader(buf))
}
func (s *xlStorage) renameLegacyMetadata(volume, path string) error {
@@ -1268,14 +1268,14 @@ func (s *xlStorage) renameLegacyMetadata(volume, path string) error {
}
// ReadVersion - reads metadata and returns FileInfo at path `xl.meta`
func (s *xlStorage) ReadVersion(volume, path, versionID string) (fi FileInfo, err error) {
buf, err := s.ReadAll(volume, pathJoin(path, xlStorageFormatFile))
func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID string) (fi FileInfo, err error) {
buf, err := s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile))
if err != nil {
if err == errFileNotFound {
if err = s.renameLegacyMetadata(volume, path); err != nil {
return fi, err
}
buf, err = s.ReadAll(volume, pathJoin(path, xlStorageFormatFile))
buf, err = s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile))
if err != nil {
return fi, err
}
@@ -1300,7 +1300,7 @@ func (s *xlStorage) ReadVersion(volume, path, versionID string) (fi FileInfo, er
// as an error to be reported.
// This API is meant to be used on files which have small memory footprint, do
// not use this on large files as it would cause server to crash.
func (s *xlStorage) ReadAll(volume, path string) (buf []byte, err error) {
func (s *xlStorage) ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@@ -1363,7 +1363,7 @@ func (s *xlStorage) ReadAll(volume, path string) (buf []byte, err error) {
//
// Additionally ReadFile also starts reading from an offset. ReadFile
// semantics are same as io.ReadFull.
func (s *xlStorage) ReadFile(volume, path string, offset int64, buffer []byte, verifier *BitrotVerifier) (int64, error) {
func (s *xlStorage) ReadFile(ctx context.Context, volume string, path string, offset int64, buffer []byte, verifier *BitrotVerifier) (int64, error) {
if offset < 0 {
return 0, errInvalidArgument
}
@@ -1518,7 +1518,7 @@ func (s *xlStorage) openFile(volume, path string, mode int) (f *os.File, err err
}
// ReadFileStream - Returns the read stream of the file.
func (s *xlStorage) ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error) {
func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) {
if offset < 0 {
return nil, errInvalidArgument
}
@@ -1611,7 +1611,7 @@ func (c closeWrapper) Close() error {
}
// CreateFile - creates the file.
func (s *xlStorage) CreateFile(volume, path string, fileSize int64, r io.Reader) (err error) {
func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSize int64, r io.Reader) (err error) {
if fileSize < -1 {
return errInvalidArgument
}
@@ -1720,7 +1720,7 @@ func (s *xlStorage) CreateFile(volume, path string, fileSize int64, r io.Reader)
return nil
}
func (s *xlStorage) WriteAll(volume, path string, reader io.Reader) (err error) {
func (s *xlStorage) WriteAll(ctx context.Context, volume string, path string, reader io.Reader) (err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@@ -1742,7 +1742,7 @@ func (s *xlStorage) WriteAll(volume, path string, reader io.Reader) (err error)
// AppendFile - append a byte array at path, if file doesn't exist at
// path this call explicitly creates it.
func (s *xlStorage) AppendFile(volume, path string, buf []byte) (err error) {
func (s *xlStorage) AppendFile(ctx context.Context, volume string, path string, buf []byte) (err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@@ -1764,7 +1764,7 @@ func (s *xlStorage) AppendFile(volume, path string, buf []byte) (err error) {
}
// CheckParts check if path has necessary parts available.
func (s *xlStorage) CheckParts(volume, path string, fi FileInfo) error {
func (s *xlStorage) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@@ -1809,7 +1809,7 @@ func (s *xlStorage) CheckParts(volume, path string, fi FileInfo) error {
}
// CheckFile check if path has necessary metadata.
func (s *xlStorage) CheckFile(volume, path string) error {
func (s *xlStorage) CheckFile(ctx context.Context, volume string, path string) error {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@@ -1913,7 +1913,7 @@ func deleteFile(basePath, deletePath string, recursive bool) error {
}
// DeleteFile - delete a file at path.
func (s *xlStorage) DeleteFile(volume, path string) (err error) {
func (s *xlStorage) DeleteFile(ctx context.Context, volume string, path string) (err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@@ -1988,7 +1988,7 @@ func (s *xlStorage) DeleteFileBulk(volume string, paths []string) (errs []error,
}
// RenameData - rename source path to destination path atomically, metadata and data directory.
func (s *xlStorage) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath string) (err error) {
func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath, dataDir, dstVolume, dstPath string) (err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@@ -2170,7 +2170,7 @@ func (s *xlStorage) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath s
return errFileCorrupt
}
if err = s.WriteAll(srcVolume, pathJoin(srcPath, xlStorageFormatFile), bytes.NewReader(dstBuf)); err != nil {
if err = s.WriteAll(ctx, srcVolume, pathJoin(srcPath, xlStorageFormatFile), bytes.NewReader(dstBuf)); err != nil {
return err
}
@@ -2201,7 +2201,7 @@ func (s *xlStorage) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath s
}
// RenameFile - rename source path to destination path atomically.
func (s *xlStorage) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err error) {
func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) (err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@@ -2357,7 +2357,7 @@ func (s *xlStorage) bitrotVerify(partPath string, partSize int64, algo BitrotAlg
}
}
func (s *xlStorage) VerifyFile(volume, path string, fi FileInfo) (err error) {
func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)