Support bucket versioning (#9377)

- Implement a new xl.json 2.0.0 format to support,
  this moves the entire marshaling logic to POSIX
  layer, top layer always consumes a common FileInfo
  construct which simplifies the metadata reads.
- Implement list object versions
- Migrate to siphash from crchash for new deployments
  for object placements.

Fixes #2111
This commit is contained in:
Harshavardhana
2020-06-12 20:04:01 -07:00
committed by GitHub
parent 43d6e3ae06
commit 4915433bd2
203 changed files with 13833 additions and 6919 deletions

View File

@@ -44,24 +44,31 @@ type StorageAPI interface {
StatVol(volume string) (vol VolInfo, err error)
DeleteVol(volume string, forceDelete bool) (err error)
// WalkVersions in sorted order directly on disk.
WalkVersions(volume, dirPath string, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error)
// Walk in sorted order directly on disk.
Walk(volume, dirPath string, marker string, recursive bool, leafFile string,
readMetadataFn readMetadataFunc, endWalkCh <-chan struct{}) (chan FileInfo, error)
Walk(volume, dirPath string, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error)
// Walk in sorted order directly on disk.
WalkSplunk(volume, dirPath string, marker string, endWalkCh <-chan struct{}) (chan FileInfo, error)
// Metadata operations
DeleteVersion(volume, path string, fi FileInfo) error
DeleteVersions(volume string, versions []FileInfo) []error
WriteMetadata(volume, path string, fi FileInfo) error
ReadVersion(volume, path, versionID string) (FileInfo, error)
RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath string) error
// File operations.
ListDir(volume, dirPath string, count int, leafFile string) ([]string, error)
ListDir(volume, dirPath string, count int) ([]string, error)
ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error)
AppendFile(volume string, path string, buf []byte) (err error)
CreateFile(volume, path string, size int64, reader io.Reader) error
ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error)
RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error
StatFile(volume string, path string) (file FileInfo, err error)
CheckParts(volume string, path string, fi FileInfo) error
CheckFile(volume string, path string) (err error)
DeleteFile(volume string, path string) (err error)
DeleteFileBulk(volume string, paths []string) (errs []error, err error)
DeletePrefixes(volume string, paths []string) (errs []error, err error)
VerifyFile(volume, path string, size int64, algo BitrotAlgorithm, sum []byte, shardSize int64) error
VerifyFile(volume, path string, fi FileInfo) error
// Write all data, syncs the data to disk.
WriteAll(volume string, path string, reader io.Reader) (err error)