Support bucket versioning (#9377)

- Implement a new xl.json 2.0.0 format to support,
  this moves the entire marshaling logic to POSIX
  layer, top layer always consumes a common FileInfo
  construct which simplifies the metadata reads.
- Implement list object versions
- Migrate to siphash from crchash for new deployments
  for object placements.

Fixes #2111

cmd/storage-rest-client.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018-2019 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2018-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -270,6 +270,33 @@ func (client *storageRESTClient) CreateFile(volume, path string, length int64, r
 	return err
 }
 
+func (client *storageRESTClient) WriteMetadata(volume, path string, fi FileInfo) error {
+	values := make(url.Values)
+	values.Set(storageRESTVolume, volume)
+	values.Set(storageRESTFilePath, path)
+
+	var reader bytes.Buffer
+	if err := gob.NewEncoder(&reader).Encode(fi); err != nil {
+		return err
+	}
+
+	respBody, err := client.call(storageRESTMethodWriteMetadata, values, &reader, -1)
+	defer http.DrainBody(respBody)
+	return err
+}
+
+func (client *storageRESTClient) DeleteVersion(volume, path string, fi FileInfo) error {
+	values := make(url.Values)
+	values.Set(storageRESTVolume, volume)
+	values.Set(storageRESTFilePath, path)
+	values.Set(storageRESTVersionID, fi.VersionID)
+	values.Set(storageRESTDeleteMarker, strconv.FormatBool(fi.Deleted))
+
+	respBody, err := client.call(storageRESTMethodDeleteVersion, values, nil, -1)
+	defer http.DrainBody(respBody)
+	return err
+}
+
 // WriteAll - write all data to a file.
 func (client *storageRESTClient) WriteAll(volume, path string, reader io.Reader) error {
 	values := make(url.Values)
@@ -280,18 +307,60 @@ func (client *storageRESTClient) WriteAll(volume, path string, reader io.Reader)
 	return err
 }
 
-// StatFile - stat a file.
-func (client *storageRESTClient) StatFile(volume, path string) (info FileInfo, err error) {
+// CheckFile - stat a file metadata.
+func (client *storageRESTClient) CheckFile(volume, path string) error {
 	values := make(url.Values)
 	values.Set(storageRESTVolume, volume)
 	values.Set(storageRESTFilePath, path)
-	respBody, err := client.call(storageRESTMethodStatFile, values, nil, -1)
+	respBody, err := client.call(storageRESTMethodCheckFile, values, nil, -1)
+	defer http.DrainBody(respBody)
+	return err
+}
+
+// CheckParts - stat all file parts.
+func (client *storageRESTClient) CheckParts(volume, path string, fi FileInfo) error {
+	values := make(url.Values)
+	values.Set(storageRESTVolume, volume)
+	values.Set(storageRESTFilePath, path)
+
+	var reader bytes.Buffer
+	if err := gob.NewEncoder(&reader).Encode(fi); err != nil {
+		return err
+	}
+
+	respBody, err := client.call(storageRESTMethodWriteMetadata, values, &reader, -1)
+	defer http.DrainBody(respBody)
+	return err
+}
+
+// RenameData - rename source path to destination path atomically, metadata and data file.
+func (client *storageRESTClient) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath string) (err error) {
+	values := make(url.Values)
+	values.Set(storageRESTSrcVolume, srcVolume)
+	values.Set(storageRESTSrcPath, srcPath)
+	values.Set(storageRESTDataDir, dataDir)
+	values.Set(storageRESTDstVolume, dstVolume)
+	values.Set(storageRESTDstPath, dstPath)
+	respBody, err := client.call(storageRESTMethodRenameData, values, nil, -1)
+	defer http.DrainBody(respBody)
+
+	return err
+}
+
+func (client *storageRESTClient) ReadVersion(volume, path, versionID string) (fi FileInfo, err error) {
+	values := make(url.Values)
+	values.Set(storageRESTVolume, volume)
+	values.Set(storageRESTFilePath, path)
+	values.Set(storageRESTVersionID, versionID)
+
+	respBody, err := client.call(storageRESTMethodReadVersion, values, nil, -1)
 	if err != nil {
-		return info, err
+		return fi, err
 	}
 	defer http.DrainBody(respBody)
-	err = gob.NewDecoder(respBody).Decode(&info)
-	return info, err
+
+	err = gob.NewDecoder(respBody).Decode(&fi)
+	return fi, err
 }
 
 // ReadAll - reads all contents of a file.
@@ -378,14 +447,47 @@ func (client *storageRESTClient) WalkSplunk(volume, dirPath, marker string, endW
 	return ch, nil
 }
 
-func (client *storageRESTClient) Walk(volume, dirPath, marker string, recursive bool, leafFile string,
-	readMetadataFn readMetadataFunc, endWalkCh <-chan struct{}) (chan FileInfo, error) {
+func (client *storageRESTClient) WalkVersions(volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error) {
+	values := make(url.Values)
+	values.Set(storageRESTVolume, volume)
+	values.Set(storageRESTDirPath, dirPath)
+	values.Set(storageRESTMarkerPath, marker)
+	values.Set(storageRESTRecursive, strconv.FormatBool(recursive))
+	respBody, err := client.call(storageRESTMethodWalk, values, nil, -1)
+	if err != nil {
+		return nil, err
+	}
+
+	ch := make(chan FileInfoVersions)
+	go func() {
+		defer close(ch)
+		defer http.DrainBody(respBody)
+
+		decoder := gob.NewDecoder(respBody)
+		for {
+			var fi FileInfoVersions
+			if gerr := decoder.Decode(&fi); gerr != nil {
+				// Upon error return
+				return
+			}
+			select {
+			case ch <- fi:
+			case <-endWalkCh:
+				return
+			}
+
+		}
+	}()
+
+	return ch, nil
+}
+
+func (client *storageRESTClient) Walk(volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) {
 	values := make(url.Values)
 	values.Set(storageRESTVolume, volume)
 	values.Set(storageRESTDirPath, dirPath)
 	values.Set(storageRESTMarkerPath, marker)
 	values.Set(storageRESTRecursive, strconv.FormatBool(recursive))
-	values.Set(storageRESTLeafFile, leafFile)
 	respBody, err := client.call(storageRESTMethodWalk, values, nil, -1)
 	if err != nil {
 		return nil, err
@@ -416,12 +518,11 @@ func (client *storageRESTClient) Walk(volume, dirPath, marker string, recursive
 }
 
 // ListDir - lists a directory.
-func (client *storageRESTClient) ListDir(volume, dirPath string, count int, leafFile string) (entries []string, err error) {
+func (client *storageRESTClient) ListDir(volume, dirPath string, count int) (entries []string, err error) {
 	values := make(url.Values)
 	values.Set(storageRESTVolume, volume)
 	values.Set(storageRESTDirPath, dirPath)
 	values.Set(storageRESTCount, strconv.Itoa(count))
-	values.Set(storageRESTLeafFile, leafFile)
 	respBody, err := client.call(storageRESTMethodListDir, values, nil, -1)
 	if err != nil {
 		return nil, err
@@ -441,78 +542,54 @@ func (client *storageRESTClient) DeleteFile(volume, path string) error {
 	return err
 }
 
-// DeleteFileBulk - deletes files in bulk.
-func (client *storageRESTClient) DeleteFileBulk(volume string, paths []string) (errs []error, err error) {
-	if len(paths) == 0 {
-		return errs, err
+// DeleteVersions - deletes list of specified versions if present
+func (client *storageRESTClient) DeleteVersions(volume string, versions []FileInfo) (errs []error) {
+	if len(versions) == 0 {
+		return errs
 	}
+
 	values := make(url.Values)
 	values.Set(storageRESTVolume, volume)
+	values.Set(storageRESTTotalVersions, strconv.Itoa(len(versions)))
 
 	var buffer bytes.Buffer
-	for _, path := range paths {
-		buffer.WriteString(path)
-		buffer.WriteString("\n")
+	encoder := gob.NewEncoder(&buffer)
+	for _, version := range versions {
+		encoder.Encode(&version)
 	}
 
-	respBody, err := client.call(storageRESTMethodDeleteFileBulk, values, &buffer, -1)
+	errs = make([]error, len(versions))
+
+	respBody, err := client.call(storageRESTMethodDeleteVersions, values, &buffer, -1)
 	defer http.DrainBody(respBody)
 	if err != nil {
-		return nil, err
+		for i := range errs {
+			errs[i] = err
+		}
+		return errs
 	}
 
 	reader, err := waitForHTTPResponse(respBody)
 	if err != nil {
-		return nil, err
+		for i := range errs {
+			errs[i] = err
+		}
+		return errs
 	}
 
-	dErrResp := &DeleteFileBulkErrsResp{}
+	dErrResp := &DeleteVersionsErrsResp{}
 	if err = gob.NewDecoder(reader).Decode(dErrResp); err != nil {
-		return nil, err
+		for i := range errs {
+			errs[i] = err
+		}
+		return errs
 	}
 
-	for _, dErr := range dErrResp.Errs {
-		errs = append(errs, toStorageErr(dErr))
+	for i, dErr := range dErrResp.Errs {
+		errs[i] = toStorageErr(dErr)
 	}
 
-	return errs, nil
-}
-
-// DeletePrefixes - deletes prefixes in bulk.
-func (client *storageRESTClient) DeletePrefixes(volume string, paths []string) (errs []error, err error) {
-	if len(paths) == 0 {
-		return errs, err
-	}
-	values := make(url.Values)
-	values.Set(storageRESTVolume, volume)
-
-	var buffer bytes.Buffer
-	for _, path := range paths {
-		buffer.WriteString(path)
-		buffer.WriteString("\n")
-	}
-
-	respBody, err := client.call(storageRESTMethodDeletePrefixes, values, &buffer, -1)
-	defer http.DrainBody(respBody)
-	if err != nil {
-		return nil, err
-	}
-
-	reader, err := waitForHTTPResponse(respBody)
-	if err != nil {
-		return nil, err
-	}
-
-	dErrResp := &DeletePrefixesErrsResp{}
-	if err = gob.NewDecoder(reader).Decode(dErrResp); err != nil {
-		return nil, err
-	}
-
-	for _, dErr := range dErrResp.Errs {
-		errs = append(errs, toStorageErr(dErr))
-	}
-
-	return errs, nil
+	return errs
 }
 
 // RenameFile - renames a file.
@@ -527,28 +604,32 @@ func (client *storageRESTClient) RenameFile(srcVolume, srcPath, dstVolume, dstPa
 	return err
 }
 
-func (client *storageRESTClient) VerifyFile(volume, path string, size int64, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
+func (client *storageRESTClient) VerifyFile(volume, path string, fi FileInfo) error {
 	values := make(url.Values)
 	values.Set(storageRESTVolume, volume)
 	values.Set(storageRESTFilePath, path)
-	values.Set(storageRESTBitrotAlgo, algo.String())
-	values.Set(storageRESTLength, strconv.FormatInt(size, 10))
-	values.Set(storageRESTShardSize, strconv.Itoa(int(shardSize)))
-	values.Set(storageRESTBitrotHash, hex.EncodeToString(sum))
 
-	respBody, err := client.call(storageRESTMethodVerifyFile, values, nil, -1)
+	var reader bytes.Buffer
+	if err := gob.NewEncoder(&reader).Encode(fi); err != nil {
+		return err
+	}
+
+	respBody, err := client.call(storageRESTMethodVerifyFile, values, &reader, -1)
 	defer http.DrainBody(respBody)
 	if err != nil {
 		return err
 	}
-	reader, err := waitForHTTPResponse(respBody)
+
+	respReader, err := waitForHTTPResponse(respBody)
 	if err != nil {
 		return err
 	}
+
 	verifyResp := &VerifyFileResp{}
-	if err = gob.NewDecoder(reader).Decode(verifyResp); err != nil {
+	if err = gob.NewDecoder(respReader).Decode(verifyResp); err != nil {
 		return err
 	}
+
 	return toStorageErr(verifyResp.Err)
 }