xl: code refactor, cleanup ReadFile and CreateFile.

This commit is contained in:
Krishna Srinivas 2016-04-20 21:02:47 +05:30 committed by Harshavardhana
parent 45b3d3e21f
commit 5c33b68318
4 changed files with 289 additions and 165 deletions

View File

@ -48,18 +48,82 @@ func closeAndRemoveWriters(writers ...io.WriteCloser) {
} }
} }
type quorumDisk struct {
disk StorageAPI
index int
}
// getQuorumDisks - get the current quorum disks.
func (xl XL) getQuorumDisks(volume, path string) (quorumDisks []quorumDisk, higherVersion int64) {
fileQuorumVersionMap := xl.getFileQuorumVersionMap(volume, path)
for diskIndex, formatVersion := range fileQuorumVersionMap {
if formatVersion > higherVersion {
higherVersion = formatVersion
quorumDisks = []quorumDisk{{
disk: xl.storageDisks[diskIndex],
index: diskIndex,
}}
} else if formatVersion == higherVersion {
quorumDisks = append(quorumDisks, quorumDisk{
disk: xl.storageDisks[diskIndex],
index: diskIndex,
})
}
}
return quorumDisks, higherVersion
}
func (xl XL) getFileQuorumVersionMap(volume, path string) map[int]int64 {
metadataFilePath := slashpath.Join(path, metadataFile)
// Set offset to 0 to read entire file.
offset := int64(0)
metadata := make(map[string]string)
// Allocate disk index format map - do not use maps directly
// without allocating.
fileQuorumVersionMap := make(map[int]int64)
// TODO - all errors should be logged here.
// Read meta data from all disks
for index, disk := range xl.storageDisks {
fileQuorumVersionMap[index] = -1
metadataReader, err := disk.ReadFile(volume, metadataFilePath, offset)
if err != nil {
continue
} else if err = json.NewDecoder(metadataReader).Decode(&metadata); err != nil {
continue
} else if _, ok := metadata["file.version"]; !ok {
fileQuorumVersionMap[index] = 0
}
// Convert string to integer.
fileVersion, err := strconv.ParseInt(metadata["file.version"], 10, 64)
if err != nil {
continue
}
fileQuorumVersionMap[index] = fileVersion
}
return fileQuorumVersionMap
}
// WriteErasure reads predefined blocks, encodes them and writes to // WriteErasure reads predefined blocks, encodes them and writes to
// configured storage disks. // configured storage disks.
func (xl XL) writeErasure(volume, path string, reader *io.PipeReader) { func (xl XL) writeErasure(volume, path string, reader *io.PipeReader) {
xl.lockNS(volume, path, false) xl.lockNS(volume, path, false)
defer xl.unlockNS(volume, path, false) defer xl.unlockNS(volume, path, false)
// get available quorum for existing file path // Get available quorum for existing file path.
_, higherVersion := xl.getQuorumDisks(volume, path) _, higherVersion := xl.getQuorumDisks(volume, path)
// increment to have next higher version // Increment to have next higher version.
higherVersion++ higherVersion++
quorumDisks := make([]quorumDisk, len(xl.storageDisks))
writers := make([]io.WriteCloser, len(xl.storageDisks)) writers := make([]io.WriteCloser, len(xl.storageDisks))
sha512Writers := make([]hash.Hash, len(xl.storageDisks)) sha512Writers := make([]hash.Hash, len(xl.storageDisks))
@ -70,15 +134,14 @@ func (xl XL) writeErasure(volume, path string, reader *io.PipeReader) {
modTime := time.Now().UTC() modTime := time.Now().UTC()
createFileError := 0 createFileError := 0
maxIndex := 0
for index, disk := range xl.storageDisks { for index, disk := range xl.storageDisks {
erasurePart := slashpath.Join(path, fmt.Sprintf("part.%d", index)) erasurePart := slashpath.Join(path, fmt.Sprintf("part.%d", index))
writer, err := disk.CreateFile(volume, erasurePart) writer, err := disk.CreateFile(volume, erasurePart)
if err != nil { if err != nil {
createFileError++ createFileError++
// we can safely allow CreateFile errors up to len(xl.storageDisks) - xl.writeQuorum // We can safely allow CreateFile errors up to len(xl.storageDisks) - xl.writeQuorum
// otherwise return failure // otherwise return failure.
if createFileError <= len(xl.storageDisks)-xl.writeQuorum { if createFileError <= len(xl.storageDisks)-xl.writeQuorum {
continue continue
} }
@ -95,8 +158,8 @@ func (xl XL) writeErasure(volume, path string, reader *io.PipeReader) {
if err != nil { if err != nil {
createFileError++ createFileError++
// we can safely allow CreateFile errors up to len(xl.storageDisks) - xl.writeQuorum // We can safely allow CreateFile errors up to
// otherwise return failure // len(xl.storageDisks) - xl.writeQuorum otherwise return failure.
if createFileError <= len(xl.storageDisks)-xl.writeQuorum { if createFileError <= len(xl.storageDisks)-xl.writeQuorum {
continue continue
} }
@ -107,19 +170,17 @@ func (xl XL) writeErasure(volume, path string, reader *io.PipeReader) {
return return
} }
writers[maxIndex] = writer writers[index] = writer
metadataWriters[maxIndex] = metadataWriter metadataWriters[index] = metadataWriter
sha512Writers[maxIndex] = fastSha512.New() sha512Writers[index] = fastSha512.New()
quorumDisks[maxIndex] = quorumDisk{disk, index}
maxIndex++
} }
// Allocate 4MiB block size buffer for reading. // Allocate 4MiB block size buffer for reading.
buffer := make([]byte, erasureBlockSize) dataBuffer := make([]byte, erasureBlockSize)
var totalSize int64 // Saves total incoming stream size. var totalSize int64 // Saves total incoming stream size.
for { for {
// Read up to allocated block size. // Read up to allocated block size.
n, err := io.ReadFull(reader, buffer) n, err := io.ReadFull(reader, dataBuffer)
if err != nil { if err != nil {
// Any unexpected errors, close the pipe reader with error. // Any unexpected errors, close the pipe reader with error.
if err != io.ErrUnexpectedEOF && err != io.EOF { if err != io.ErrUnexpectedEOF && err != io.EOF {
@ -135,16 +196,17 @@ func (xl XL) writeErasure(volume, path string, reader *io.PipeReader) {
} }
if n > 0 { if n > 0 {
// Split the input buffer into data and parity blocks. // Split the input buffer into data and parity blocks.
var blocks [][]byte var dataBlocks [][]byte
blocks, err = xl.ReedSolomon.Split(buffer[0:n]) dataBlocks, err = xl.ReedSolomon.Split(dataBuffer[0:n])
if err != nil { if err != nil {
// Remove all temp writers. // Remove all temp writers.
xl.cleanupCreateFileOps(volume, path, append(writers, metadataWriters...)...) xl.cleanupCreateFileOps(volume, path, append(writers, metadataWriters...)...)
reader.CloseWithError(err) reader.CloseWithError(err)
return return
} }
// Encode parity blocks using data blocks. // Encode parity blocks using data blocks.
err = xl.ReedSolomon.Encode(blocks) err = xl.ReedSolomon.Encode(dataBlocks)
if err != nil { if err != nil {
// Remove all temp writers upon error. // Remove all temp writers upon error.
xl.cleanupCreateFileOps(volume, path, append(writers, metadataWriters...)...) xl.cleanupCreateFileOps(volume, path, append(writers, metadataWriters...)...)
@ -153,18 +215,23 @@ func (xl XL) writeErasure(volume, path string, reader *io.PipeReader) {
} }
// Loop through and write encoded data to quorum disks. // Loop through and write encoded data to quorum disks.
for i := 0; i < maxIndex; i++ { for index, writer := range writers {
encodedData := blocks[quorumDisks[i].index] if writer == nil {
continue
_, err = writers[i].Write(encodedData) }
encodedData := dataBlocks[index]
_, err = writers[index].Write(encodedData)
if err != nil { if err != nil {
// Remove all temp writers upon error. // Remove all temp writers upon error.
xl.cleanupCreateFileOps(volume, path, append(writers, metadataWriters...)...) xl.cleanupCreateFileOps(volume, path, append(writers, metadataWriters...)...)
reader.CloseWithError(err) reader.CloseWithError(err)
return return
} }
sha512Writers[i].Write(encodedData) if sha512Writers[index] != nil {
sha512Writers[index].Write(encodedData)
}
} }
// Update total written. // Update total written.
totalSize += int64(n) totalSize += int64(n)
} }
@ -178,7 +245,8 @@ func (xl XL) writeErasure(volume, path string, reader *io.PipeReader) {
metadata["format.patch"] = "0" metadata["format.patch"] = "0"
metadata["file.size"] = strconv.FormatInt(totalSize, 10) metadata["file.size"] = strconv.FormatInt(totalSize, 10)
if len(xl.storageDisks) > len(writers) { if len(xl.storageDisks) > len(writers) {
// save file.version only if we wrote to less disks than all disks // Save file.version only if we wrote to less disks than all
// storage disks.
metadata["file.version"] = strconv.FormatInt(higherVersion, 10) metadata["file.version"] = strconv.FormatInt(higherVersion, 10)
} }
metadata["file.modTime"] = modTime.Format(timeFormatAMZ) metadata["file.modTime"] = modTime.Format(timeFormatAMZ)
@ -191,9 +259,14 @@ func (xl XL) writeErasure(volume, path string, reader *io.PipeReader) {
// Case: when storageDisks is 16 and write quorumDisks is 13, // Case: when storageDisks is 16 and write quorumDisks is 13,
// meta data write failure up to 2 can be considered. // meta data write failure up to 2 can be considered.
// currently we fail for any meta data writes // currently we fail for any meta data writes
for i := 0; i < maxIndex; i++ { for index, metadataWriter := range metadataWriters {
// Save sha512 checksum of each encoded blocks. if metadataWriter == nil {
metadata["file.xl.block512Sum"] = hex.EncodeToString(sha512Writers[i].Sum(nil)) continue
}
if sha512Writers[index] != nil {
// Save sha512 checksum of each encoded blocks.
metadata["file.xl.block512Sum"] = hex.EncodeToString(sha512Writers[index].Sum(nil))
}
// Marshal metadata into json strings. // Marshal metadata into json strings.
metadataBytes, err := json.Marshal(metadata) metadataBytes, err := json.Marshal(metadata)
@ -205,7 +278,7 @@ func (xl XL) writeErasure(volume, path string, reader *io.PipeReader) {
} }
// Write metadata to disk. // Write metadata to disk.
_, err = metadataWriters[i].Write(metadataBytes) _, err = metadataWriter.Write(metadataBytes)
if err != nil { if err != nil {
xl.cleanupCreateFileOps(volume, path, append(writers, metadataWriters...)...) xl.cleanupCreateFileOps(volume, path, append(writers, metadataWriters...)...)
reader.CloseWithError(err) reader.CloseWithError(err)
@ -214,10 +287,18 @@ func (xl XL) writeErasure(volume, path string, reader *io.PipeReader) {
} }
// Close all writers and metadata writers in routines. // Close all writers and metadata writers in routines.
for i := 0; i < maxIndex; i++ { for index, writer := range writers {
if writer == nil {
continue
}
// Safely wrote, now rename to its actual location. // Safely wrote, now rename to its actual location.
writers[i].Close() writer.Close()
metadataWriters[i].Close()
if metadataWriters[index] == nil {
continue
}
// Safely wrote, now rename to its actual location.
metadataWriters[index].Close()
} }
// Close the pipe reader and return. // Close the pipe reader and return.

View File

@ -26,11 +26,11 @@ import (
) )
func (xl XL) selfHeal(volume string, path string) error { func (xl XL) selfHeal(volume string, path string) error {
totalShards := xl.DataBlocks + xl.ParityBlocks totalBlocks := xl.DataBlocks + xl.ParityBlocks
needsSelfHeal := make([]bool, totalShards) needsSelfHeal := make([]bool, totalBlocks)
var metadata = make(map[string]string) var metadata = make(map[string]string)
var readers = make([]io.Reader, totalShards) var readers = make([]io.Reader, totalBlocks)
var writers = make([]io.WriteCloser, totalShards) var writers = make([]io.WriteCloser, totalBlocks)
for index, disk := range xl.storageDisks { for index, disk := range xl.storageDisks {
metadataFile := slashpath.Join(path, metadataFile) metadataFile := slashpath.Join(path, metadataFile)
@ -108,59 +108,59 @@ func (xl XL) selfHeal(volume string, path string) error {
} else { } else {
curBlockSize = int(totalLeft) curBlockSize = int(totalLeft)
} }
// Calculate the current shard size. // Calculate the current block size.
curShardSize := getEncodedBlockLen(curBlockSize, xl.DataBlocks) curBlockSize = getEncodedBlockLen(curBlockSize, xl.DataBlocks)
enShards := make([][]byte, totalShards) enBlocks := make([][]byte, totalBlocks)
// Loop through all readers and read. // Loop through all readers and read.
for index, reader := range readers { for index, reader := range readers {
// Initialize shard slice and fill the data from each parts. // Initialize block slice and fill the data from each parts.
// ReedSolomon.Verify() expects that slice is not nil even if the particular // ReedSolomon.Verify() expects that slice is not nil even if the particular
// part needs healing. // part needs healing.
enShards[index] = make([]byte, curShardSize) enBlocks[index] = make([]byte, curBlockSize)
if needsSelfHeal[index] { if needsSelfHeal[index] {
// Skip reading if the part needs healing. // Skip reading if the part needs healing.
continue continue
} }
_, e := io.ReadFull(reader, enShards[index]) _, e := io.ReadFull(reader, enBlocks[index])
if e != nil && e != io.ErrUnexpectedEOF { if e != nil && e != io.ErrUnexpectedEOF {
enShards[index] = nil enBlocks[index] = nil
} }
} }
// Check blocks if they are all zero in length. // Check blocks if they are all zero in length.
if checkBlockSize(enShards) == 0 { if checkBlockSize(enBlocks) == 0 {
err = errors.New("Data likely corrupted, all blocks are zero in length.") err = errors.New("Data likely corrupted, all blocks are zero in length.")
return err return err
} }
// Verify the shards. // Verify the blocks.
ok, e := xl.ReedSolomon.Verify(enShards) ok, e := xl.ReedSolomon.Verify(enBlocks)
if e != nil { if e != nil {
closeAndRemoveWriters(writers...) closeAndRemoveWriters(writers...)
return e return e
} }
// Verification failed, shards require reconstruction. // Verification failed, blocks require reconstruction.
if !ok { if !ok {
for index, shNeeded := range needsSelfHeal { for index, shNeeded := range needsSelfHeal {
if shNeeded { if shNeeded {
// Reconstructs() reconstructs the parts if the array is nil. // Reconstructs() reconstructs the parts if the array is nil.
enShards[index] = nil enBlocks[index] = nil
} }
} }
e = xl.ReedSolomon.Reconstruct(enShards) e = xl.ReedSolomon.Reconstruct(enBlocks)
if e != nil { if e != nil {
closeAndRemoveWriters(writers...) closeAndRemoveWriters(writers...)
return e return e
} }
// Verify reconstructed shards again. // Verify reconstructed blocks again.
ok, e = xl.ReedSolomon.Verify(enShards) ok, e = xl.ReedSolomon.Verify(enBlocks)
if e != nil { if e != nil {
closeAndRemoveWriters(writers...) closeAndRemoveWriters(writers...)
return e return e
} }
if !ok { if !ok {
// Shards cannot be reconstructed, corrupted data. // Blocks cannot be reconstructed, corrupted data.
e = errors.New("Verification failed after reconstruction, data likely corrupted.") e = errors.New("Verification failed after reconstruction, data likely corrupted.")
closeAndRemoveWriters(writers...) closeAndRemoveWriters(writers...)
return e return e
@ -170,7 +170,7 @@ func (xl XL) selfHeal(volume string, path string) error {
if !shNeeded { if !shNeeded {
continue continue
} }
_, e := writers[index].Write(enShards[index]) _, e := writers[index].Write(enBlocks[index])
if e != nil { if e != nil {
closeAndRemoveWriters(writers...) closeAndRemoveWriters(writers...)
return e return e

View File

@ -17,7 +17,6 @@
package main package main
import ( import (
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"io" "io"
@ -44,85 +43,66 @@ func getEncodedBlockLen(inputLen, dataBlocks int) (curBlockSize int) {
return return
} }
func (xl XL) getMetaFileVersionMap(volume, path string) (diskFileVersionMap map[int]int64) { // Returns slice of disks needed for ReadFile operation:
metadataFilePath := slashpath.Join(path, metadataFile) // - slice returing readable disks.
// Set offset to 0 to read entire file. // - file size
offset := int64(0) // - error if any.
metadata := make(map[string]string) func (xl XL) getReadableDisks(volume, path string) ([]StorageAPI, int64, error) {
partsMetadata, errs := xl.getPartsMetadata(volume, path)
// Allocate disk index format map - do not use maps directly without allocating. highestVersion := int64(0)
diskFileVersionMap = make(map[int]int64) versions := make([]int64, len(xl.storageDisks))
quorumDisks := make([]StorageAPI, len(xl.storageDisks))
// TODO - all errors should be logged here. fileSize := int64(0)
for index, metadata := range partsMetadata {
// Read meta data from all disks if errs[index] == nil {
for index, disk := range xl.storageDisks { if versionStr, ok := metadata["file.version"]; ok {
diskFileVersionMap[index] = -1 // Convert string to integer.
version, err := strconv.ParseInt(versionStr, 10, 64)
metadataReader, err := disk.ReadFile(volume, metadataFilePath, offset) if err != nil {
if err != nil { // Unexpected, return error.
continue return nil, 0, err
} else if err = json.NewDecoder(metadataReader).Decode(&metadata); err != nil { }
continue if version > highestVersion {
} else if _, ok := metadata["file.version"]; !ok { highestVersion = version
diskFileVersionMap[index] = 0 }
} versions[index] = version
// Convert string to integer. } else {
fileVersion, err := strconv.ParseInt(metadata["file.version"], 10, 64) versions[index] = 0
if err != nil { }
continue } else {
} versions[index] = -1
diskFileVersionMap[index] = fileVersion
}
return diskFileVersionMap
}
type quorumDisk struct {
disk StorageAPI
index int
}
// getQuorumDisks - get the current quorum disks.
func (xl XL) getQuorumDisks(volume, path string) (quorumDisks []quorumDisk, higherVersion int64) {
diskVersionMap := xl.getMetaFileVersionMap(volume, path)
for diskIndex, formatVersion := range diskVersionMap {
if formatVersion > higherVersion {
higherVersion = formatVersion
quorumDisks = []quorumDisk{{
disk: xl.storageDisks[diskIndex],
index: diskIndex,
}}
} else if formatVersion == higherVersion {
quorumDisks = append(quorumDisks, quorumDisk{
disk: xl.storageDisks[diskIndex],
index: diskIndex,
})
} }
} }
return
}
// getFileSize - extract file size from metadata. quorumCount := 0
func (xl XL) getFileSize(volume, path string, disk StorageAPI) (size int64, err error) { for index, version := range versions {
metadataFilePath := slashpath.Join(path, metadataFile) if version == highestVersion {
// set offset to 0 to read entire file quorumDisks[index] = xl.storageDisks[index]
offset := int64(0) quorumCount++
metadata := make(map[string]string) } else {
quorumDisks[index] = nil
metadataReader, err := disk.ReadFile(volume, metadataFilePath, offset) }
if err != nil { }
return 0, err if quorumCount < xl.readQuorum {
return nil, 0, errReadQuorum
} }
if err = json.NewDecoder(metadataReader).Decode(&metadata); err != nil { for index, disk := range quorumDisks {
return 0, err if disk == nil {
continue
}
if sizeStr, ok := partsMetadata[index]["file.size"]; ok {
var err error
fileSize, err = strconv.ParseInt(sizeStr, 10, 64)
if err != nil {
return nil, 0, err
}
break
} else {
return nil, 0, errors.New("Missing 'file.size' in meta data.")
}
} }
return quorumDisks, fileSize, nil
if _, ok := metadata["file.size"]; !ok {
return 0, errors.New("missing 'file.size' in meta data")
}
return strconv.ParseInt(metadata["file.size"], 10, 64)
} }
// ReadFile - read file // ReadFile - read file
@ -140,43 +120,23 @@ func (xl XL) ReadFile(volume, path string, offset int64) (io.ReadCloser, error)
xl.lockNS(volume, path, readLock) xl.lockNS(volume, path, readLock)
defer xl.unlockNS(volume, path, readLock) defer xl.unlockNS(volume, path, readLock)
// Check read quorum. quorumDisks, fileSize, err := xl.getReadableDisks(volume, path)
quorumDisks, _ := xl.getQuorumDisks(volume, path)
if len(quorumDisks) < xl.readQuorum {
return nil, errReadQuorum
}
// Get file size.
fileSize, err := xl.getFileSize(volume, path, quorumDisks[0].disk)
if err != nil { if err != nil {
return nil, err return nil, err
} }
totalBlocks := xl.DataBlocks + xl.ParityBlocks // Total blocks.
readers := make([]io.ReadCloser, len(quorumDisks)) readers := make([]io.ReadCloser, len(xl.storageDisks))
readFileError := 0 for index, disk := range quorumDisks {
for _, quorumDisk := range quorumDisks { if disk == nil {
erasurePart := slashpath.Join(path, fmt.Sprintf("part.%d", quorumDisk.index)) continue
var erasuredPartReader io.ReadCloser }
if erasuredPartReader, err = quorumDisk.disk.ReadFile(volume, erasurePart, offset); err != nil { erasurePart := slashpath.Join(path, fmt.Sprintf("part.%d", index))
// We can safely allow ReadFile errors up to len(quorumDisks) - xl.readQuorum // If disk.ReadFile returns error and we don't have read quorum it will be taken care as
// otherwise return failure // ReedSolomon.Reconstruct() will fail later.
if readFileError < len(quorumDisks)-xl.readQuorum { var reader io.ReadCloser
// Set the reader to 'nil' to be able to reconstruct later. if reader, err = disk.ReadFile(volume, erasurePart, offset); err == nil {
readers[quorumDisk.index] = nil readers[index] = reader
readFileError++
continue
}
// Control reaches here we do not have quorum
// anymore. Close all the readers.
for _, reader := range readers {
if reader != nil {
reader.Close()
}
}
return nil, errReadQuorum
} }
readers[quorumDisk.index] = erasuredPartReader
} }
// Initialize pipe. // Initialize pipe.
@ -194,19 +154,17 @@ func (xl XL) ReadFile(volume, path string, offset int64) (io.ReadCloser, error)
} }
// Calculate the current encoded block size. // Calculate the current encoded block size.
curEncBlockSize := getEncodedBlockLen(curBlockSize, xl.DataBlocks) curEncBlockSize := getEncodedBlockLen(curBlockSize, xl.DataBlocks)
enBlocks := make([][]byte, totalBlocks) enBlocks := make([][]byte, len(xl.storageDisks))
// Loop through all readers and read. // Loop through all readers and read.
for index, reader := range readers { for index, reader := range readers {
if reader == nil {
// One of files missing, save it for reconstruction.
enBlocks[index] = nil
continue
}
// Initialize shard slice and fill the data from each parts. // Initialize shard slice and fill the data from each parts.
enBlocks[index] = make([]byte, curEncBlockSize) enBlocks[index] = make([]byte, curEncBlockSize)
if reader == nil {
continue
}
_, err = io.ReadFull(reader, enBlocks[index]) _, err = io.ReadFull(reader, enBlocks[index])
if err != nil && err != io.ErrUnexpectedEOF { if err != nil && err != io.ErrUnexpectedEOF {
enBlocks[index] = nil readers[index] = nil
} }
} }
@ -229,6 +187,12 @@ func (xl XL) ReadFile(volume, path string, offset int64) (io.ReadCloser, error)
// Verification failed, blocks require reconstruction. // Verification failed, blocks require reconstruction.
if !ok { if !ok {
for index, reader := range readers {
if reader == nil {
// Reconstruct expects missing blocks to be nil.
enBlocks[index] = nil
}
}
err = xl.ReedSolomon.Reconstruct(enBlocks) err = xl.ReedSolomon.Reconstruct(enBlocks)
if err != nil { if err != nil {
pipeWriter.CloseWithError(err) pipeWriter.CloseWithError(err)
@ -264,6 +228,9 @@ func (xl XL) ReadFile(volume, path string, offset int64) (io.ReadCloser, error)
// Cleanly close all the underlying data readers. // Cleanly close all the underlying data readers.
for _, reader := range readers { for _, reader := range readers {
if reader == nil {
continue
}
reader.Close() reader.Close()
} }
}() }()

76
xl-v1-utils.go Normal file
View File

@ -0,0 +1,76 @@
package main
import (
"encoding/json"
"errors"
slashpath "path"
"path/filepath"
)
// Get parts.json metadata as a map slice.
// Returns error slice indicating the failed metadata reads.
func (xl XL) getPartsMetadata(volume, path string) ([]map[string]string, []error) {
errs := make([]error, len(xl.storageDisks))
metadataArray := make([]map[string]string, len(xl.storageDisks))
metadataFilePath := slashpath.Join(path, metadataFile)
for index, disk := range xl.storageDisks {
metadata := make(map[string]string)
offset := int64(0)
metadataReader, err := disk.ReadFile(volume, metadataFilePath, offset)
if err != nil {
errs[index] = err
continue
}
defer metadataReader.Close()
decoder := json.NewDecoder(metadataReader)
if err = decoder.Decode(&metadata); err != nil {
// Unable to parse parts.json, set error.
errs[index] = err
continue
}
metadataArray[index] = metadata
}
return metadataArray, errs
}
// Writes/Updates `parts.json` for given file. updateParts carries
// index of disks where `parts.json` needs to be updated.
//
// Returns collection of errors, indexed in accordance with input
// updateParts order.
func (xl XL) setPartsMetadata(volume, path string, metadata map[string]string, updateParts []bool) []error {
metadataFilePath := filepath.Join(path, metadataFile)
errs := make([]error, len(xl.storageDisks))
for index := range updateParts {
errs[index] = errors.New("metadata not updated")
}
metadataBytes, err := json.Marshal(metadata)
if err != nil {
for index := range updateParts {
errs[index] = err
}
return errs
}
for index, shouldUpdate := range updateParts {
if !shouldUpdate {
continue
}
writer, err := xl.storageDisks[index].CreateFile(volume, metadataFilePath)
errs[index] = err
if err != nil {
continue
}
_, err = writer.Write(metadataBytes)
if err != nil {
errs[index] = err
safeCloseAndRemove(writer)
continue
}
writer.Close()
}
return errs
}