MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-06 20:33:07 -05:00
Code Issues Packages Projects Releases Wiki Activity

Inline small file data in xl.meta file (#11758)

Browse Source
This commit is contained in:
Klaus Post
2021-03-30 02:00:55 +02:00
committed by GitHub
parent f5831174e6
commit 2623338dc5
19 changed files with 1311 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
cmd/erasure-object.go
View File

@@ -17,6 +17,7 @@
package cmd
import (
"bytes"
"context"
"errors"
"fmt"
@@ -735,10 +736,21 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
}()
writers := make([]io.Writer, len(onlineDisks))
dataSize := data.Size()
var inlineBuffers []*bytes.Buffer
if dataSize >= 0 && dataSize < smallFileThreshold {
inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
}
for i, disk := range onlineDisks {
if disk == nil {
continue
}
if len(inlineBuffers) > 0 {
inlineBuffers[i] = bytes.NewBuffer(make([]byte, 0, erasure.ShardFileSize(data.Size())))
writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
continue
}
writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, tempErasureObj,
erasure.ShardFileSize(data.Size()), DefaultBitrotAlgorithm, erasure.ShardSize(), false)
}
@@ -770,6 +782,9 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
onlineDisks[i] = nil
continue
}
if len(inlineBuffers) > 0 && inlineBuffers[i] != nil {
partsMetadata[i].Data = inlineBuffers[i].Bytes()
}
partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize())
partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{
PartNumber: 1,
@@ -797,16 +812,29 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
partsMetadata[index].Metadata = opts.UserDefined
partsMetadata[index].Size = n
partsMetadata[index].ModTime = modTime
if len(inlineBuffers) > 0 && inlineBuffers[index] != nil {
partsMetadata[index].Data = inlineBuffers[index].Bytes()
}
}
// Write unique `xl.meta` for each disk.
if onlineDisks, err = writeUniqueFileInfo(ctx, onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, writeQuorum); err != nil {
return ObjectInfo{}, toObjectErr(err, bucket, object)
}
if len(inlineBuffers) == 0 {
if onlineDisks, err = writeUniqueFileInfo(ctx, onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, writeQuorum); err != nil {
logger.LogIf(ctx, err)
return ObjectInfo{}, toObjectErr(err, bucket, object)
}
// Rename the successfully written temporary object to final location.
if onlineDisks, err = renameData(ctx, onlineDisks, minioMetaTmpBucket, tempObj, fi.DataDir, bucket, object, writeQuorum, nil); err != nil {
return ObjectInfo{}, toObjectErr(err, bucket, object)
// Rename the successfully written temporary object to final location.
if onlineDisks, err = renameData(ctx, onlineDisks, minioMetaTmpBucket, tempObj, fi.DataDir, bucket, object, writeQuorum, nil); err != nil {
logger.LogIf(ctx, err)
return ObjectInfo{}, toObjectErr(err, bucket, object)
}
} else {
// Write directly...
if onlineDisks, err = writeUniqueFileInfo(ctx, onlineDisks, bucket, object, partsMetadata, writeQuorum); err != nil {
logger.LogIf(ctx, err)
return ObjectInfo{}, toObjectErr(err, bucket, object)
}
}
// Whether a disk was initially or becomes offline