MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-10 05:59:43 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: implement readMetadata common function for re-use (#12353)

Browse Source 
Previous PR #12351 added functions to read from the reader
stream to reduce memory usage, use the same technique in
few other places where we are not interested in reading the
data part.
This commit is contained in:
Harshavardhana
2021-05-21 11:41:25 -07:00
committed by GitHub
parent 9d1b6fb37d
commit 0287711dc9
6 changed files with 25 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
cmd/metacache-walk.go
View File

@@ -88,7 +88,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
// Fast exit track to check if we are listing an object with
// a trailing slash, this will avoid to list the object content.
if HasSuffix(opts.BaseDir, SlashSeparator) {
metadata, err := xioutil.ReadFile(pathJoin(volumeDir,
metadata, err := s.readMetadata(pathJoin(volumeDir,
opts.BaseDir[:len(opts.BaseDir)-1]+globalDirSuffix,
xlStorageFormatFile))
if err == nil {
@@ -97,7 +97,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
// behavior.
out <- metaCacheEntry{
name: opts.BaseDir,
metadata: xlMetaV2TrimData(metadata),
metadata: metadata,
}
} else {
if st, err := os.Lstat(pathJoin(volumeDir, opts.BaseDir, xlStorageFormatFile)); err == nil && st.Mode().IsRegular() {
@@ -154,25 +154,11 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
// If root was an object return it as such.
if HasSuffix(entry, xlStorageFormatFile) {
var meta metaCacheEntry
f, err := os.OpenFile(pathJoin(volumeDir, current, entry), readMode, 0)
meta.metadata, err = s.readMetadata(pathJoin(volumeDir, current, entry))
if err != nil {
logger.LogIf(ctx, err)
continue
}
stat, err := f.Stat()
if err != nil {
logger.LogIf(ctx, err)
f.Close()
continue
}
meta.metadata, err = readXLMetaNoData(f, stat.Size())
if err != nil {
logger.LogIf(ctx, err)
f.Close()
continue
}
f.Close()
meta.metadata = xlMetaV2TrimData(meta.metadata)
meta.name = strings.TrimSuffix(entry, xlStorageFormatFile)
meta.name = strings.TrimSuffix(meta.name, SlashSeparator)
meta.name = pathJoin(current, meta.name)
@@ -233,7 +219,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
meta.name = meta.name[:len(meta.name)-1] + globalDirSuffixWithSlash
}
meta.metadata, err = xioutil.ReadFile(pathJoin(volumeDir, meta.name, xlStorageFormatFile))
meta.metadata, err = s.readMetadata(pathJoin(volumeDir, meta.name, xlStorageFormatFile))
switch {
case err == nil:
// It was an object

50
cmd/xl-storage-format-v2.go
View File

@@ -1420,7 +1420,7 @@ func readXLMetaNoData(r io.Reader, size int64) ([]byte, error) {
err = readMore(size)
return buf, err
case 1, 2:
sz, tmp, err := ReadBytesHeader(tmp)
sz, tmp, err := msgp.ReadBytesHeader(tmp)
if err != nil {
return nil, err
}
@@ -1459,51 +1459,3 @@ func readXLMetaNoData(r io.Reader, size int64) ([]byte, error) {
return nil, errors.New("unknown major metadata version")
}
}
// ReadBytesHeader reads the 'bin' header size
// off of 'b' and returns the size and remaining bytes.
// Possible errors:
// - ErrShortBytes (too few bytes)
// - TypeError{} (not a bin object)
// TODO: Replace when https://github.com/tinylib/msgp/pull/289 is merged.
func ReadBytesHeader(b []byte) (sz uint32, o []byte, err error) {
if len(b) < 1 {
return 0, nil, msgp.ErrShortBytes
}
var big = binary.BigEndian
const (
mbin8 uint8 = 0xc4
mbin16 uint8 = 0xc5
mbin32 uint8 = 0xc6
)
switch b[0] {
case mbin8:
if len(b) < 2 {
err = msgp.ErrShortBytes
return
}
sz = uint32(b[1])
o = b[2:]
return
case mbin16:
if len(b) < 3 {
err = msgp.ErrShortBytes
return
}
sz = uint32(big.Uint16(b[1:]))
o = b[3:]
return
case mbin32:
if len(b) < 5 {
err = msgp.ErrShortBytes
return
}
sz = big.Uint32(b[1:])
o = b[5:]
return
default:
err = msgp.TypeError{Method: msgp.BinType, Encoded: msgp.NextType(b)}
return
}
}

31
cmd/xl-storage.go
View File

@@ -384,6 +384,19 @@ func (s *xlStorage) Healing() *healingTracker {
return &h
}
func (s *xlStorage) readMetadata(itemPath string) ([]byte, error) {
f, err := os.OpenFile(itemPath, readMode, 0)
if err != nil {
return nil, err
}
defer f.Close()
stat, err := f.Stat()
if err != nil {
return nil, err
}
return readXLMetaNoData(f, stat.Size())
}
func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry) (dataUsageCache, error) {
// Updates must be closed before we return.
defer close(updates)
@@ -421,29 +434,13 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
return sizeSummary{}, errSkipFile
}
f, err := os.OpenFile(item.Path, readMode, 0)
buf, err := s.readMetadata(item.Path)
if err != nil {
if intDataUpdateTracker.debug {
console.Debugf(color.Green("scannerBucket:")+" object path missing: %v: %w\n", item.Path, err)
}
return sizeSummary{}, errSkipFile
}
defer f.Close()
stat, err := f.Stat()
if err != nil {
if intDataUpdateTracker.debug {
console.Debugf(color.Green("scannerBucket:")+" stat failed: %v: %w\n", item.Path, err)
}
return sizeSummary{}, errSkipFile
}
buf, err := readXLMetaNoData(f, stat.Size())
if err != nil {
if intDataUpdateTracker.debug {
console.Debugf(color.Green("scannerBucket:")+" readXLMetaNoData: %v: %w\n", item.Path, err)
}
return sizeSummary{}, errSkipFile
}
// Remove filename which is the meta file.
item.transformMetaDir()