MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 21:02:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: simplify data structure before release (#9968)

Browse Source 
- additionally upgrade to msgp@v1.1.2
- change StatModTime,StatSize fields as
  simple Size/ModTime
- reduce 50000 entries per List batch to 10000
  as client needs to wait too long to see the
  first batch some times which is not desired
  and it is worth we write the data as soon
  as we have it.
This commit is contained in:
Harshavardhana
2020-07-04 12:25:53 -07:00
committed by GitHub
parent cdb0e6ffed
commit c087a05b43
5 changed files with 71 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
cmd/xl-storage-format-v2.go
View File

@@ -156,8 +156,8 @@ type xlMetaV2Object struct {
PartETags []string `json:"PartETags" msg:"PartETags"` // Part ETags
PartSizes []int64 `json:"PartSizes" msg:"PartSizes"` // Part Sizes
PartActualSizes []int64 `json:"PartASizes,omitempty" msg:"PartASizes,omitempty"` // Part ActualSizes (compression)
StatSize int64 `json:"Size" msg:"Size"` // Object version size
StatModTime int64 `json:"MTime" msg:"MTime"` // Object version modified time
Size int64 `json:"Size" msg:"Size"` // Object version size
ModTime int64 `json:"MTime" msg:"MTime"` // Object version modified time
MetaSys map[string][]byte `json:"MetaSys,omitempty" msg:"MetaSys,omitempty"` // Object version internal metadata
MetaUser map[string]string `json:"MetaUsr,omitempty" msg:"MetaUsr,omitempty"` // Object version metadata set by user
}
@@ -184,7 +184,7 @@ func (j xlMetaV2Version) Valid() bool {
j.ObjectV2.ErasureAlgorithm.valid() &&
j.ObjectV2.BitrotChecksumAlgo.valid() &&
isXLMetaErasureInfoValid(j.ObjectV2.ErasureM, j.ObjectV2.ErasureN) &&
j.ObjectV2.StatModTime > 0
j.ObjectV2.ModTime > 0
case DeleteType:
return j.DeleteMarker != nil &&
j.DeleteMarker.ModTime > 0
@@ -271,8 +271,8 @@ func (z *xlMetaV2) AddVersion(fi FileInfo) error {
ObjectV2: &xlMetaV2Object{
VersionID: uv,
DataDir: dd,
StatSize: fi.Size,
StatModTime: fi.ModTime.UnixNano(),
Size: fi.Size,
ModTime: fi.ModTime.UnixNano(),
ErasureAlgorithm: ReedSolomon,
ErasureM: fi.Erasure.DataBlocks,
ErasureN: fi.Erasure.ParityBlocks,
@@ -373,8 +373,8 @@ func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) {
fi := FileInfo{
Volume: volume,
Name: path,
Size: j.StatSize,
ModTime: time.Unix(0, j.StatModTime).UTC(),
Size: j.Size,
ModTime: time.Unix(0, j.ModTime).UTC(),
VersionID: versionID,
}
fi.Parts = make([]ObjectPartInfo, len(j.PartNumbers))
@@ -463,7 +463,7 @@ func (z xlMetaV2) TotalSize() int64 {
for i := range z.Versions {
switch z.Versions[i].Type {
case ObjectType:
total += z.Versions[i].ObjectV2.StatSize
total += z.Versions[i].ObjectV2.Size
case LegacyType:
total += z.Versions[i].ObjectV1.Stat.Size
}
@@ -488,8 +488,6 @@ func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTim
fi, err = version.DeleteMarker.ToFileInfo(volume, path)
case LegacyType:
fi, err = version.ObjectV1.ToFileInfo(volume, path)
default:
continue
}
if err != nil {
return nil, latestModTime, err
@@ -498,12 +496,7 @@ func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTim
latestModTime = fi.ModTime
latestVersionID = fi.VersionID
}
switch version.Type {
case LegacyType:
fallthrough
case ObjectType, DeleteType:
versions = append(versions, fi)
}
versions = append(versions, fi)
}
// We didn't find the version in delete markers so latest version
@@ -521,44 +514,53 @@ func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTim
// ToFileInfo converts xlMetaV2 into a common FileInfo datastructure
// for consumption across callers.
func (z xlMetaV2) ToFileInfo(volume, path, versionID string) (FileInfo, error) {
func (z xlMetaV2) ToFileInfo(volume, path, versionID string) (fi FileInfo, err error) {
var uv uuid.UUID
if versionID != "" {
uv, _ = uuid.Parse(versionID)
}
var latestModTime time.Time
var latestIndex int
for i, version := range z.Versions {
if !version.Valid() {
logger.LogIf(GlobalContext, fmt.Errorf("invalid version detected %#v", version))
return FileInfo{}, errFileNotFound
}
var modTime time.Time
switch version.Type {
case ObjectType:
modTime = time.Unix(0, version.ObjectV2.ModTime)
case DeleteType:
modTime = time.Unix(0, version.DeleteMarker.ModTime)
case LegacyType:
modTime = version.ObjectV1.Stat.ModTime
}
if modTime.After(latestModTime) {
latestModTime = modTime
latestIndex = i
}
}
if versionID == "" {
var latestModTime time.Time
var latestIndex int
for i, version := range z.Versions {
if !version.Valid() {
logger.LogIf(GlobalContext, fmt.Errorf("invalid version detected %#v", version))
if len(z.Versions) >= 1 {
if !z.Versions[latestIndex].Valid() {
logger.LogIf(GlobalContext, fmt.Errorf("invalid version detected %#v", z.Versions[latestIndex]))
return FileInfo{}, errFileNotFound
}
var modTime time.Time
switch version.Type {
case ObjectType:
modTime = time.Unix(0, version.ObjectV2.StatModTime)
case DeleteType:
modTime = time.Unix(0, version.DeleteMarker.ModTime)
case LegacyType:
modTime = version.ObjectV1.Stat.ModTime
default:
continue
}
if modTime.After(latestModTime) {
latestModTime = modTime
latestIndex = i
}
}
if len(z.Versions) >= 1 {
switch z.Versions[latestIndex].Type {
case ObjectType:
return z.Versions[latestIndex].ObjectV2.ToFileInfo(volume, path)
fi, err = z.Versions[latestIndex].ObjectV2.ToFileInfo(volume, path)
fi.IsLatest = true
return fi, err
case DeleteType:
return z.Versions[latestIndex].DeleteMarker.ToFileInfo(volume, path)
fi, err = z.Versions[latestIndex].DeleteMarker.ToFileInfo(volume, path)
fi.IsLatest = true
return fi, err
case LegacyType:
return z.Versions[latestIndex].ObjectV1.ToFileInfo(volume, path)
fi, err = z.Versions[latestIndex].ObjectV1.ToFileInfo(volume, path)
fi.IsLatest = true
return fi, err
}
}
return FileInfo{}, errFileNotFound
@@ -575,23 +577,22 @@ func (z xlMetaV2) ToFileInfo(volume, path, versionID string) (FileInfo, error) {
switch version.Type {
case ObjectType:
if bytes.Equal(version.ObjectV2.VersionID[:], uv[:]) {
return version.ObjectV2.ToFileInfo(volume, path)
fi, err = version.ObjectV2.ToFileInfo(volume, path)
fi.IsLatest = latestModTime.Equal(fi.ModTime)
return fi, err
}
case LegacyType:
if version.ObjectV1.VersionID == versionID {
return version.ObjectV1.ToFileInfo(volume, path)
fi, err = version.ObjectV1.ToFileInfo(volume, path)
fi.IsLatest = latestModTime.Equal(fi.ModTime)
return fi, err
}
case DeleteType:
if bytes.Equal(version.DeleteMarker.VersionID[:], uv[:]) {
return version.DeleteMarker.ToFileInfo(volume, path)
fi, err = version.DeleteMarker.ToFileInfo(volume, path)
fi.IsLatest = latestModTime.Equal(fi.ModTime)
return fi, err
}
default:
logger.LogIf(GlobalContext, fmt.Errorf("unknown version type: %v", version.Type))
if versionID == "" {
return FileInfo{}, errFileNotFound
}
return FileInfo{}, errFileVersionNotFound
}
}