From c087a05b43a390070e4174ae5f03911aecac1b8a Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sat, 4 Jul 2020 12:25:53 -0700 Subject: [PATCH] fix: simplify data structure before release (#9968) - additionally upgrade to msgp@v1.1.2 - change StatModTime,StatSize fields as simple Size/ModTime - reduce 50000 entries per List batch to 10000 as client needs to wait too long to see the first batch some times which is not desired and it is worth we write the data as soon as we have it. --- cmd/api-response.go | 2 +- cmd/xl-storage-format-v2.go | 105 ++++++++++++++++---------------- cmd/xl-storage-format-v2_gen.go | 28 ++++----- go.mod | 2 +- go.sum | 4 +- 5 files changed, 71 insertions(+), 70 deletions(-) diff --git a/cmd/api-response.go b/cmd/api-response.go index ec3bfac45..d4e22782c 100644 --- a/cmd/api-response.go +++ b/cmd/api-response.go @@ -35,7 +35,7 @@ import ( const ( // RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision. - maxObjectList = 50000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse. + maxObjectList = 10000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse. maxDeleteList = 10000 // Limit number of objects deleted in a delete call. maxUploadsList = 10000 // Limit number of uploads in a listUploadsResponse. maxPartsList = 10000 // Limit number of parts in a listPartsResponse. diff --git a/cmd/xl-storage-format-v2.go b/cmd/xl-storage-format-v2.go index 91bd11438..f00ec154f 100644 --- a/cmd/xl-storage-format-v2.go +++ b/cmd/xl-storage-format-v2.go @@ -156,8 +156,8 @@ type xlMetaV2Object struct { PartETags []string `json:"PartETags" msg:"PartETags"` // Part ETags PartSizes []int64 `json:"PartSizes" msg:"PartSizes"` // Part Sizes PartActualSizes []int64 `json:"PartASizes,omitempty" msg:"PartASizes,omitempty"` // Part ActualSizes (compression) - StatSize int64 `json:"Size" msg:"Size"` // Object version size - StatModTime int64 `json:"MTime" msg:"MTime"` // Object version modified time + Size int64 `json:"Size" msg:"Size"` // Object version size + ModTime int64 `json:"MTime" msg:"MTime"` // Object version modified time MetaSys map[string][]byte `json:"MetaSys,omitempty" msg:"MetaSys,omitempty"` // Object version internal metadata MetaUser map[string]string `json:"MetaUsr,omitempty" msg:"MetaUsr,omitempty"` // Object version metadata set by user } @@ -184,7 +184,7 @@ func (j xlMetaV2Version) Valid() bool { j.ObjectV2.ErasureAlgorithm.valid() && j.ObjectV2.BitrotChecksumAlgo.valid() && isXLMetaErasureInfoValid(j.ObjectV2.ErasureM, j.ObjectV2.ErasureN) && - j.ObjectV2.StatModTime > 0 + j.ObjectV2.ModTime > 0 case DeleteType: return j.DeleteMarker != nil && j.DeleteMarker.ModTime > 0 @@ -271,8 +271,8 @@ func (z *xlMetaV2) AddVersion(fi FileInfo) error { ObjectV2: &xlMetaV2Object{ VersionID: uv, DataDir: dd, - StatSize: fi.Size, - StatModTime: fi.ModTime.UnixNano(), + Size: fi.Size, + ModTime: fi.ModTime.UnixNano(), ErasureAlgorithm: ReedSolomon, ErasureM: fi.Erasure.DataBlocks, ErasureN: fi.Erasure.ParityBlocks, @@ -373,8 +373,8 @@ func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) { fi := FileInfo{ Volume: volume, Name: path, - Size: j.StatSize, - ModTime: time.Unix(0, j.StatModTime).UTC(), + Size: j.Size, + ModTime: time.Unix(0, j.ModTime).UTC(), VersionID: versionID, } fi.Parts = make([]ObjectPartInfo, len(j.PartNumbers)) @@ -463,7 +463,7 @@ func (z xlMetaV2) TotalSize() int64 { for i := range z.Versions { switch z.Versions[i].Type { case ObjectType: - total += z.Versions[i].ObjectV2.StatSize + total += z.Versions[i].ObjectV2.Size case LegacyType: total += z.Versions[i].ObjectV1.Stat.Size } @@ -488,8 +488,6 @@ func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTim fi, err = version.DeleteMarker.ToFileInfo(volume, path) case LegacyType: fi, err = version.ObjectV1.ToFileInfo(volume, path) - default: - continue } if err != nil { return nil, latestModTime, err @@ -498,12 +496,7 @@ func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTim latestModTime = fi.ModTime latestVersionID = fi.VersionID } - switch version.Type { - case LegacyType: - fallthrough - case ObjectType, DeleteType: - versions = append(versions, fi) - } + versions = append(versions, fi) } // We didn't find the version in delete markers so latest version @@ -521,44 +514,53 @@ func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTim // ToFileInfo converts xlMetaV2 into a common FileInfo datastructure // for consumption across callers. -func (z xlMetaV2) ToFileInfo(volume, path, versionID string) (FileInfo, error) { +func (z xlMetaV2) ToFileInfo(volume, path, versionID string) (fi FileInfo, err error) { var uv uuid.UUID if versionID != "" { uv, _ = uuid.Parse(versionID) } + var latestModTime time.Time + var latestIndex int + for i, version := range z.Versions { + if !version.Valid() { + logger.LogIf(GlobalContext, fmt.Errorf("invalid version detected %#v", version)) + return FileInfo{}, errFileNotFound + } + var modTime time.Time + switch version.Type { + case ObjectType: + modTime = time.Unix(0, version.ObjectV2.ModTime) + case DeleteType: + modTime = time.Unix(0, version.DeleteMarker.ModTime) + case LegacyType: + modTime = version.ObjectV1.Stat.ModTime + } + if modTime.After(latestModTime) { + latestModTime = modTime + latestIndex = i + } + } + if versionID == "" { - var latestModTime time.Time - var latestIndex int - for i, version := range z.Versions { - if !version.Valid() { - logger.LogIf(GlobalContext, fmt.Errorf("invalid version detected %#v", version)) + if len(z.Versions) >= 1 { + if !z.Versions[latestIndex].Valid() { + logger.LogIf(GlobalContext, fmt.Errorf("invalid version detected %#v", z.Versions[latestIndex])) return FileInfo{}, errFileNotFound } - var modTime time.Time - switch version.Type { - case ObjectType: - modTime = time.Unix(0, version.ObjectV2.StatModTime) - case DeleteType: - modTime = time.Unix(0, version.DeleteMarker.ModTime) - case LegacyType: - modTime = version.ObjectV1.Stat.ModTime - default: - continue - } - if modTime.After(latestModTime) { - latestModTime = modTime - latestIndex = i - } - } - if len(z.Versions) >= 1 { switch z.Versions[latestIndex].Type { case ObjectType: - return z.Versions[latestIndex].ObjectV2.ToFileInfo(volume, path) + fi, err = z.Versions[latestIndex].ObjectV2.ToFileInfo(volume, path) + fi.IsLatest = true + return fi, err case DeleteType: - return z.Versions[latestIndex].DeleteMarker.ToFileInfo(volume, path) + fi, err = z.Versions[latestIndex].DeleteMarker.ToFileInfo(volume, path) + fi.IsLatest = true + return fi, err case LegacyType: - return z.Versions[latestIndex].ObjectV1.ToFileInfo(volume, path) + fi, err = z.Versions[latestIndex].ObjectV1.ToFileInfo(volume, path) + fi.IsLatest = true + return fi, err } } return FileInfo{}, errFileNotFound @@ -575,23 +577,22 @@ func (z xlMetaV2) ToFileInfo(volume, path, versionID string) (FileInfo, error) { switch version.Type { case ObjectType: if bytes.Equal(version.ObjectV2.VersionID[:], uv[:]) { - return version.ObjectV2.ToFileInfo(volume, path) + fi, err = version.ObjectV2.ToFileInfo(volume, path) + fi.IsLatest = latestModTime.Equal(fi.ModTime) + return fi, err } case LegacyType: if version.ObjectV1.VersionID == versionID { - return version.ObjectV1.ToFileInfo(volume, path) + fi, err = version.ObjectV1.ToFileInfo(volume, path) + fi.IsLatest = latestModTime.Equal(fi.ModTime) + return fi, err } case DeleteType: if bytes.Equal(version.DeleteMarker.VersionID[:], uv[:]) { - return version.DeleteMarker.ToFileInfo(volume, path) + fi, err = version.DeleteMarker.ToFileInfo(volume, path) + fi.IsLatest = latestModTime.Equal(fi.ModTime) + return fi, err } - default: - logger.LogIf(GlobalContext, fmt.Errorf("unknown version type: %v", version.Type)) - if versionID == "" { - return FileInfo{}, errFileNotFound - } - - return FileInfo{}, errFileVersionNotFound } } diff --git a/cmd/xl-storage-format-v2_gen.go b/cmd/xl-storage-format-v2_gen.go index da111d395..bc604eb81 100644 --- a/cmd/xl-storage-format-v2_gen.go +++ b/cmd/xl-storage-format-v2_gen.go @@ -606,15 +606,15 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) { } } case "Size": - z.StatSize, err = dc.ReadInt64() + z.Size, err = dc.ReadInt64() if err != nil { - err = msgp.WrapError(err, "StatSize") + err = msgp.WrapError(err, "Size") return } case "MTime": - z.StatModTime, err = dc.ReadInt64() + z.ModTime, err = dc.ReadInt64() if err != nil { - err = msgp.WrapError(err, "StatModTime") + err = msgp.WrapError(err, "ModTime") return } case "MetaSys": @@ -885,9 +885,9 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) { if err != nil { return } - err = en.WriteInt64(z.StatSize) + err = en.WriteInt64(z.Size) if err != nil { - err = msgp.WrapError(err, "StatSize") + err = msgp.WrapError(err, "Size") return } // write "MTime" @@ -895,9 +895,9 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) { if err != nil { return } - err = en.WriteInt64(z.StatModTime) + err = en.WriteInt64(z.ModTime) if err != nil { - err = msgp.WrapError(err, "StatModTime") + err = msgp.WrapError(err, "ModTime") return } if (zb0001Mask & 0x8000) == 0 { // if not empty @@ -1032,10 +1032,10 @@ func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) { } // string "Size" o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65) - o = msgp.AppendInt64(o, z.StatSize) + o = msgp.AppendInt64(o, z.Size) // string "MTime" o = append(o, 0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65) - o = msgp.AppendInt64(o, z.StatModTime) + o = msgp.AppendInt64(o, z.ModTime) if (zb0001Mask & 0x8000) == 0 { // if not empty // string "MetaSys" o = append(o, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73) @@ -1227,15 +1227,15 @@ func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) { } } case "Size": - z.StatSize, bts, err = msgp.ReadInt64Bytes(bts) + z.Size, bts, err = msgp.ReadInt64Bytes(bts) if err != nil { - err = msgp.WrapError(err, "StatSize") + err = msgp.WrapError(err, "Size") return } case "MTime": - z.StatModTime, bts, err = msgp.ReadInt64Bytes(bts) + z.ModTime, bts, err = msgp.ReadInt64Bytes(bts) if err != nil { - err = msgp.WrapError(err, "StatModTime") + err = msgp.WrapError(err, "ModTime") return } case "MetaSys": diff --git a/go.mod b/go.mod index e9669b3d3..fc386f618 100644 --- a/go.mod +++ b/go.mod @@ -106,7 +106,7 @@ require ( github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94 github.com/tidwall/gjson v1.3.5 github.com/tidwall/sjson v1.0.4 - github.com/tinylib/msgp v1.1.1 + github.com/tinylib/msgp v1.1.2 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect github.com/ugorji/go v1.1.5-pre // indirect github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a diff --git a/go.sum b/go.sum index dc15314c7..a73d9b20d 100644 --- a/go.sum +++ b/go.sum @@ -419,8 +419,8 @@ github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/sjson v1.0.4 h1:UcdIRXff12Lpnu3OLtZvnc03g4vH2suXDXhBwBqmzYg= github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= -github.com/tinylib/msgp v1.1.1 h1:TnCZ3FIuKeaIy+F45+Cnp+caqdXGy4z74HvwXN+570Y= -github.com/tinylib/msgp v1.1.1/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ= +github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=