MyFavMirrors
/
minio
mirror of https://github.com/minio/minio.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

remove double reads delete versions (#13544) 

deleting collection of versions belonging
to same object, we can avoid re-reading
the xl.meta from the disk instead purge
all the requested versions in-memory,

the tradeoff is to allocate a map to de-dup
the versions, allow disks to be read only
once per object.

additionally reduce the data transfer between
nodes by shortening msgp data values.
This commit is contained in:
Harshavardhana 2021-11-01 10:50:07 -07:00 committed by GitHub
parent 15dcacc1fc
commit bb639d9f29
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 310 additions and 259 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
cmd/erasure-object.go
View File

@ -1088,37 +1088,64 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
writeQuorums[i] = getWriteQuorum(len(storageDisks))
}
versions := make([]FileInfo, len(objects))
versionsMap := make(map[string]FileInfoVersions, len(objects))
for i := range objects {
if objects[i].VersionID == "" {
modTime := opts.MTime
if opts.MTime.IsZero() {
modTime = UTCNow()
}
uuid := opts.VersionID
if uuid == "" {
uuid = mustGetUUID()
}
if opts.Versioned || opts.VersionSuspended {
versions[i] = FileInfo{
Name: objects[i].ObjectName,
ModTime: modTime,
Deleted: true, // delete marker
ReplicationState: objects[i].ReplicationState(),
}
versions[i].SetTierFreeVersionID(mustGetUUID())
if opts.Versioned {
versions[i].VersionID = uuid
}
continue
}
}
versions[i] = FileInfo{
// Construct the FileInfo data that needs to be preserved on the disk.
vr := FileInfo{
Name: objects[i].ObjectName,
VersionID: objects[i].VersionID,
ReplicationState: objects[i].ReplicationState(),
// save the index to set correct error at this index.
Idx: i,
}
versions[i].SetTierFreeVersionID(mustGetUUID())
vr.SetTierFreeVersionID(mustGetUUID())
// VersionID is not set means delete is not specific about
// any version, look for if the bucket is versioned or not.
if objects[i].VersionID == "" {
if opts.Versioned || opts.VersionSuspended {
// Bucket is versioned and no version was explicitly
// mentioned for deletes, create a delete marker instead.
vr.ModTime = UTCNow()
vr.Deleted = true
// Versioning suspended means that we add a `null` version
// delete marker, if not add a new version for this delete
// marker.
if opts.Versioned {
vr.VersionID = mustGetUUID()
}
}
}
// De-dup same object name to collect multiple versions for same object.
v, ok := versionsMap[objects[i].ObjectName]
if ok {
v.Versions = append(v.Versions, vr)
} else {
v = FileInfoVersions{
Name: vr.Name,
Versions: []FileInfo{vr},
}
}
if vr.Deleted {
dobjects[i] = DeletedObject{
DeleteMarker: vr.Deleted,
DeleteMarkerVersionID: vr.VersionID,
DeleteMarkerMTime: DeleteMarkerMTime{vr.ModTime},
ObjectName: vr.Name,
ReplicationState: vr.ReplicationState,
}
} else {
dobjects[i] = DeletedObject{
ObjectName: vr.Name,
VersionID: vr.VersionID,
ReplicationState: vr.ReplicationState,
}
}
versionsMap[objects[i].ObjectName] = v
}
dedupVersions := make([]FileInfoVersions, 0, len(versionsMap))
for _, version := range versionsMap {
dedupVersions = append(dedupVersions, version)
}
// Initialize list of errors.
@ -1130,17 +1157,24 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
wg.Add(1)
go func(index int, disk StorageAPI) {
defer wg.Done()
delObjErrs[index] = make([]error, len(objects))
if disk == nil {
delObjErrs[index] = make([]error, len(versions))
for i := range versions {
for i := range objects {
delObjErrs[index][i] = errDiskNotFound
}
return
}
delObjErrs[index] = disk.DeleteVersions(ctx, bucket, versions)
errs := disk.DeleteVersions(ctx, bucket, dedupVersions)
for i, err := range errs {
if err == nil {
continue
}
for _, v := range dedupVersions[i].Versions {
delObjErrs[index][v.Idx] = err
}
}
}(index, disk)
}
wg.Wait()
// Reduce errors for each object
@ -1162,28 +1196,17 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
}
if errs[objIndex] == nil {
NSUpdated(bucket, objects[objIndex].ObjectName)
}
if versions[objIndex].Deleted {
dobjects[objIndex] = DeletedObject{
DeleteMarker: versions[objIndex].Deleted,
DeleteMarkerVersionID: versions[objIndex].VersionID,
DeleteMarkerMTime: DeleteMarkerMTime{versions[objIndex].ModTime},
ObjectName: versions[objIndex].Name,
ReplicationState: versions[objIndex].ReplicationState,
}
} else {
dobjects[objIndex] = DeletedObject{
ObjectName: versions[objIndex].Name,
VersionID: versions[objIndex].VersionID,
ReplicationState: versions[objIndex].ReplicationState,
}
defer NSUpdated(bucket, objects[objIndex].ObjectName)
}
}
// Check failed deletes across multiple objects
for _, version := range versions {
for i, dobj := range dobjects {
// This object errored, no need to attempt a heal.
if errs[i] != nil {
continue
}
// Check if there is any offline disk and add it to the MRF list
for _, disk := range storageDisks {
if disk != nil && disk.IsOnline() {
@ -1193,7 +1216,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
// all other direct versionId references we should
// ensure no dangling file is left over.
er.addPartial(bucket, version.Name, version.VersionID, -1)
er.addPartial(bucket, dobj.ObjectName, dobj.VersionID, -1)
break
}
}

2
cmd/naughty-disk_test.go
View File

@ -225,7 +225,7 @@ func (d *naughtyDisk) Delete(ctx context.Context, volume string, path string, re
return d.disk.Delete(ctx, volume, path, recursive)
}
func (d *naughtyDisk) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) []error {
func (d *naughtyDisk) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions) []error {
if err := d.calcError(); err != nil {
errs := make([]error, len(versions))
for i := range errs {

65
cmd/storage-datatypes.go
View File

@ -80,20 +80,20 @@ type FilesInfoVersions struct {
}
// FileInfoVersions represent a list of versions for a given file.
//msgp:tuple FileInfoVersions
// The above means that any added/deleted fields are incompatible.
type FileInfoVersions struct {
// Name of the volume.
Volume string
Volume string `msg:"v,omitempty"`
// Name of the file.
Name string
IsEmptyDir bool
Name string `msg:"n,omitempty"`
// Represents the latest mod time of the
// latest version.
LatestModTime time.Time
LatestModTime time.Time `msg:"lm"`
Versions []FileInfo
Versions []FileInfo `msg:"vs"`
}
// findVersionIndex will return the version index where the version
@ -115,69 +115,74 @@ func (f *FileInfoVersions) findVersionIndex(v string) int {
// The above means that any added/deleted fields are incompatible.
type FileInfo struct {
// Name of the volume.
Volume string
Volume string `msg:"v,omitempty"`
// Name of the file.
Name string
Name string `msg:"n,omitempty"`
// Version of the file.
VersionID string
VersionID string `msg:"vid,omitempty"`
// Indicates if the version is the latest
IsLatest bool
IsLatest bool `msg:"is"`
// Deleted is set when this FileInfo represents
// a deleted marker for a versioned bucket.
Deleted bool
Deleted bool `msg:"del"`
// TransitionStatus is set to Pending/Complete for transitioned
// entries based on state of transition
TransitionStatus string
TransitionStatus string `msg:"ts"`
// TransitionedObjName is the object name on the remote tier corresponding
// to object (version) on the source tier.
TransitionedObjName string
TransitionedObjName string `msg:"to"`
// TransitionTier is the storage class label assigned to remote tier.
TransitionTier string
TransitionTier string `msg:"tt"`
// TransitionVersionID stores a version ID of the object associate
// with the remote tier.
TransitionVersionID string
TransitionVersionID string `msg:"tv"`
// ExpireRestored indicates that the restored object is to be expired.
ExpireRestored bool
ExpireRestored bool `msg:"exp"`
// DataDir of the file
DataDir string
DataDir string `msg:"dd"`
// Indicates if this object is still in V1 format.
XLV1 bool
XLV1 bool `msg:"v1"`
// Date and time when the file was last modified, if Deleted
// is 'true' this value represents when while was deleted.
ModTime time.Time
ModTime time.Time `msg:"mt"`
// Total file size.
Size int64
Size int64 `msg:"sz"`
// File mode bits.
Mode uint32
Mode uint32 `msg:"m"`
// File metadata
Metadata map[string]string
Metadata map[string]string `msg:"meta"`
// All the parts per object.
Parts []ObjectPartInfo
Parts []ObjectPartInfo `msg:"parts"`
// Erasure info for all objects.
Erasure ErasureInfo
Erasure ErasureInfo `msg:"ei"`
MarkDeleted bool // mark this version as deleted
ReplicationState ReplicationState // Internal replication state to be passed back in ObjectInfo
MarkDeleted bool `msg:"md"` // mark this version as deleted
ReplicationState ReplicationState `msg:"rs"` // Internal replication state to be passed back in ObjectInfo
Data []byte // optionally carries object data
Data []byte `msg:"d,allownil"` // optionally carries object data
NumVersions int
SuccessorModTime time.Time
NumVersions int `msg:"nv"`
SuccessorModTime time.Time `msg:"smt"`
Fresh bool // indicates this is a first time call to write FileInfo.
Fresh bool `msg:"fr"` // indicates this is a first time call to write FileInfo.
// Position of this version or object in a multi-object delete call,
// no other caller must set this value other than multi-object delete call.
// usage in other calls in undefined please avoid.
Idx int `msg:"i"`
}
// InlineData returns true if object contents are inlined alongside its metadata.

264
cmd/storage-datatypes_gen.go
View File

@ -550,8 +550,8 @@ func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) {
err = msgp.WrapError(err)
return
}
if zb0001 != 24 {
err = msgp.ArrayError{Wanted: 24, Got: zb0001}
if zb0001 != 25 {
err = msgp.ArrayError{Wanted: 25, Got: zb0001}
return
}
z.Volume, err = dc.ReadString()
@ -711,13 +711,18 @@ func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) {
err = msgp.WrapError(err, "Fresh")
return
}
z.Idx, err = dc.ReadInt()
if err != nil {
err = msgp.WrapError(err, "Idx")
return
}
return
}
// EncodeMsg implements msgp.Encodable
func (z *FileInfo) EncodeMsg(en *msgp.Writer) (err error) {
// array header, size 24
err = en.Append(0xdc, 0x0, 0x18)
// array header, size 25
err = en.Append(0xdc, 0x0, 0x19)
if err != nil {
return
}
@ -860,14 +865,19 @@ func (z *FileInfo) EncodeMsg(en *msgp.Writer) (err error) {
err = msgp.WrapError(err, "Fresh")
return
}
err = en.WriteInt(z.Idx)
if err != nil {
err = msgp.WrapError(err, "Idx")
return
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z *FileInfo) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// array header, size 24
o = append(o, 0xdc, 0x0, 0x18)
// array header, size 25
o = append(o, 0xdc, 0x0, 0x19)
o = msgp.AppendString(o, z.Volume)
o = msgp.AppendString(o, z.Name)
o = msgp.AppendString(o, z.VersionID)
@ -911,6 +921,7 @@ func (z *FileInfo) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.AppendInt(o, z.NumVersions)
o = msgp.AppendTime(o, z.SuccessorModTime)
o = msgp.AppendBool(o, z.Fresh)
o = msgp.AppendInt(o, z.Idx)
return
}
@ -922,8 +933,8 @@ func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
err = msgp.WrapError(err)
return
}
if zb0001 != 24 {
err = msgp.ArrayError{Wanted: 24, Got: zb0001}
if zb0001 != 25 {
err = msgp.ArrayError{Wanted: 25, Got: zb0001}
return
}
z.Volume, bts, err = msgp.ReadStringBytes(bts)
@ -1083,6 +1094,11 @@ func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
err = msgp.WrapError(err, "Fresh")
return
}
z.Idx, bts, err = msgp.ReadIntBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Idx")
return
}
o = bts
return
}
@ -1100,87 +1116,62 @@ func (z *FileInfo) Msgsize() (s int) {
for za0003 := range z.Parts {
s += z.Parts[za0003].Msgsize()
}
s += z.Erasure.Msgsize() + msgp.BoolSize + z.ReplicationState.Msgsize() + msgp.BytesPrefixSize + len(z.Data) + msgp.IntSize + msgp.TimeSize + msgp.BoolSize
s += z.Erasure.Msgsize() + msgp.BoolSize + z.ReplicationState.Msgsize() + msgp.BytesPrefixSize + len(z.Data) + msgp.IntSize + msgp.TimeSize + msgp.BoolSize + msgp.IntSize
return
}
// DecodeMsg implements msgp.Decodable
func (z *FileInfoVersions) DecodeMsg(dc *msgp.Reader) (err error) {
var field []byte
_ = field
var zb0001 uint32
zb0001, err = dc.ReadMapHeader()
zb0001, err = dc.ReadArrayHeader()
if err != nil {
err = msgp.WrapError(err)
return
}
for zb0001 > 0 {
zb0001--
field, err = dc.ReadMapKeyPtr()
if zb0001 != 4 {
err = msgp.ArrayError{Wanted: 4, Got: zb0001}
return
}
z.Volume, err = dc.ReadString()
if err != nil {
err = msgp.WrapError(err, "Volume")
return
}
z.Name, err = dc.ReadString()
if err != nil {
err = msgp.WrapError(err, "Name")
return
}
z.LatestModTime, err = dc.ReadTime()
if err != nil {
err = msgp.WrapError(err, "LatestModTime")
return
}
var zb0002 uint32
zb0002, err = dc.ReadArrayHeader()
if err != nil {
err = msgp.WrapError(err, "Versions")
return
}
if cap(z.Versions) >= int(zb0002) {
z.Versions = (z.Versions)[:zb0002]
} else {
z.Versions = make([]FileInfo, zb0002)
}
for za0001 := range z.Versions {
err = z.Versions[za0001].DecodeMsg(dc)
if err != nil {
err = msgp.WrapError(err)
err = msgp.WrapError(err, "Versions", za0001)
return
}
switch msgp.UnsafeString(field) {
case "Volume":
z.Volume, err = dc.ReadString()
if err != nil {
err = msgp.WrapError(err, "Volume")
return
}
case "Name":
z.Name, err = dc.ReadString()
if err != nil {
err = msgp.WrapError(err, "Name")
return
}
case "IsEmptyDir":
z.IsEmptyDir, err = dc.ReadBool()
if err != nil {
err = msgp.WrapError(err, "IsEmptyDir")
return
}
case "LatestModTime":
z.LatestModTime, err = dc.ReadTime()
if err != nil {
err = msgp.WrapError(err, "LatestModTime")
return
}
case "Versions":
var zb0002 uint32
zb0002, err = dc.ReadArrayHeader()
if err != nil {
err = msgp.WrapError(err, "Versions")
return
}
if cap(z.Versions) >= int(zb0002) {
z.Versions = (z.Versions)[:zb0002]
} else {
z.Versions = make([]FileInfo, zb0002)
}
for za0001 := range z.Versions {
err = z.Versions[za0001].DecodeMsg(dc)
if err != nil {
err = msgp.WrapError(err, "Versions", za0001)
return
}
}
default:
err = dc.Skip()
if err != nil {
err = msgp.WrapError(err)
return
}
}
}
return
}
// EncodeMsg implements msgp.Encodable
func (z *FileInfoVersions) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 5
// write "Volume"
err = en.Append(0x85, 0xa6, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65)
// array header, size 4
err = en.Append(0x94)
if err != nil {
return
}
@ -1189,41 +1180,16 @@ func (z *FileInfoVersions) EncodeMsg(en *msgp.Writer) (err error) {
err = msgp.WrapError(err, "Volume")
return
}
// write "Name"
err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65)
if err != nil {
return
}
err = en.WriteString(z.Name)
if err != nil {
err = msgp.WrapError(err, "Name")
return
}
// write "IsEmptyDir"
err = en.Append(0xaa, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x44, 0x69, 0x72)
if err != nil {
return
}
err = en.WriteBool(z.IsEmptyDir)
if err != nil {
err = msgp.WrapError(err, "IsEmptyDir")
return
}
// write "LatestModTime"
err = en.Append(0xad, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65)
if err != nil {
return
}
err = en.WriteTime(z.LatestModTime)
if err != nil {
err = msgp.WrapError(err, "LatestModTime")
return
}
// write "Versions"
err = en.Append(0xa8, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
if err != nil {
return
}
err = en.WriteArrayHeader(uint32(len(z.Versions)))
if err != nil {
err = msgp.WrapError(err, "Versions")
@ -1242,21 +1208,11 @@ func (z *FileInfoVersions) EncodeMsg(en *msgp.Writer) (err error) {
// MarshalMsg implements msgp.Marshaler
func (z *FileInfoVersions) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 5
// string "Volume"
o = append(o, 0x85, 0xa6, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65)
// array header, size 4
o = append(o, 0x94)
o = msgp.AppendString(o, z.Volume)
// string "Name"
o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
o = msgp.AppendString(o, z.Name)
// string "IsEmptyDir"
o = append(o, 0xaa, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x44, 0x69, 0x72)
o = msgp.AppendBool(o, z.IsEmptyDir)
// string "LatestModTime"
o = append(o, 0xad, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65)
o = msgp.AppendTime(o, z.LatestModTime)
// string "Versions"
o = append(o, 0xa8, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
o = msgp.AppendArrayHeader(o, uint32(len(z.Versions)))
for za0001 := range z.Versions {
o, err = z.Versions[za0001].MarshalMsg(o)
@ -1270,72 +1226,48 @@ func (z *FileInfoVersions) MarshalMsg(b []byte) (o []byte, err error) {
// UnmarshalMsg implements msgp.Unmarshaler
func (z *FileInfoVersions) UnmarshalMsg(bts []byte) (o []byte, err error) {
var field []byte
_ = field
var zb0001 uint32
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
for zb0001 > 0 {
zb0001--
field, bts, err = msgp.ReadMapKeyZC(bts)
if zb0001 != 4 {
err = msgp.ArrayError{Wanted: 4, Got: zb0001}
return
}
z.Volume, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Volume")
return
}
z.Name, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Name")
return
}
z.LatestModTime, bts, err = msgp.ReadTimeBytes(bts)
if err != nil {
err = msgp.WrapError(err, "LatestModTime")
return
}
var zb0002 uint32
zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Versions")
return
}
if cap(z.Versions) >= int(zb0002) {
z.Versions = (z.Versions)[:zb0002]
} else {
z.Versions = make([]FileInfo, zb0002)
}
for za0001 := range z.Versions {
bts, err = z.Versions[za0001].UnmarshalMsg(bts)
if err != nil {
err = msgp.WrapError(err)
err = msgp.WrapError(err, "Versions", za0001)
return
}
switch msgp.UnsafeString(field) {
case "Volume":
z.Volume, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Volume")
return
}
case "Name":
z.Name, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Name")
return
}
case "IsEmptyDir":
z.IsEmptyDir, bts, err = msgp.ReadBoolBytes(bts)
if err != nil {
err = msgp.WrapError(err, "IsEmptyDir")
return
}
case "LatestModTime":
z.LatestModTime, bts, err = msgp.ReadTimeBytes(bts)
if err != nil {
err = msgp.WrapError(err, "LatestModTime")
return
}
case "Versions":
var zb0002 uint32
zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Versions")
return
}
if cap(z.Versions) >= int(zb0002) {
z.Versions = (z.Versions)[:zb0002]
} else {
z.Versions = make([]FileInfo, zb0002)
}
for za0001 := range z.Versions {
bts, err = z.Versions[za0001].UnmarshalMsg(bts)
if err != nil {
err = msgp.WrapError(err, "Versions", za0001)
return
}
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
}
}
o = bts
return
@ -1343,7 +1275,7 @@ func (z *FileInfoVersions) UnmarshalMsg(bts []byte) (o []byte, err error) {
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *FileInfoVersions) Msgsize() (s int) {
s = 1 + 7 + msgp.StringPrefixSize + len(z.Volume) + 5 + msgp.StringPrefixSize + len(z.Name) + 11 + msgp.BoolSize + 14 + msgp.TimeSize + 9 + msgp.ArrayHeaderSize
s = 1 + msgp.StringPrefixSize + len(z.Volume) + msgp.StringPrefixSize + len(z.Name) + msgp.TimeSize + msgp.ArrayHeaderSize
for za0001 := range z.Versions {
s += z.Versions[za0001].Msgsize()
}

2
cmd/storage-interface.go
View File

@ -57,7 +57,7 @@ type StorageAPI interface {
// Metadata operations
DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool) error
DeleteVersions(ctx context.Context, volume string, versions []FileInfo) []error
DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions) []error
WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) error
UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo) error
ReadVersion(ctx context.Context, volume, path, versionID string, readData bool) (FileInfo, error)

2
cmd/storage-rest-client.go
View File

@ -590,7 +590,7 @@ func (client *storageRESTClient) Delete(ctx context.Context, volume string, path
}
// DeleteVersions - deletes list of specified versions if present
func (client *storageRESTClient) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) (errs []error) {
func (client *storageRESTClient) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions) (errs []error) {
if len(versions) == 0 {
return errs
}

2
cmd/storage-rest-common.go
View File

@ -18,7 +18,7 @@
package cmd
const (
storageRESTVersion = "v40" // Add ReplicationState field
storageRESTVersion = "v41" // Optimized DeleteVersions API
storageRESTVersionPrefix = SlashSeparator + storageRESTVersion
storageRESTPrefix = minioReservedBucketPath + "/storage"
)

2
cmd/storage-rest-server.go
View File

@ -643,7 +643,7 @@ func (s *storageRESTServer) DeleteVersionsHandler(w http.ResponseWriter, r *http
return
}
versions := make([]FileInfo, totalVersions)
versions := make([]FileInfoVersions, totalVersions)
decoder := msgp.NewReader(r.Body)
for i := 0; i < totalVersions; i++ {
dst := &versions[i]

4
cmd/xl-storage-disk-id-check.go
View File

@ -421,8 +421,8 @@ func (p *xlStorageDiskIDCheck) Delete(ctx context.Context, volume string, path s
// DeleteVersions deletes slice of versions, it can be same object
// or multiple objects.
func (p *xlStorageDiskIDCheck) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) (errs []error) {
// Mererly for tracing storage
func (p *xlStorageDiskIDCheck) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions) (errs []error) {
// Merely for tracing storage
path := ""
if len(versions) > 0 {
path = versions[0].Name

95
cmd/xl-storage.go
View File

@ -821,13 +821,102 @@ func (s *xlStorage) ListDir(ctx context.Context, volume, dirPath string, count i
return entries, nil
}
func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis ...FileInfo) error {
buf, err := s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile))
if err != nil {
if err != errFileNotFound {
return err
}
metaDataPoolPut(buf) // Never used, return it
buf, err = s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFileV1))
if err != nil {
return err
}
}
if len(buf) == 0 {
return errFileNotFound
}
volumeDir, err := s.getVolDir(volume)
if err != nil {
return err
}
if !isXL2V1Format(buf) {
// Delete the meta file, if there are no more versions the
// top level parent is automatically removed.
return s.deleteFile(volumeDir, pathJoin(volumeDir, path), true)
}
var xlMeta xlMetaV2
if err = xlMeta.Load(buf); err != nil {
return err
}
var (
dataDir string
lastVersion bool
)
for _, fi := range fis {
dataDir, lastVersion, err = xlMeta.DeleteVersion(fi)
if err != nil {
return err
}
if dataDir != "" {
versionID := fi.VersionID
if versionID == "" {
versionID = nullVersionID
}
// PR #11758 used DataDir, preserve it
// for users who might have used master
// branch
if !xlMeta.data.remove(versionID, dataDir) {
filePath := pathJoin(volumeDir, path, dataDir)
if err = checkPathLength(filePath); err != nil {
return err
}
if err = s.moveToTrash(filePath, true); err != nil {
if err != errFileNotFound {
return err
}
}
}
}
}
if !lastVersion {
buf, err = xlMeta.AppendTo(metaDataPoolGet())
defer metaDataPoolPut(buf)
if err != nil {
return err
}
return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), buf)
}
// Move xl.meta to trash
filePath := pathJoin(volumeDir, path, xlStorageFormatFile)
if err = checkPathLength(filePath); err != nil {
return err
}
err = s.moveToTrash(filePath, false)
if err == nil || err == errFileNotFound {
s.deleteFile(volumeDir, pathJoin(volumeDir, path), false)
}
return err
}
// DeleteVersions deletes slice of versions, it can be same object
// or multiple objects.
func (s *xlStorage) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) []error {
func (s *xlStorage) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions) []error {
errs := make([]error, len(versions))
for i, version := range versions {
if err := s.DeleteVersion(ctx, volume, version.Name, version, false); err != nil {
for i, fiv := range versions {
if err := s.deleteVersions(ctx, volume, fiv.Name, fiv.Versions...); err != nil {
errs[i] = err
}
}

4
go.mod
View File

@ -63,7 +63,7 @@ require (
github.com/nats-io/stan.go v0.8.3
github.com/ncw/directio v1.0.5
github.com/nsqio/go-nsq v1.0.8
github.com/philhofer/fwd v1.1.1
github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9
github.com/pierrec/lz4 v2.6.0+incompatible
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.11.0
@ -74,7 +74,7 @@ require (
github.com/secure-io/sio-go v0.3.1
github.com/shirou/gopsutil/v3 v3.21.9
github.com/streadway/amqp v1.0.0
github.com/tinylib/msgp v1.1.6
github.com/tinylib/msgp v1.1.7-0.20211026165309-e818a1881b0e
github.com/valyala/bytebufferpool v1.0.0
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
github.com/yargevad/filepathx v1.0.0

6
go.sum
View File

@ -1218,8 +1218,9 @@ github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bA
github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw=
github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ=
github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9 h1:6ob53CVz+ja2i7easAStApZJlh7sxyq3Cm7g1Di6iqA=
github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
@ -1430,8 +1431,9 @@ github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiff
github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
github.com/tinylib/msgp v1.1.3/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg=
github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw=
github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
github.com/tinylib/msgp v1.1.7-0.20211026165309-e818a1881b0e h1:P5tyWbssToKowBPTA1/EzqPXwrZNc8ZeNPdjgpcDEoI=
github.com/tinylib/msgp v1.1.7-0.20211026165309-e818a1881b0e/go.mod h1:g7jEyb18KPe65d9RRhGw+ThaJr5duyBH8eaFgBUor7Y=
github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=