diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go index 3b9182766..928b707bf 100644 --- a/cmd/erasure-multipart.go +++ b/cmd/erasure-multipart.go @@ -103,10 +103,9 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object return fi, partsMetadata, err } -// Removes part.meta given by partName belonging to a multipart upload from minioMetaBucket -func (er erasureObjects) removePartMeta(bucket, object, uploadID, dataDir string, partNumber int) { - uploadIDPath := er.getUploadIDDir(bucket, object, uploadID) - curpartPath := pathJoin(uploadIDPath, dataDir, fmt.Sprintf("part.%d", partNumber)) +// cleanMultipartPath removes all extraneous files and parts from the multipart folder, this is used per CompleteMultipart. +// do not use this function outside of completeMultipartUpload() +func (er erasureObjects) cleanupMultipartPath(ctx context.Context, paths ...string) { storageDisks := er.getDisks() g := errgroup.WithNErrs(len(storageDisks)) @@ -116,42 +115,7 @@ func (er erasureObjects) removePartMeta(bucket, object, uploadID, dataDir string } index := index g.Go(func() error { - _ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath+".meta", DeleteOptions{ - Recursive: false, - Immediate: false, - }) - - return nil - }, index) - } - g.Wait() -} - -// Removes part given by partName belonging to a multipart upload from minioMetaBucket -func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir string, partNumber int) { - uploadIDPath := er.getUploadIDDir(bucket, object, uploadID) - curpartPath := pathJoin(uploadIDPath, dataDir, fmt.Sprintf("part.%d", partNumber)) - storageDisks := er.getDisks() - - g := errgroup.WithNErrs(len(storageDisks)) - for index, disk := range storageDisks { - if disk == nil { - continue - } - index := index - g.Go(func() error { - // Ignoring failure to remove parts that weren't present in CompleteMultipartUpload - // requests. xl.meta is the authoritative source of truth on which parts constitute - // the object. The presence of parts that don't belong in the object doesn't affect correctness. - _ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath, DeleteOptions{ - Recursive: false, - Immediate: false, - }) - _ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath+".meta", DeleteOptions{ - Recursive: false, - Immediate: false, - }) - + _ = storageDisks[index].DeleteBulk(ctx, minioMetaMultipartBucket, paths...) return nil }, index) } @@ -1359,10 +1323,10 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str } } + paths := make([]string, 0, len(currentFI.Parts)) // Remove parts that weren't present in CompleteMultipartUpload request. for _, curpart := range currentFI.Parts { - // Remove part.meta which is not needed anymore. - er.removePartMeta(bucket, object, uploadID, currentFI.DataDir, curpart.Number) + paths = append(paths, pathJoin(uploadIDPath, currentFI.DataDir, fmt.Sprintf("part.%d.meta", curpart.Number))) if objectPartIndex(fi.Parts, curpart.Number) == -1 { // Delete the missing part files. e.g, @@ -1371,10 +1335,12 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str // Request 3: PutObjectPart 2 // Request 4: CompleteMultipartUpload --part 2 // N.B. 1st part is not present. This part should be removed from the storage. - er.removeObjectPart(bucket, object, uploadID, currentFI.DataDir, curpart.Number) + paths = append(paths, pathJoin(uploadIDPath, currentFI.DataDir, fmt.Sprintf("part.%d", curpart.Number))) } } + er.cleanupMultipartPath(ctx, paths...) // cleanup all part.N.meta, and skipped part.N's before final rename(). + defer func() { if err == nil { er.deleteAll(context.Background(), minioMetaMultipartBucket, uploadIDPath) diff --git a/cmd/naughty-disk_test.go b/cmd/naughty-disk_test.go index 692b7b80d..91173421e 100644 --- a/cmd/naughty-disk_test.go +++ b/cmd/naughty-disk_test.go @@ -222,6 +222,13 @@ func (d *naughtyDisk) CheckParts(ctx context.Context, volume string, path string return d.disk.CheckParts(ctx, volume, path, fi) } +func (d *naughtyDisk) DeleteBulk(ctx context.Context, volume string, paths ...string) (err error) { + if err := d.calcError(); err != nil { + return err + } + return d.disk.DeleteBulk(ctx, volume, paths...) +} + func (d *naughtyDisk) Delete(ctx context.Context, volume string, path string, deleteOpts DeleteOptions) (err error) { if err := d.calcError(); err != nil { return err diff --git a/cmd/storage-datatypes.go b/cmd/storage-datatypes.go index 8a0abcabe..f1eec9d97 100644 --- a/cmd/storage-datatypes.go +++ b/cmd/storage-datatypes.go @@ -392,24 +392,24 @@ func newFileInfo(object string, dataBlocks, parityBlocks int) (fi FileInfo) { // ReadMultipleReq contains information of multiple files to read from disk. type ReadMultipleReq struct { - Bucket string // Bucket. Can be empty if multiple buckets. - Prefix string // Shared prefix of all files. Can be empty. Will be joined to filename without modification. - Files []string // Individual files to read. - MaxSize int64 // Return error if size is exceed. - MetadataOnly bool // Read as XL meta and truncate data. - AbortOn404 bool // Stop reading after first file not found. - MaxResults int // Stop after this many successful results. <= 0 means all. + Bucket string `msg:"bk"` // Bucket. Can be empty if multiple buckets. + Prefix string `msg:"pr,omitempty"` // Shared prefix of all files. Can be empty. Will be joined to filename without modification. + Files []string `msg:"fl"` // Individual files to read. + MaxSize int64 `msg:"ms"` // Return error if size is exceed. + MetadataOnly bool `msg:"mo"` // Read as XL meta and truncate data. + AbortOn404 bool `msg:"ab"` // Stop reading after first file not found. + MaxResults int `msg:"mr"` // Stop after this many successful results. <= 0 means all. } // ReadMultipleResp contains a single response from a ReadMultipleReq. type ReadMultipleResp struct { - Bucket string // Bucket as given by request. - Prefix string // Prefix as given by request. - File string // File name as given in request. - Exists bool // Returns whether the file existed on disk. - Error string // Returns any error when reading. - Data []byte // Contains all data of file. - Modtime time.Time // Modtime of file on disk. + Bucket string `msg:"bk"` // Bucket as given by request. + Prefix string `msg:"pr,omitempty"` // Prefix as given by request. + File string `msg:"fl"` // File name as given in request. + Exists bool `msg:"ex"` // Returns whether the file existed on disk. + Error string `msg:"er,omitempty"` // Returns any error when reading. + Data []byte `msg:"d"` // Contains all data of file. + Modtime time.Time `msg:"m"` // Modtime of file on disk. } // DeleteVersionHandlerParams are parameters for DeleteVersionHandler @@ -516,8 +516,8 @@ type WriteAllHandlerParams struct { // only after as a 2-phase call, allowing the older dataDir to // hang-around in-case we need some form of recovery. type RenameDataResp struct { - Sign []byte - OldDataDir string // contains '', it is designed to be passed as value to Delete(bucket, pathJoin(object, dataDir)) + Sign []byte `msg:"s"` + OldDataDir string `msg:"od"` // contains '', it is designed to be passed as value to Delete(bucket, pathJoin(object, dataDir)) } const ( @@ -534,15 +534,26 @@ const ( // CheckPartsResp is a response of the storage CheckParts and VerifyFile APIs type CheckPartsResp struct { - Results []int + Results []int `msg:"r"` } // LocalDiskIDs - GetLocalIDs response. type LocalDiskIDs struct { - IDs []string + IDs []string `msg:"i"` } // ListDirResult - ListDir()'s response. type ListDirResult struct { Entries []string `msg:"e"` } + +// DeleteBulkReq - send multiple paths in same delete request. +type DeleteBulkReq struct { + Paths []string `msg:"p"` +} + +// DeleteVersionsErrsResp - collection of delete errors +// for bulk version deletes +type DeleteVersionsErrsResp struct { + Errs []string `msg:"e"` +} diff --git a/cmd/storage-datatypes_gen.go b/cmd/storage-datatypes_gen.go index f70f1b186..a6c755652 100644 --- a/cmd/storage-datatypes_gen.go +++ b/cmd/storage-datatypes_gen.go @@ -291,7 +291,7 @@ func (z *CheckPartsResp) DecodeMsg(dc *msgp.Reader) (err error) { return } switch msgp.UnsafeString(field) { - case "Results": + case "r": var zb0002 uint32 zb0002, err = dc.ReadArrayHeader() if err != nil { @@ -324,8 +324,8 @@ func (z *CheckPartsResp) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *CheckPartsResp) EncodeMsg(en *msgp.Writer) (err error) { // map header, size 1 - // write "Results" - err = en.Append(0x81, 0xa7, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73) + // write "r" + err = en.Append(0x81, 0xa1, 0x72) if err != nil { return } @@ -348,8 +348,8 @@ func (z *CheckPartsResp) EncodeMsg(en *msgp.Writer) (err error) { func (z *CheckPartsResp) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // map header, size 1 - // string "Results" - o = append(o, 0x81, 0xa7, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73) + // string "r" + o = append(o, 0x81, 0xa1, 0x72) o = msgp.AppendArrayHeader(o, uint32(len(z.Results))) for za0001 := range z.Results { o = msgp.AppendInt(o, z.Results[za0001]) @@ -375,7 +375,7 @@ func (z *CheckPartsResp) UnmarshalMsg(bts []byte) (o []byte, err error) { return } switch msgp.UnsafeString(field) { - case "Results": + case "r": var zb0002 uint32 zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { @@ -408,7 +408,149 @@ func (z *CheckPartsResp) UnmarshalMsg(bts []byte) (o []byte, err error) { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *CheckPartsResp) Msgsize() (s int) { - s = 1 + 8 + msgp.ArrayHeaderSize + (len(z.Results) * (msgp.IntSize)) + s = 1 + 2 + msgp.ArrayHeaderSize + (len(z.Results) * (msgp.IntSize)) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *DeleteBulkReq) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "p": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Paths") + return + } + if cap(z.Paths) >= int(zb0002) { + z.Paths = (z.Paths)[:zb0002] + } else { + z.Paths = make([]string, zb0002) + } + for za0001 := range z.Paths { + z.Paths[za0001], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Paths", za0001) + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *DeleteBulkReq) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "p" + err = en.Append(0x81, 0xa1, 0x70) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Paths))) + if err != nil { + err = msgp.WrapError(err, "Paths") + return + } + for za0001 := range z.Paths { + err = en.WriteString(z.Paths[za0001]) + if err != nil { + err = msgp.WrapError(err, "Paths", za0001) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *DeleteBulkReq) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "p" + o = append(o, 0x81, 0xa1, 0x70) + o = msgp.AppendArrayHeader(o, uint32(len(z.Paths))) + for za0001 := range z.Paths { + o = msgp.AppendString(o, z.Paths[za0001]) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *DeleteBulkReq) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "p": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Paths") + return + } + if cap(z.Paths) >= int(zb0002) { + z.Paths = (z.Paths)[:zb0002] + } else { + z.Paths = make([]string, zb0002) + } + for za0001 := range z.Paths { + z.Paths[za0001], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Paths", za0001) + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *DeleteBulkReq) Msgsize() (s int) { + s = 1 + 2 + msgp.ArrayHeaderSize + for za0001 := range z.Paths { + s += msgp.StringPrefixSize + len(z.Paths[za0001]) + } return } @@ -1101,6 +1243,148 @@ func (z *DeleteVersionHandlerParams) Msgsize() (s int) { return } +// DecodeMsg implements msgp.Decodable +func (z *DeleteVersionsErrsResp) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "e": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Errs") + return + } + if cap(z.Errs) >= int(zb0002) { + z.Errs = (z.Errs)[:zb0002] + } else { + z.Errs = make([]string, zb0002) + } + for za0001 := range z.Errs { + z.Errs[za0001], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Errs", za0001) + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *DeleteVersionsErrsResp) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "e" + err = en.Append(0x81, 0xa1, 0x65) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Errs))) + if err != nil { + err = msgp.WrapError(err, "Errs") + return + } + for za0001 := range z.Errs { + err = en.WriteString(z.Errs[za0001]) + if err != nil { + err = msgp.WrapError(err, "Errs", za0001) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *DeleteVersionsErrsResp) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "e" + o = append(o, 0x81, 0xa1, 0x65) + o = msgp.AppendArrayHeader(o, uint32(len(z.Errs))) + for za0001 := range z.Errs { + o = msgp.AppendString(o, z.Errs[za0001]) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *DeleteVersionsErrsResp) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "e": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Errs") + return + } + if cap(z.Errs) >= int(zb0002) { + z.Errs = (z.Errs)[:zb0002] + } else { + z.Errs = make([]string, zb0002) + } + for za0001 := range z.Errs { + z.Errs[za0001], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Errs", za0001) + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *DeleteVersionsErrsResp) Msgsize() (s int) { + s = 1 + 2 + msgp.ArrayHeaderSize + for za0001 := range z.Errs { + s += msgp.StringPrefixSize + len(z.Errs[za0001]) + } + return +} + // DecodeMsg implements msgp.Decodable func (z *DiskInfo) DecodeMsg(dc *msgp.Reader) (err error) { var zb0001 uint32 @@ -3234,7 +3518,7 @@ func (z *LocalDiskIDs) DecodeMsg(dc *msgp.Reader) (err error) { return } switch msgp.UnsafeString(field) { - case "IDs": + case "i": var zb0002 uint32 zb0002, err = dc.ReadArrayHeader() if err != nil { @@ -3267,8 +3551,8 @@ func (z *LocalDiskIDs) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *LocalDiskIDs) EncodeMsg(en *msgp.Writer) (err error) { // map header, size 1 - // write "IDs" - err = en.Append(0x81, 0xa3, 0x49, 0x44, 0x73) + // write "i" + err = en.Append(0x81, 0xa1, 0x69) if err != nil { return } @@ -3291,8 +3575,8 @@ func (z *LocalDiskIDs) EncodeMsg(en *msgp.Writer) (err error) { func (z *LocalDiskIDs) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // map header, size 1 - // string "IDs" - o = append(o, 0x81, 0xa3, 0x49, 0x44, 0x73) + // string "i" + o = append(o, 0x81, 0xa1, 0x69) o = msgp.AppendArrayHeader(o, uint32(len(z.IDs))) for za0001 := range z.IDs { o = msgp.AppendString(o, z.IDs[za0001]) @@ -3318,7 +3602,7 @@ func (z *LocalDiskIDs) UnmarshalMsg(bts []byte) (o []byte, err error) { return } switch msgp.UnsafeString(field) { - case "IDs": + case "i": var zb0002 uint32 zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { @@ -3351,7 +3635,7 @@ func (z *LocalDiskIDs) UnmarshalMsg(bts []byte) (o []byte, err error) { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *LocalDiskIDs) Msgsize() (s int) { - s = 1 + 4 + msgp.ArrayHeaderSize + s = 1 + 2 + msgp.ArrayHeaderSize for za0001 := range z.IDs { s += msgp.StringPrefixSize + len(z.IDs[za0001]) } @@ -3944,19 +4228,19 @@ func (z *ReadMultipleReq) DecodeMsg(dc *msgp.Reader) (err error) { return } switch msgp.UnsafeString(field) { - case "Bucket": + case "bk": z.Bucket, err = dc.ReadString() if err != nil { err = msgp.WrapError(err, "Bucket") return } - case "Prefix": + case "pr": z.Prefix, err = dc.ReadString() if err != nil { err = msgp.WrapError(err, "Prefix") return } - case "Files": + case "fl": var zb0002 uint32 zb0002, err = dc.ReadArrayHeader() if err != nil { @@ -3975,25 +4259,25 @@ func (z *ReadMultipleReq) DecodeMsg(dc *msgp.Reader) (err error) { return } } - case "MaxSize": + case "ms": z.MaxSize, err = dc.ReadInt64() if err != nil { err = msgp.WrapError(err, "MaxSize") return } - case "MetadataOnly": + case "mo": z.MetadataOnly, err = dc.ReadBool() if err != nil { err = msgp.WrapError(err, "MetadataOnly") return } - case "AbortOn404": + case "ab": z.AbortOn404, err = dc.ReadBool() if err != nil { err = msgp.WrapError(err, "AbortOn404") return } - case "MaxResults": + case "mr": z.MaxResults, err = dc.ReadInt() if err != nil { err = msgp.WrapError(err, "MaxResults") @@ -4012,9 +4296,24 @@ func (z *ReadMultipleReq) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *ReadMultipleReq) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 7 - // write "Bucket" - err = en.Append(0x87, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + // check for omitted fields + zb0001Len := uint32(7) + var zb0001Mask uint8 /* 7 bits */ + _ = zb0001Mask + if z.Prefix == "" { + zb0001Len-- + zb0001Mask |= 0x2 + } + // variable map header, size zb0001Len + err = en.Append(0x80 | uint8(zb0001Len)) + if err != nil { + return + } + if zb0001Len == 0 { + return + } + // write "bk" + err = en.Append(0xa2, 0x62, 0x6b) if err != nil { return } @@ -4023,18 +4322,20 @@ func (z *ReadMultipleReq) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "Bucket") return } - // write "Prefix" - err = en.Append(0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78) - if err != nil { - return + if (zb0001Mask & 0x2) == 0 { // if not omitted + // write "pr" + err = en.Append(0xa2, 0x70, 0x72) + if err != nil { + return + } + err = en.WriteString(z.Prefix) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } } - err = en.WriteString(z.Prefix) - if err != nil { - err = msgp.WrapError(err, "Prefix") - return - } - // write "Files" - err = en.Append(0xa5, 0x46, 0x69, 0x6c, 0x65, 0x73) + // write "fl" + err = en.Append(0xa2, 0x66, 0x6c) if err != nil { return } @@ -4050,8 +4351,8 @@ func (z *ReadMultipleReq) EncodeMsg(en *msgp.Writer) (err error) { return } } - // write "MaxSize" - err = en.Append(0xa7, 0x4d, 0x61, 0x78, 0x53, 0x69, 0x7a, 0x65) + // write "ms" + err = en.Append(0xa2, 0x6d, 0x73) if err != nil { return } @@ -4060,8 +4361,8 @@ func (z *ReadMultipleReq) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "MaxSize") return } - // write "MetadataOnly" - err = en.Append(0xac, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x4f, 0x6e, 0x6c, 0x79) + // write "mo" + err = en.Append(0xa2, 0x6d, 0x6f) if err != nil { return } @@ -4070,8 +4371,8 @@ func (z *ReadMultipleReq) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "MetadataOnly") return } - // write "AbortOn404" - err = en.Append(0xaa, 0x41, 0x62, 0x6f, 0x72, 0x74, 0x4f, 0x6e, 0x34, 0x30, 0x34) + // write "ab" + err = en.Append(0xa2, 0x61, 0x62) if err != nil { return } @@ -4080,8 +4381,8 @@ func (z *ReadMultipleReq) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "AbortOn404") return } - // write "MaxResults" - err = en.Append(0xaa, 0x4d, 0x61, 0x78, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73) + // write "mr" + err = en.Append(0xa2, 0x6d, 0x72) if err != nil { return } @@ -4096,30 +4397,44 @@ func (z *ReadMultipleReq) EncodeMsg(en *msgp.Writer) (err error) { // MarshalMsg implements msgp.Marshaler func (z *ReadMultipleReq) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // map header, size 7 - // string "Bucket" - o = append(o, 0x87, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + // check for omitted fields + zb0001Len := uint32(7) + var zb0001Mask uint8 /* 7 bits */ + _ = zb0001Mask + if z.Prefix == "" { + zb0001Len-- + zb0001Mask |= 0x2 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len == 0 { + return + } + // string "bk" + o = append(o, 0xa2, 0x62, 0x6b) o = msgp.AppendString(o, z.Bucket) - // string "Prefix" - o = append(o, 0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78) - o = msgp.AppendString(o, z.Prefix) - // string "Files" - o = append(o, 0xa5, 0x46, 0x69, 0x6c, 0x65, 0x73) + if (zb0001Mask & 0x2) == 0 { // if not omitted + // string "pr" + o = append(o, 0xa2, 0x70, 0x72) + o = msgp.AppendString(o, z.Prefix) + } + // string "fl" + o = append(o, 0xa2, 0x66, 0x6c) o = msgp.AppendArrayHeader(o, uint32(len(z.Files))) for za0001 := range z.Files { o = msgp.AppendString(o, z.Files[za0001]) } - // string "MaxSize" - o = append(o, 0xa7, 0x4d, 0x61, 0x78, 0x53, 0x69, 0x7a, 0x65) + // string "ms" + o = append(o, 0xa2, 0x6d, 0x73) o = msgp.AppendInt64(o, z.MaxSize) - // string "MetadataOnly" - o = append(o, 0xac, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x4f, 0x6e, 0x6c, 0x79) + // string "mo" + o = append(o, 0xa2, 0x6d, 0x6f) o = msgp.AppendBool(o, z.MetadataOnly) - // string "AbortOn404" - o = append(o, 0xaa, 0x41, 0x62, 0x6f, 0x72, 0x74, 0x4f, 0x6e, 0x34, 0x30, 0x34) + // string "ab" + o = append(o, 0xa2, 0x61, 0x62) o = msgp.AppendBool(o, z.AbortOn404) - // string "MaxResults" - o = append(o, 0xaa, 0x4d, 0x61, 0x78, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73) + // string "mr" + o = append(o, 0xa2, 0x6d, 0x72) o = msgp.AppendInt(o, z.MaxResults) return } @@ -4142,19 +4457,19 @@ func (z *ReadMultipleReq) UnmarshalMsg(bts []byte) (o []byte, err error) { return } switch msgp.UnsafeString(field) { - case "Bucket": + case "bk": z.Bucket, bts, err = msgp.ReadStringBytes(bts) if err != nil { err = msgp.WrapError(err, "Bucket") return } - case "Prefix": + case "pr": z.Prefix, bts, err = msgp.ReadStringBytes(bts) if err != nil { err = msgp.WrapError(err, "Prefix") return } - case "Files": + case "fl": var zb0002 uint32 zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { @@ -4173,25 +4488,25 @@ func (z *ReadMultipleReq) UnmarshalMsg(bts []byte) (o []byte, err error) { return } } - case "MaxSize": + case "ms": z.MaxSize, bts, err = msgp.ReadInt64Bytes(bts) if err != nil { err = msgp.WrapError(err, "MaxSize") return } - case "MetadataOnly": + case "mo": z.MetadataOnly, bts, err = msgp.ReadBoolBytes(bts) if err != nil { err = msgp.WrapError(err, "MetadataOnly") return } - case "AbortOn404": + case "ab": z.AbortOn404, bts, err = msgp.ReadBoolBytes(bts) if err != nil { err = msgp.WrapError(err, "AbortOn404") return } - case "MaxResults": + case "mr": z.MaxResults, bts, err = msgp.ReadIntBytes(bts) if err != nil { err = msgp.WrapError(err, "MaxResults") @@ -4211,11 +4526,11 @@ func (z *ReadMultipleReq) UnmarshalMsg(bts []byte) (o []byte, err error) { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *ReadMultipleReq) Msgsize() (s int) { - s = 1 + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 6 + msgp.ArrayHeaderSize + s = 1 + 3 + msgp.StringPrefixSize + len(z.Bucket) + 3 + msgp.StringPrefixSize + len(z.Prefix) + 3 + msgp.ArrayHeaderSize for za0001 := range z.Files { s += msgp.StringPrefixSize + len(z.Files[za0001]) } - s += 8 + msgp.Int64Size + 13 + msgp.BoolSize + 11 + msgp.BoolSize + 11 + msgp.IntSize + s += 3 + msgp.Int64Size + 3 + msgp.BoolSize + 3 + msgp.BoolSize + 3 + msgp.IntSize return } @@ -4237,43 +4552,43 @@ func (z *ReadMultipleResp) DecodeMsg(dc *msgp.Reader) (err error) { return } switch msgp.UnsafeString(field) { - case "Bucket": + case "bk": z.Bucket, err = dc.ReadString() if err != nil { err = msgp.WrapError(err, "Bucket") return } - case "Prefix": + case "pr": z.Prefix, err = dc.ReadString() if err != nil { err = msgp.WrapError(err, "Prefix") return } - case "File": + case "fl": z.File, err = dc.ReadString() if err != nil { err = msgp.WrapError(err, "File") return } - case "Exists": + case "ex": z.Exists, err = dc.ReadBool() if err != nil { err = msgp.WrapError(err, "Exists") return } - case "Error": + case "er": z.Error, err = dc.ReadString() if err != nil { err = msgp.WrapError(err, "Error") return } - case "Data": + case "d": z.Data, err = dc.ReadBytes(z.Data) if err != nil { err = msgp.WrapError(err, "Data") return } - case "Modtime": + case "m": z.Modtime, err = dc.ReadTime() if err != nil { err = msgp.WrapError(err, "Modtime") @@ -4292,9 +4607,28 @@ func (z *ReadMultipleResp) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *ReadMultipleResp) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 7 - // write "Bucket" - err = en.Append(0x87, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + // check for omitted fields + zb0001Len := uint32(7) + var zb0001Mask uint8 /* 7 bits */ + _ = zb0001Mask + if z.Prefix == "" { + zb0001Len-- + zb0001Mask |= 0x2 + } + if z.Error == "" { + zb0001Len-- + zb0001Mask |= 0x10 + } + // variable map header, size zb0001Len + err = en.Append(0x80 | uint8(zb0001Len)) + if err != nil { + return + } + if zb0001Len == 0 { + return + } + // write "bk" + err = en.Append(0xa2, 0x62, 0x6b) if err != nil { return } @@ -4303,18 +4637,20 @@ func (z *ReadMultipleResp) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "Bucket") return } - // write "Prefix" - err = en.Append(0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78) - if err != nil { - return + if (zb0001Mask & 0x2) == 0 { // if not omitted + // write "pr" + err = en.Append(0xa2, 0x70, 0x72) + if err != nil { + return + } + err = en.WriteString(z.Prefix) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } } - err = en.WriteString(z.Prefix) - if err != nil { - err = msgp.WrapError(err, "Prefix") - return - } - // write "File" - err = en.Append(0xa4, 0x46, 0x69, 0x6c, 0x65) + // write "fl" + err = en.Append(0xa2, 0x66, 0x6c) if err != nil { return } @@ -4323,8 +4659,8 @@ func (z *ReadMultipleResp) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "File") return } - // write "Exists" - err = en.Append(0xa6, 0x45, 0x78, 0x69, 0x73, 0x74, 0x73) + // write "ex" + err = en.Append(0xa2, 0x65, 0x78) if err != nil { return } @@ -4333,18 +4669,20 @@ func (z *ReadMultipleResp) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "Exists") return } - // write "Error" - err = en.Append(0xa5, 0x45, 0x72, 0x72, 0x6f, 0x72) - if err != nil { - return + if (zb0001Mask & 0x10) == 0 { // if not omitted + // write "er" + err = en.Append(0xa2, 0x65, 0x72) + if err != nil { + return + } + err = en.WriteString(z.Error) + if err != nil { + err = msgp.WrapError(err, "Error") + return + } } - err = en.WriteString(z.Error) - if err != nil { - err = msgp.WrapError(err, "Error") - return - } - // write "Data" - err = en.Append(0xa4, 0x44, 0x61, 0x74, 0x61) + // write "d" + err = en.Append(0xa1, 0x64) if err != nil { return } @@ -4353,8 +4691,8 @@ func (z *ReadMultipleResp) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "Data") return } - // write "Modtime" - err = en.Append(0xa7, 0x4d, 0x6f, 0x64, 0x74, 0x69, 0x6d, 0x65) + // write "m" + err = en.Append(0xa1, 0x6d) if err != nil { return } @@ -4369,27 +4707,47 @@ func (z *ReadMultipleResp) EncodeMsg(en *msgp.Writer) (err error) { // MarshalMsg implements msgp.Marshaler func (z *ReadMultipleResp) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // map header, size 7 - // string "Bucket" - o = append(o, 0x87, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + // check for omitted fields + zb0001Len := uint32(7) + var zb0001Mask uint8 /* 7 bits */ + _ = zb0001Mask + if z.Prefix == "" { + zb0001Len-- + zb0001Mask |= 0x2 + } + if z.Error == "" { + zb0001Len-- + zb0001Mask |= 0x10 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len == 0 { + return + } + // string "bk" + o = append(o, 0xa2, 0x62, 0x6b) o = msgp.AppendString(o, z.Bucket) - // string "Prefix" - o = append(o, 0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78) - o = msgp.AppendString(o, z.Prefix) - // string "File" - o = append(o, 0xa4, 0x46, 0x69, 0x6c, 0x65) + if (zb0001Mask & 0x2) == 0 { // if not omitted + // string "pr" + o = append(o, 0xa2, 0x70, 0x72) + o = msgp.AppendString(o, z.Prefix) + } + // string "fl" + o = append(o, 0xa2, 0x66, 0x6c) o = msgp.AppendString(o, z.File) - // string "Exists" - o = append(o, 0xa6, 0x45, 0x78, 0x69, 0x73, 0x74, 0x73) + // string "ex" + o = append(o, 0xa2, 0x65, 0x78) o = msgp.AppendBool(o, z.Exists) - // string "Error" - o = append(o, 0xa5, 0x45, 0x72, 0x72, 0x6f, 0x72) - o = msgp.AppendString(o, z.Error) - // string "Data" - o = append(o, 0xa4, 0x44, 0x61, 0x74, 0x61) + if (zb0001Mask & 0x10) == 0 { // if not omitted + // string "er" + o = append(o, 0xa2, 0x65, 0x72) + o = msgp.AppendString(o, z.Error) + } + // string "d" + o = append(o, 0xa1, 0x64) o = msgp.AppendBytes(o, z.Data) - // string "Modtime" - o = append(o, 0xa7, 0x4d, 0x6f, 0x64, 0x74, 0x69, 0x6d, 0x65) + // string "m" + o = append(o, 0xa1, 0x6d) o = msgp.AppendTime(o, z.Modtime) return } @@ -4412,43 +4770,43 @@ func (z *ReadMultipleResp) UnmarshalMsg(bts []byte) (o []byte, err error) { return } switch msgp.UnsafeString(field) { - case "Bucket": + case "bk": z.Bucket, bts, err = msgp.ReadStringBytes(bts) if err != nil { err = msgp.WrapError(err, "Bucket") return } - case "Prefix": + case "pr": z.Prefix, bts, err = msgp.ReadStringBytes(bts) if err != nil { err = msgp.WrapError(err, "Prefix") return } - case "File": + case "fl": z.File, bts, err = msgp.ReadStringBytes(bts) if err != nil { err = msgp.WrapError(err, "File") return } - case "Exists": + case "ex": z.Exists, bts, err = msgp.ReadBoolBytes(bts) if err != nil { err = msgp.WrapError(err, "Exists") return } - case "Error": + case "er": z.Error, bts, err = msgp.ReadStringBytes(bts) if err != nil { err = msgp.WrapError(err, "Error") return } - case "Data": + case "d": z.Data, bts, err = msgp.ReadBytesBytes(bts, z.Data) if err != nil { err = msgp.WrapError(err, "Data") return } - case "Modtime": + case "m": z.Modtime, bts, err = msgp.ReadTimeBytes(bts) if err != nil { err = msgp.WrapError(err, "Modtime") @@ -4468,7 +4826,7 @@ func (z *ReadMultipleResp) UnmarshalMsg(bts []byte) (o []byte, err error) { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *ReadMultipleResp) Msgsize() (s int) { - s = 1 + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 5 + msgp.StringPrefixSize + len(z.File) + 7 + msgp.BoolSize + 6 + msgp.StringPrefixSize + len(z.Error) + 5 + msgp.BytesPrefixSize + len(z.Data) + 8 + msgp.TimeSize + s = 1 + 3 + msgp.StringPrefixSize + len(z.Bucket) + 3 + msgp.StringPrefixSize + len(z.Prefix) + 3 + msgp.StringPrefixSize + len(z.File) + 3 + msgp.BoolSize + 3 + msgp.StringPrefixSize + len(z.Error) + 2 + msgp.BytesPrefixSize + len(z.Data) + 2 + msgp.TimeSize return } @@ -4946,13 +5304,13 @@ func (z *RenameDataResp) DecodeMsg(dc *msgp.Reader) (err error) { return } switch msgp.UnsafeString(field) { - case "Sign": + case "s": z.Sign, err = dc.ReadBytes(z.Sign) if err != nil { err = msgp.WrapError(err, "Sign") return } - case "OldDataDir": + case "od": z.OldDataDir, err = dc.ReadString() if err != nil { err = msgp.WrapError(err, "OldDataDir") @@ -4972,8 +5330,8 @@ func (z *RenameDataResp) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *RenameDataResp) EncodeMsg(en *msgp.Writer) (err error) { // map header, size 2 - // write "Sign" - err = en.Append(0x82, 0xa4, 0x53, 0x69, 0x67, 0x6e) + // write "s" + err = en.Append(0x82, 0xa1, 0x73) if err != nil { return } @@ -4982,8 +5340,8 @@ func (z *RenameDataResp) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "Sign") return } - // write "OldDataDir" - err = en.Append(0xaa, 0x4f, 0x6c, 0x64, 0x44, 0x61, 0x74, 0x61, 0x44, 0x69, 0x72) + // write "od" + err = en.Append(0xa2, 0x6f, 0x64) if err != nil { return } @@ -4999,11 +5357,11 @@ func (z *RenameDataResp) EncodeMsg(en *msgp.Writer) (err error) { func (z *RenameDataResp) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // map header, size 2 - // string "Sign" - o = append(o, 0x82, 0xa4, 0x53, 0x69, 0x67, 0x6e) + // string "s" + o = append(o, 0x82, 0xa1, 0x73) o = msgp.AppendBytes(o, z.Sign) - // string "OldDataDir" - o = append(o, 0xaa, 0x4f, 0x6c, 0x64, 0x44, 0x61, 0x74, 0x61, 0x44, 0x69, 0x72) + // string "od" + o = append(o, 0xa2, 0x6f, 0x64) o = msgp.AppendString(o, z.OldDataDir) return } @@ -5026,13 +5384,13 @@ func (z *RenameDataResp) UnmarshalMsg(bts []byte) (o []byte, err error) { return } switch msgp.UnsafeString(field) { - case "Sign": + case "s": z.Sign, bts, err = msgp.ReadBytesBytes(bts, z.Sign) if err != nil { err = msgp.WrapError(err, "Sign") return } - case "OldDataDir": + case "od": z.OldDataDir, bts, err = msgp.ReadStringBytes(bts) if err != nil { err = msgp.WrapError(err, "OldDataDir") @@ -5052,7 +5410,7 @@ func (z *RenameDataResp) UnmarshalMsg(bts []byte) (o []byte, err error) { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *RenameDataResp) Msgsize() (s int) { - s = 1 + 5 + msgp.BytesPrefixSize + len(z.Sign) + 11 + msgp.StringPrefixSize + len(z.OldDataDir) + s = 1 + 2 + msgp.BytesPrefixSize + len(z.Sign) + 3 + msgp.StringPrefixSize + len(z.OldDataDir) return } diff --git a/cmd/storage-datatypes_gen_test.go b/cmd/storage-datatypes_gen_test.go index fe7d592a7..09c795e48 100644 --- a/cmd/storage-datatypes_gen_test.go +++ b/cmd/storage-datatypes_gen_test.go @@ -348,6 +348,119 @@ func BenchmarkDecodeCheckPartsResp(b *testing.B) { } } +func TestMarshalUnmarshalDeleteBulkReq(t *testing.T) { + v := DeleteBulkReq{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgDeleteBulkReq(b *testing.B) { + v := DeleteBulkReq{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgDeleteBulkReq(b *testing.B) { + v := DeleteBulkReq{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalDeleteBulkReq(b *testing.B) { + v := DeleteBulkReq{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeDeleteBulkReq(t *testing.T) { + v := DeleteBulkReq{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeDeleteBulkReq Msgsize() is inaccurate") + } + + vn := DeleteBulkReq{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeDeleteBulkReq(b *testing.B) { + v := DeleteBulkReq{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeDeleteBulkReq(b *testing.B) { + v := DeleteBulkReq{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + func TestMarshalUnmarshalDeleteFileHandlerParams(t *testing.T) { v := DeleteFileHandlerParams{} bts, err := v.MarshalMsg(nil) @@ -687,6 +800,119 @@ func BenchmarkDecodeDeleteVersionHandlerParams(b *testing.B) { } } +func TestMarshalUnmarshalDeleteVersionsErrsResp(t *testing.T) { + v := DeleteVersionsErrsResp{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgDeleteVersionsErrsResp(b *testing.B) { + v := DeleteVersionsErrsResp{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgDeleteVersionsErrsResp(b *testing.B) { + v := DeleteVersionsErrsResp{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalDeleteVersionsErrsResp(b *testing.B) { + v := DeleteVersionsErrsResp{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeDeleteVersionsErrsResp(t *testing.T) { + v := DeleteVersionsErrsResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeDeleteVersionsErrsResp Msgsize() is inaccurate") + } + + vn := DeleteVersionsErrsResp{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeDeleteVersionsErrsResp(b *testing.B) { + v := DeleteVersionsErrsResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeDeleteVersionsErrsResp(b *testing.B) { + v := DeleteVersionsErrsResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + func TestMarshalUnmarshalDiskInfo(t *testing.T) { v := DiskInfo{} bts, err := v.MarshalMsg(nil) diff --git a/cmd/storage-interface.go b/cmd/storage-interface.go index 04f77da7d..1a98c548b 100644 --- a/cmd/storage-interface.go +++ b/cmd/storage-interface.go @@ -81,6 +81,7 @@ type StorageAPI interface { // Metadata operations DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool, opts DeleteOptions) error DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions, opts DeleteOptions) []error + DeleteBulk(ctx context.Context, volume string, paths ...string) error WriteMetadata(ctx context.Context, origvolume, volume, path string, fi FileInfo) error UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo, opts UpdateMetadataOpts) error ReadVersion(ctx context.Context, origvolume, volume, path, versionID string, opts ReadOptions) (FileInfo, error) diff --git a/cmd/storage-rest-client.go b/cmd/storage-rest-client.go index 468ab058d..3f7e63e44 100644 --- a/cmd/storage-rest-client.go +++ b/cmd/storage-rest-client.go @@ -737,7 +737,9 @@ func (client *storageRESTClient) DeleteVersions(ctx context.Context, volume stri } dErrResp := &DeleteVersionsErrsResp{} - if err = gob.NewDecoder(reader).Decode(dErrResp); err != nil { + decoder := msgpNewReader(reader) + defer readMsgpReaderPoolPut(decoder) + if err = dErrResp.DecodeMsg(decoder); err != nil { for i := range errs { errs[i] = toStorageErr(err) } @@ -745,7 +747,11 @@ func (client *storageRESTClient) DeleteVersions(ctx context.Context, volume stri } for i, dErr := range dErrResp.Errs { - errs[i] = toStorageErr(dErr) + if dErr != "" { + errs[i] = toStorageErr(errors.New(dErr)) + } else { + errs[i] = nil + } } return errs @@ -795,6 +801,26 @@ func (client *storageRESTClient) VerifyFile(ctx context.Context, volume, path st return verifyResp, nil } +func (client *storageRESTClient) DeleteBulk(ctx context.Context, volume string, paths ...string) (err error) { + values := make(url.Values) + values.Set(storageRESTVolume, volume) + + req := &DeleteBulkReq{Paths: paths} + body, err := req.MarshalMsg(nil) + if err != nil { + return err + } + + respBody, err := client.call(ctx, storageRESTMethodDeleteBulk, values, bytes.NewReader(body), int64(len(body))) + if err != nil { + return err + } + defer xhttp.DrainBody(respBody) + + _, err = waitForHTTPResponse(respBody) + return toStorageErr(err) +} + func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) { values := make(url.Values) values.Set(storageRESTVolume, volume) diff --git a/cmd/storage-rest-common.go b/cmd/storage-rest-common.go index 2435d091b..25401ce94 100644 --- a/cmd/storage-rest-common.go +++ b/cmd/storage-rest-common.go @@ -20,7 +20,7 @@ package cmd //go:generate msgp -file $GOFILE -unexported const ( - storageRESTVersion = "v61" // Move all Read* calls to http.MethodGet, compact handlers and query params fields + storageRESTVersion = "v62" // Introduce DeleteBulk internode API. storageRESTVersionPrefix = SlashSeparator + storageRESTVersion storageRESTPrefix = minioReservedBucketPath + "/storage" ) @@ -43,6 +43,7 @@ const ( storageRESTMethodStatInfoFile = "/sfile" storageRESTMethodReadMultiple = "/rmpl" storageRESTMethodCleanAbandoned = "/cln" + storageRESTMethodDeleteBulk = "/dblk" ) const ( diff --git a/cmd/storage-rest-server.go b/cmd/storage-rest-server.go index 6847a4a19..a4727d2ee 100644 --- a/cmd/storage-rest-server.go +++ b/cmd/storage-rest-server.go @@ -21,7 +21,6 @@ import ( "bufio" "context" "encoding/binary" - "encoding/gob" "encoding/hex" "errors" "fmt" @@ -629,12 +628,6 @@ func (s *storageRESTServer) DeleteFileHandler(p *DeleteFileHandlerParams) (grid. return grid.NewNPErr(s.getStorage().Delete(context.Background(), p.Volume, p.FilePath, p.Opts)) } -// DeleteVersionsErrsResp - collection of delete errors -// for bulk version deletes -type DeleteVersionsErrsResp struct { - Errs []error -} - // DeleteVersionsHandler - delete a set of a versions. func (s *storageRESTServer) DeleteVersionsHandler(w http.ResponseWriter, r *http.Request) { if !s.IsValid(w, r) { @@ -659,21 +652,20 @@ func (s *storageRESTServer) DeleteVersionsHandler(w http.ResponseWriter, r *http } } - dErrsResp := &DeleteVersionsErrsResp{Errs: make([]error, totalVersions)} - - setEventStreamHeaders(w) - encoder := gob.NewEncoder(w) done := keepHTTPResponseAlive(w) - opts := DeleteOptions{} errs := s.getStorage().DeleteVersions(r.Context(), volume, versions, opts) done(nil) + + dErrsResp := &DeleteVersionsErrsResp{Errs: make([]string, totalVersions)} for idx := range versions { if errs[idx] != nil { - dErrsResp.Errs[idx] = StorageErr(errs[idx].Error()) + dErrsResp.Errs[idx] = errs[idx].Error() } } - encoder.Encode(dErrsResp) + + buf, _ := dErrsResp.MarshalMsg(nil) + w.Write(buf) } // RenameDataHandler - renames a meta object and data dir to destination. @@ -1107,18 +1099,15 @@ func (s *storageRESTServer) VerifyFileHandler(w http.ResponseWriter, r *http.Req return } - setEventStreamHeaders(w) - encoder := gob.NewEncoder(w) done := keepHTTPResponseAlive(w) resp, err := s.getStorage().VerifyFile(r.Context(), volume, filePath, fi) - done(nil) - + done(err) if err != nil { - s.writeErrorResponse(w, err) return } - encoder.Encode(resp) + buf, _ := resp.MarshalMsg(nil) + w.Write(buf) } func checkDiskFatalErrs(errs []error) error { @@ -1243,6 +1232,24 @@ func (s *storageRESTServer) StatInfoFile(w http.ResponseWriter, r *http.Request) } } +func (s *storageRESTServer) DeleteBulkHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + return + } + + var req DeleteBulkReq + mr := msgpNewReader(r.Body) + defer readMsgpReaderPoolPut(mr) + + if err := req.DecodeMsg(mr); err != nil { + s.writeErrorResponse(w, err) + return + } + + volume := r.Form.Get(storageRESTVolume) + keepHTTPResponseAlive(w)(s.getStorage().DeleteBulk(r.Context(), volume, req.Paths...)) +} + // ReadMultiple returns multiple files func (s *storageRESTServer) ReadMultiple(w http.ResponseWriter, r *http.Request) { if !s.IsValid(w, r) { @@ -1325,6 +1332,7 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatInfoFile).HandlerFunc(h(server.StatInfoFile)) subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadMultiple).HandlerFunc(h(server.ReadMultiple)) subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCleanAbandoned).HandlerFunc(h(server.CleanAbandonedDataHandler)) + subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteBulk).HandlerFunc(h(server.DeleteBulkHandler)) subrouter.Methods(http.MethodGet).Path(storageRESTVersionPrefix + storageRESTMethodReadFileStream).HandlerFunc(h(server.ReadFileStreamHandler)) subrouter.Methods(http.MethodGet).Path(storageRESTVersionPrefix + storageRESTMethodReadVersion).HandlerFunc(h(server.ReadVersionHandler)) diff --git a/cmd/storagemetric_string.go b/cmd/storagemetric_string.go index 8cb65838d..cb0d02032 100644 --- a/cmd/storagemetric_string.go +++ b/cmd/storagemetric_string.go @@ -36,12 +36,13 @@ func _() { _ = x[storageMetricReadMultiple-25] _ = x[storageMetricDeleteAbandonedParts-26] _ = x[storageMetricDiskInfo-27] - _ = x[storageMetricLast-28] + _ = x[storageMetricDeleteBulk-28] + _ = x[storageMetricLast-29] } -const _storageMetric_name = "MakeVolBulkMakeVolListVolsStatVolDeleteVolWalkDirListDirReadFileAppendFileCreateFileReadFileStreamRenameFileRenameDataCheckPartsDeleteDeleteVersionsVerifyFileWriteAllDeleteVersionWriteMetadataUpdateMetadataReadVersionReadXLReadAllStatInfoFileReadMultipleDeleteAbandonedPartsDiskInfoLast" +const _storageMetric_name = "MakeVolBulkMakeVolListVolsStatVolDeleteVolWalkDirListDirReadFileAppendFileCreateFileReadFileStreamRenameFileRenameDataCheckPartsDeleteDeleteVersionsVerifyFileWriteAllDeleteVersionWriteMetadataUpdateMetadataReadVersionReadXLReadAllStatInfoFileReadMultipleDeleteAbandonedPartsDiskInfoDeleteBulkLast" -var _storageMetric_index = [...]uint16{0, 11, 18, 26, 33, 42, 49, 56, 64, 74, 84, 98, 108, 118, 128, 134, 148, 158, 166, 179, 192, 206, 217, 223, 230, 242, 254, 274, 282, 286} +var _storageMetric_index = [...]uint16{0, 11, 18, 26, 33, 42, 49, 56, 64, 74, 84, 98, 108, 118, 128, 134, 148, 158, 166, 179, 192, 206, 217, 223, 230, 242, 254, 274, 282, 292, 296} func (i storageMetric) String() string { if i >= storageMetric(len(_storageMetric_index)-1) { diff --git a/cmd/xl-storage-disk-id-check.go b/cmd/xl-storage-disk-id-check.go index 97b896a7a..19c624fb0 100644 --- a/cmd/xl-storage-disk-id-check.go +++ b/cmd/xl-storage-disk-id-check.go @@ -70,6 +70,7 @@ const ( storageMetricReadMultiple storageMetricDeleteAbandonedParts storageMetricDiskInfo + storageMetricDeleteBulk // .... add more @@ -499,6 +500,16 @@ func (p *xlStorageDiskIDCheck) CheckParts(ctx context.Context, volume string, pa }) } +func (p *xlStorageDiskIDCheck) DeleteBulk(ctx context.Context, volume string, paths ...string) (err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricDeleteBulk, append([]string{volume}, paths...)...) + if err != nil { + return err + } + defer done(0, &err) + + return p.storage.DeleteBulk(ctx, volume, paths...) +} + func (p *xlStorageDiskIDCheck) Delete(ctx context.Context, volume string, path string, deleteOpts DeleteOptions) (err error) { ctx, done, err := p.TrackDiskHealth(ctx, storageMetricDelete, volume, path) if err != nil { diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index 0f5867af3..ac663080f 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -1079,32 +1079,37 @@ func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis return err } + s.RLock() + legacy := s.formatLegacy + s.RUnlock() + var legacyJSON bool - buf, _, err := s.readAllData(ctx, volume, volumeDir, pathJoin(volumeDir, path, xlStorageFormatFile)) - if err != nil { - if !errors.Is(err, errFileNotFound) { - return err + buf, err := xioutil.WithDeadline[[]byte](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) ([]byte, error) { + buf, _, err := s.readAllData(ctx, volume, volumeDir, pathJoin(volumeDir, path, xlStorageFormatFile)) + if err != nil && !errors.Is(err, errFileNotFound) { + return nil, err } - s.RLock() - legacy := s.formatLegacy - s.RUnlock() - if legacy { + if errors.Is(err, errFileNotFound) && legacy { buf, _, err = s.readAllData(ctx, volume, volumeDir, pathJoin(volumeDir, path, xlStorageFormatFileV1)) if err != nil { - return err + return nil, err } legacyJSON = true } - } - if len(buf) == 0 { - if errors.Is(err, errFileNotFound) && !skipAccessChecks(volume) { - if aerr := Access(volumeDir); aerr != nil && osIsNotExist(aerr) { - return errVolumeNotFound + if len(buf) == 0 { + if errors.Is(err, errFileNotFound) && !skipAccessChecks(volume) { + if aerr := Access(volumeDir); aerr != nil && osIsNotExist(aerr) { + return nil, errVolumeNotFound + } + return nil, errFileNotFound } } - return errFileNotFound + return buf, nil + }) + if err != nil { + return err } if legacyJSON { @@ -1178,10 +1183,7 @@ func (s *xlStorage) DeleteVersions(ctx context.Context, volume string, versions errs[i] = ctx.Err() continue } - w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) - if err := w.Run(func() error { return s.deleteVersions(ctx, volume, fiv.Name, fiv.Versions...) }); err != nil { - errs[i] = err - } + errs[i] = s.deleteVersions(ctx, volume, fiv.Name, fiv.Versions...) diskHealthCheckOK(ctx, errs[i]) } @@ -1212,7 +1214,7 @@ func (s *xlStorage) diskAlmostFilled() bool { return (float64(info.Free)/float64(info.Used)) < almostFilledPercent || (float64(info.FreeInodes)/float64(info.UsedInodes)) < almostFilledPercent } -func (s *xlStorage) moveToTrash(filePath string, recursive, immediatePurge bool) (err error) { +func (s *xlStorage) moveToTrashNoDeadline(filePath string, recursive, immediatePurge bool) (err error) { pathUUID := mustGetUUID() targetPath := pathutil.Join(s.drivePath, minioMetaTmpDeletedBucket, pathUUID) @@ -1265,10 +1267,16 @@ func (s *xlStorage) moveToTrash(filePath string, recursive, immediatePurge bool) } } } - return nil } +func (s *xlStorage) moveToTrash(filePath string, recursive, immediatePurge bool) (err error) { + w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) + return w.Run(func() (err error) { + return s.moveToTrashNoDeadline(filePath, recursive, immediatePurge) + }) +} + // DeleteVersion - deletes FileInfo metadata for path at `xl.meta`. forceDelMarker // will force creating a new `xl.meta` to create a new delete marker func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool, opts DeleteOptions) (err error) { @@ -2417,7 +2425,41 @@ func (s *xlStorage) deleteFile(basePath, deletePath string, recursive, immediate return nil } -// DeleteFile - delete a file at path. +// DeleteBulk - delete many files in bulk to trash. +// this delete does not recursively delete empty +// parents, if you need empty parent delete support +// please use Delete() instead. This API is meant as +// an optimization for Multipart operations. +func (s *xlStorage) DeleteBulk(ctx context.Context, volume string, paths ...string) (err error) { + volumeDir, err := s.getVolDir(volume) + if err != nil { + return err + } + + if !skipAccessChecks(volume) { + // Stat a volume entry. + if err = Access(volumeDir); err != nil { + return convertAccessError(err, errVolumeAccessDenied) + } + } + + for _, fp := range paths { + // Following code is needed so that we retain SlashSeparator suffix if any in + // path argument. + filePath := pathJoin(volumeDir, fp) + if err = checkPathLength(filePath); err != nil { + return err + } + + if err = s.moveToTrash(filePath, false, false); err != nil { + return err + } + } + + return nil +} + +// Delete - delete a file at path. func (s *xlStorage) Delete(ctx context.Context, volume string, path string, deleteOpts DeleteOptions) (err error) { volumeDir, err := s.getVolDir(volume) if err != nil {