implement a safer completeMultipart implementation (#20227)

- optimize writing part.N.meta by writing both part.N
  and its meta in sequence without network component.

- remove part.N.meta, part.N which were partially success
  ful, in quorum loss situations during renamePart()

- allow for strict read quorum check arbitrated via ETag
  for the given part number, this makes it double safer
  upon final commit.

- return an appropriate error when read quorum is missing,
  instead of returning InvalidPart{}, which is non-retryable
  error. This kind of situation can happen when many
  nodes are going offline in rotation, an example of such
  a restart() behavior is statefulset updates in k8s.

fixes #20091

cmd/xl-storage-format-v1.go

@@ -159,13 +159,14 @@ const (
 // ObjectPartInfo Info of each part kept in the multipart metadata
 // file after CompleteMultipartUpload() is called.
 type ObjectPartInfo struct {
-	ETag       string            `json:"etag,omitempty"`
-	Number     int               `json:"number"`
-	Size       int64             `json:"size"`       // Size of the part on the disk.
-	ActualSize int64             `json:"actualSize"` // Original size of the part without compression or encryption bytes.
-	ModTime    time.Time         `json:"modTime"`    // Date and time at which the part was uploaded.
-	Index      []byte            `json:"index,omitempty" msg:"index,omitempty"`
-	Checksums  map[string]string `json:"crc,omitempty" msg:"crc,omitempty"` // Content Checksums
+	ETag       string            `json:"etag,omitempty" msg:"e"`
+	Number     int               `json:"number" msg:"n"`
+	Size       int64             `json:"size" msg:"s"`        // Size of the part on the disk.
+	ActualSize int64             `json:"actualSize" msg:"as"` // Original size of the part without compression or encryption bytes.
+	ModTime    time.Time         `json:"modTime" msg:"mt"`    // Date and time at which the part was uploaded.
+	Index      []byte            `json:"index,omitempty" msg:"i,omitempty"`
+	Checksums  map[string]string `json:"crc,omitempty" msg:"crc,omitempty"`   // Content Checksums
+	Error      string            `json:"error,omitempty" msg:"err,omitempty"` // only set while reading part meta from drive.
 }
 
 // ChecksumInfo - carries checksums of individual scattered parts per disk.