xl: Implement posix.DeletePrefixes to enhance delete perf (#9100)

Bulk delete API was using cleanupObjectsBulk() which calls posix
listing and delete API to remove objects internal files in the
backend (xl.json and parts) one by one.

Add DeletePrefixes in the storage API to remove the content
of a directory in a single call.

Also use a remove goroutine for each disk to accelerate removal.

cmd/storage-rest-client.go

@@ -414,13 +414,54 @@ func (client *storageRESTClient) DeleteFileBulk(volume string, paths []string) (
 
 	respBody, err := client.call(storageRESTMethodDeleteFileBulk, values, &buffer, -1)
 	defer http.DrainBody(respBody)
+	if err != nil {
+		return nil, err
+	}
 
+	reader, err := clearLeadingSpaces(respBody)
 	if err != nil {
 		return nil, err
 	}
 
 	dErrResp := &DeleteFileBulkErrsResp{}
-	if err = gob.NewDecoder(respBody).Decode(dErrResp); err != nil {
+	if err = gob.NewDecoder(reader).Decode(dErrResp); err != nil {
+		return nil, err
+	}
+
+	for _, dErr := range dErrResp.Errs {
+		errs = append(errs, toStorageErr(dErr))
+	}
+
+	return errs, nil
+}
+
+// DeletePrefixes - deletes prefixes in bulk.
+func (client *storageRESTClient) DeletePrefixes(volume string, paths []string) (errs []error, err error) {
+	if len(paths) == 0 {
+		return errs, err
+	}
+	values := make(url.Values)
+	values.Set(storageRESTVolume, volume)
+
+	var buffer bytes.Buffer
+	for _, path := range paths {
+		buffer.WriteString(path)
+		buffer.WriteString("\n")
+	}
+
+	respBody, err := client.call(storageRESTMethodDeletePrefixes, values, &buffer, -1)
+	defer http.DrainBody(respBody)
+	if err != nil {
+		return nil, err
+	}
+
+	reader, err := clearLeadingSpaces(respBody)
+	if err != nil {
+		return nil, err
+	}
+
+	dErrResp := &DeletePrefixesErrsResp{}
+	if err = gob.NewDecoder(reader).Decode(dErrResp); err != nil {
 		return nil, err
 	}
 
@@ -443,6 +484,22 @@ func (client *storageRESTClient) RenameFile(srcVolume, srcPath, dstVolume, dstPa
 	return err
 }
 
+// clearLeadingSpaces removes all the first spaces returned from a reader.
+func clearLeadingSpaces(r io.Reader) (io.Reader, error) {
+	reader := bufio.NewReader(r)
+	for {
+		b, err := reader.ReadByte()
+		if err != nil {
+			return nil, err
+		}
+		if b != ' ' {
+			reader.UnreadByte()
+			break
+		}
+	}
+	return reader, nil
+}
+
 func (client *storageRESTClient) VerifyFile(volume, path string, size int64, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
 	values := make(url.Values)
 	values.Set(storageRESTVolume, volume)
@@ -457,16 +514,9 @@ func (client *storageRESTClient) VerifyFile(volume, path string, size int64, alg
 	if err != nil {
 		return err
 	}
-	reader := bufio.NewReader(respBody)
-	for {
-		b, err := reader.ReadByte()
-		if err != nil {
-			return err
-		}
-		if b != ' ' {
-			reader.UnreadByte()
-			break
-		}
+	reader, err := clearLeadingSpaces(respBody)
+	if err != nil {
+		return err
 	}
 	verifyResp := &VerifyFileResp{}
 	if err = gob.NewDecoder(reader).Decode(verifyResp); err != nil {