admin/heal: Fix deep healing to heal objects under more conditions (#8321)

- Heal if the part.1 is truncated from its original size
- Heal if the part.1 fails while being verified in between
- Heal if the part.1 fails while being at a certain offset

Other cleanups include make sure to flush the HTTP responses
properly from storage-rest-server, avoid using 'defer' to
improve call latency. 'defer' incurs latency avoid them
in our hot-paths such as storage-rest handlers.

Fixes #8319

cmd/storage-errors.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"errors"
-	"fmt"
 )
 
 // errUnexpected - unexpected error, requires manual intervention.
@@ -72,8 +71,8 @@ var errVolumeAccessDenied = errors.New("volume access denied")
 // errFileAccessDenied - cannot access file, insufficient permissions.
 var errFileAccessDenied = errors.New("file access denied")
 
-// errFileUnexpectedSize - file has an unexpected size
-var errFileUnexpectedSize = errors.New("file has unexpected size")
+// errFileCorrupt - file has an unexpected size, or is not readable
+var errFileCorrupt = errors.New("file is corrupted")
 
 // errFileParentIsFile - cannot have overlapping objects, parent is already a file.
 var errFileParentIsFile = errors.New("parent is a file")
@@ -94,17 +93,12 @@ var errLessData = errors.New("less data available than what was requested")
 // errMoreData = returned when more data was sent by the caller than what it was supposed to.
 var errMoreData = errors.New("more data was sent than what was advertised")
 
-// HashMismatchError represents a bit-rot hash verification failure error.
-type HashMismatchError struct {
-	Expected string
-	Computed string
-}
+// VerifyFileError represents error generated by VerifyFile posix call.
+type VerifyFileError string
 
 // Error method for the hashMismatchError
-func (h HashMismatchError) Error() string {
-	return fmt.Sprintf(
-		"Bitrot verification mismatch - expected %v, received %v",
-		h.Expected, h.Computed)
+func (h VerifyFileError) Error() string {
+	return string(h)
 }
 
 // Collection of basic errors.