feat: bring new HDD related performance enhancements (#18239)

Optionally allows customers to enable 

- Enable an external cache to catch GET/HEAD responses 
- Enable skipping disks that are slow to respond in GET/HEAD 
  when we have already achieved a quorum

cmd/erasure-metadata-utils.go

@@ -148,6 +148,26 @@ func hashOrder(key string, cardinality int) []int {
 	return nums
 }
 
+var readFileInfoIgnoredErrs = append(objectOpIgnoredErrs,
+	errFileNotFound,
+	errVolumeNotFound,
+	errFileVersionNotFound,
+	io.ErrUnexpectedEOF, // some times we would read without locks, ignore these errors
+	io.EOF,              // some times we would read without locks, ignore these errors
+)
+
+func readFileInfo(ctx context.Context, disk StorageAPI, bucket, object, versionID string, opts ReadOptions) (FileInfo, error) {
+	fi, err := disk.ReadVersion(ctx, bucket, object, versionID, opts)
+
+	if err != nil && !IsErr(err, readFileInfoIgnoredErrs...) {
+		logger.LogOnceIf(ctx, fmt.Errorf("Drive %s, path (%s/%s) returned an error (%w)",
+			disk.String(), bucket, object, err),
+			disk.String())
+	}
+
+	return fi, err
+}
+
 // Reads all `xl.meta` metadata as a FileInfo slice.
 // Returns error slice indicating the failed metadata reads.
 func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, versionID string, readData, healing bool) ([]FileInfo, []error) {
@@ -166,33 +186,12 @@ func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, ve
 			if disks[index] == nil {
 				return errDiskNotFound
 			}
-			metadataArray[index], err = disks[index].ReadVersion(ctx, bucket, object, versionID, opts)
+			metadataArray[index], err = readFileInfo(ctx, disks[index], bucket, object, versionID, opts)
 			return err
 		}, index)
 	}
 
-	ignoredErrs := []error{
-		errFileNotFound,
-		errVolumeNotFound,
-		errFileVersionNotFound,
-		io.ErrUnexpectedEOF, // some times we would read without locks, ignore these errors
-		io.EOF,              // some times we would read without locks, ignore these errors
-	}
-	ignoredErrs = append(ignoredErrs, objectOpIgnoredErrs...)
-	errs := g.Wait()
-	for index, err := range errs {
-		if err == nil {
-			continue
-		}
-		if !IsErr(err, ignoredErrs...) {
-			logger.LogOnceIf(ctx, fmt.Errorf("Drive %s, path (%s/%s) returned an error (%w)",
-				disks[index], bucket, object, err),
-				disks[index].String())
-		}
-	}
-
-	// Return all the metadata.
-	return metadataArray, errs
+	return metadataArray, g.Wait()
 }
 
 // shuffleDisksAndPartsMetadataByIndex this function should be always used by GetObjectNInfo()