Update reed+solomon (#9562)

Only create encoder when strictly needed.

cmd/erasure.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"sync"
 
 	"github.com/klauspost/reedsolomon"
 	"github.com/minio/minio/cmd/logger"
@@ -25,7 +26,7 @@ import (
 
 // Erasure - erasure encoding details.
 type Erasure struct {
-	encoder                  reedsolomon.Encoder
+	encoder                  func() reedsolomon.Encoder
 	dataBlocks, parityBlocks int
 	blockSize                int64
 }
@@ -37,10 +38,29 @@ func NewErasure(ctx context.Context, dataBlocks, parityBlocks int, blockSize int
 		parityBlocks: parityBlocks,
 		blockSize:    blockSize,
 	}
-	e.encoder, err = reedsolomon.New(dataBlocks, parityBlocks, reedsolomon.WithAutoGoroutines(int(e.ShardSize())))
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return e, err
+
+	// Check the parameters for sanity now.
+	if dataBlocks <= 0 || parityBlocks <= 0 {
+		return e, reedsolomon.ErrInvShardNum
+	}
+
+	if dataBlocks+parityBlocks > 256 {
+		return e, reedsolomon.ErrMaxShardNum
+	}
+
+	// Encoder when needed.
+	var enc reedsolomon.Encoder
+	var once sync.Once
+	e.encoder = func() reedsolomon.Encoder {
+		once.Do(func() {
+			e, err := reedsolomon.New(dataBlocks, parityBlocks, reedsolomon.WithAutoGoroutines(int(e.ShardSize())))
+			if err != nil {
+				// Error conditions should be checked above.
+				panic(err)
+			}
+			enc = e
+		})
+		return enc
 	}
 	return
 }
@@ -51,12 +71,12 @@ func (e *Erasure) EncodeData(ctx context.Context, data []byte) ([][]byte, error)
 	if len(data) == 0 {
 		return make([][]byte, e.dataBlocks+e.parityBlocks), nil
 	}
-	encoded, err := e.encoder.Split(data)
+	encoded, err := e.encoder().Split(data)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return nil, err
 	}
-	if err = e.encoder.Encode(encoded); err != nil {
+	if err = e.encoder().Encode(encoded); err != nil {
 		logger.LogIf(ctx, err)
 		return nil, err
 	}
@@ -77,13 +97,23 @@ func (e *Erasure) DecodeDataBlocks(data [][]byte) error {
 	if !needsReconstruction {
 		return nil
 	}
-	return e.encoder.ReconstructData(data)
+	return e.encoder().ReconstructData(data)
 }
 
 // DecodeDataAndParityBlocks decodes the given erasure-coded data and verifies it.
 // It returns an error if the decoding failed.
 func (e *Erasure) DecodeDataAndParityBlocks(ctx context.Context, data [][]byte) error {
-	if err := e.encoder.Reconstruct(data); err != nil {
+	needsReconstruction := false
+	for _, b := range data {
+		if b == nil {
+			needsReconstruction = true
+			break
+		}
+	}
+	if !needsReconstruction {
+		return nil
+	}
+	if err := e.encoder().Reconstruct(data); err != nil {
 		logger.LogIf(ctx, err)
 		return err
 	}

cmd/xl-v1-healing-common.go

@@ -171,20 +171,15 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 
 		switch scanMode {
 		case madmin.HealDeepScan:
-			erasureInfo := partsMetadata[i].Erasure
-			erasure, err := NewErasure(ctx, erasureInfo.DataBlocks, erasureInfo.ParityBlocks, erasureInfo.BlockSize)
-			if err != nil {
-				dataErrs[i] = err
-				continue
-			}
+			erasure := partsMetadata[i].Erasure
 
 			// disk has a valid xl.json but may not have all the
 			// parts. This is considered an outdated disk, since
 			// it needs healing too.
 			for _, part := range partsMetadata[i].Parts {
-				checksumInfo := erasureInfo.GetChecksumInfo(part.Number)
+				checksumInfo := erasure.GetChecksumInfo(part.Number)
 				partPath := pathJoin(object, fmt.Sprintf("part.%d", part.Number))
-				err = onlineDisk.VerifyFile(bucket, partPath, erasure.ShardFileSize(part.Size), checksumInfo.Algorithm, checksumInfo.Hash, erasure.ShardSize())
+				err := onlineDisk.VerifyFile(bucket, partPath, erasure.ShardFileSize(part.Size), checksumInfo.Algorithm, checksumInfo.Hash, erasure.ShardSize())
 				if err != nil {
 					if !IsErr(err, []error{
 						errFileNotFound,

cmd/xl-v1-metadata.go

@@ -139,6 +139,25 @@ func (e ErasureInfo) GetChecksumInfo(partNumber int) (ckSum ChecksumInfo) {
 	return ChecksumInfo{}
 }
 
+// ShardFileSize - returns final erasure size from original size.
+func (e ErasureInfo) ShardFileSize(totalLength int64) int64 {
+	if totalLength == 0 {
+		return 0
+	}
+	if totalLength == -1 {
+		return -1
+	}
+	numShards := totalLength / e.BlockSize
+	lastBlockSize := totalLength % e.BlockSize
+	lastShardSize := ceilFrac(lastBlockSize, int64(e.DataBlocks))
+	return numShards*e.ShardSize() + lastShardSize
+}
+
+// ShardSize - returns actual shared size from erasure blockSize.
+func (e ErasureInfo) ShardSize() int64 {
+	return ceilFrac(e.BlockSize, int64(e.DataBlocks))
+}
+
 // statInfo - carries stat information of the object.
 type statInfo struct {
 	Size    int64     `json:"size"`    // Size of the object `xl.json`.