mirror of
https://github.com/minio/minio.git
synced 2025-11-09 21:49:46 -05:00
Compress better on amd64 (#12974)
Since S2 has amd64 assembly, it now operates at a reasonable speed to use by default. Here are some examples of stream compression speed, 16 cores: ``` nyc-taxi-data-10M.csv s2 1 3325605752 -> 1095998837 312ms 10139.07MB/s 67.04% reduction nyc-taxi-data-10M.csv s2 2 3325605752 -> 917905514 428ms 7393.74MB/s 72.40% github-june-2days-2019.json s2 1 6273951764 -> 1043196283 391ms 15301.99 MB/s 83.37% github-june-2days-2019.json s2 2 6273951764 -> 955924506 519ms 11510.81MB/s 84.76% github-ranks-backup.bin s2 1 1862623243 -> 623911363 146ms 12133MB/s 66.50% github-ranks-backup.bin s2 2 1862623243 -> 563752759 230ms 7705.26MB/s 69.73% ``` We keep non-assembly platforms on the faster, but less efficient mode.
This commit is contained in:
@@ -884,6 +884,16 @@ func CleanMinioInternalMetadataKeys(metadata map[string]string) map[string]strin
|
||||
return newMeta
|
||||
}
|
||||
|
||||
// compressOpts are the options for writing compressed data.
|
||||
var compressOpts []s2.WriterOption
|
||||
|
||||
func init() {
|
||||
if runtime.GOARCH == "amd64" {
|
||||
// On amd64 we have assembly and can use stronger compression.
|
||||
compressOpts = append(compressOpts, s2.WriterBetterCompression())
|
||||
}
|
||||
}
|
||||
|
||||
// newS2CompressReader will read data from r, compress it and return the compressed data as a Reader.
|
||||
// Use Close to ensure resources are released on incomplete streams.
|
||||
//
|
||||
@@ -894,7 +904,7 @@ func newS2CompressReader(r io.Reader, on int64) io.ReadCloser {
|
||||
pr, pw := io.Pipe()
|
||||
// Copy input to compressor
|
||||
go func() {
|
||||
comp := s2.NewWriter(pw)
|
||||
comp := s2.NewWriter(pw, compressOpts...)
|
||||
cn, err := io.Copy(comp, r)
|
||||
if err != nil {
|
||||
comp.Close()
|
||||
|
||||
Reference in New Issue
Block a user