Upgrade compression package (#15284)

Includes mitigation for CVE-2022-30631 (Go should still be updated)

Remove functions now available upstream.

cmd/object-api-utils.go

@@ -21,7 +21,6 @@ import (
 	"bytes"
 	"context"
 	"crypto/hmac"
-	"encoding/binary"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -544,7 +543,7 @@ func getCompressedOffsets(oi ObjectInfo, offset int64, decrypt func([]byte) ([]b
 			if err == nil {
 				// Load Index
 				var idx s2.Index
-				_, err := idx.Load(restoreIndexHeaders(dec))
+				_, err := idx.Load(s2.RestoreIndexHeaders(dec))
 
 				// Find compressed/uncompressed offsets of our partskip
 				compOff, uCompOff, err2 := idx.Find(partSkip)
@@ -567,7 +566,7 @@ func getCompressedOffsets(oi ObjectInfo, offset int64, decrypt func([]byte) ([]b
 		} else {
 			// Not encrypted
 			var idx s2.Index
-			_, err := idx.Load(restoreIndexHeaders(oi.Parts[firstPartIdx].Index))
+			_, err := idx.Load(s2.RestoreIndexHeaders(oi.Parts[firstPartIdx].Index))
 
 			// Find compressed/uncompressed offsets of our partskip
 			compOff, uCompOff, err2 := idx.Find(partSkip)
@@ -1010,7 +1009,7 @@ func newS2CompressReader(r io.Reader, on int64) (rc io.ReadCloser, idx func() []
 		// If more than 8MB was written, generate index.
 		if cn > 8<<20 {
 			idx, err := comp.CloseIndex()
-			idx = removeIndexHeaders(idx)
+			idx = s2.RemoveIndexHeaders(idx)
 			indexCh <- idx
 			pw.CloseWithError(err)
 			return
@@ -1128,65 +1127,3 @@ func hasSpaceFor(di []*DiskInfo, size int64) bool {
 	wantLeft := uint64(float64(total) * (1.0 - diskFillFraction))
 	return available > wantLeft
 }
-
-// removeIndexHeaders will trim all headers and trailers from a given index.
-// This is expected to save 20 bytes.
-// These can be restored using RestoreIndexHeaders.
-// This removes a layer of security, but is the most compact representation.
-// Returns nil if headers contains errors.
-// The returned slice references the provided slice.
-func removeIndexHeaders(b []byte) []byte {
-	const save = 4 + len(s2.S2IndexHeader) + len(s2.S2IndexTrailer) + 4
-	if len(b) <= save {
-		return nil
-	}
-	if b[0] != s2.ChunkTypeIndex {
-		return nil
-	}
-	chunkLen := int(b[1]) | int(b[2])<<8 | int(b[3])<<16
-	b = b[4:]
-
-	// Validate we have enough...
-	if len(b) < chunkLen {
-		return nil
-	}
-	b = b[:chunkLen]
-
-	if !bytes.Equal(b[:len(s2.S2IndexHeader)], []byte(s2.S2IndexHeader)) {
-		return nil
-	}
-	b = b[len(s2.S2IndexHeader):]
-	if !bytes.HasSuffix(b, []byte(s2.S2IndexTrailer)) {
-		return nil
-	}
-	b = bytes.TrimSuffix(b, []byte(s2.S2IndexTrailer))
-
-	if len(b) < 4 {
-		return nil
-	}
-	return b[:len(b)-4]
-}
-
-// restoreIndexHeaders will index restore headers removed by RemoveIndexHeaders.
-// No error checking is performed on the input.
-func restoreIndexHeaders(in []byte) []byte {
-	if len(in) == 0 {
-		return nil
-	}
-	b := make([]byte, 0, 4+len(s2.S2IndexHeader)+len(in)+len(s2.S2IndexTrailer)+4)
-	b = append(b, s2.ChunkTypeIndex, 0, 0, 0)
-	b = append(b, []byte(s2.S2IndexHeader)...)
-	b = append(b, in...)
-
-	var tmp [4]byte
-	binary.LittleEndian.PutUint32(tmp[:], uint32(len(b)+4+len(s2.S2IndexTrailer)))
-	b = append(b, tmp[:4]...)
-	// Trailer
-	b = append(b, []byte(s2.S2IndexTrailer)...)
-
-	chunkLen := len(b) - 4 /*skippableFrameHeader*/
-	b[1] = uint8(chunkLen >> 0)
-	b[2] = uint8(chunkLen >> 8)
-	b[3] = uint8(chunkLen >> 16)
-	return b
-}

cmd/object-api-utils_test.go

@@ -642,7 +642,7 @@ func TestS2CompressReader(t *testing.T) {
 					t.Errorf("no index returned")
 				}
 				var index s2.Index
-				_, err = index.Load(restoreIndexHeaders(idx))
+				_, err = index.Load(s2.RestoreIndexHeaders(idx))
 				if err != nil {
 					t.Errorf("error loading index: %v", err)
 				}

go.mod

@@ -35,7 +35,7 @@ require (
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/inconshreveable/mousetrap v1.0.0
 	github.com/json-iterator/go v1.1.12
-	github.com/klauspost/compress v1.15.6
+	github.com/klauspost/compress v1.15.8
 	github.com/klauspost/cpuid/v2 v2.0.14
 	github.com/klauspost/pgzip v1.2.5
 	github.com/klauspost/readahead v1.4.0

go.sum

@@ -512,8 +512,8 @@ github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8
 github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
-github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY=
+github.com/klauspost/compress v1.15.8 h1:JahtItbkWjf2jzm/T+qgMxkP9EMHsqEUA6vCMGmXvhA=
-github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/klauspost/compress v1.15.8/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
 github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=