From 697c9973a7e4a9f506c6b5297e2624047e4f538d Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Wed, 13 Jul 2022 07:48:14 -0700 Subject: [PATCH] Upgrade compression package (#15284) Includes mitigation for CVE-2022-30631 (Go should still be updated) Remove functions now available upstream. --- cmd/object-api-utils.go | 69 ++---------------------------------- cmd/object-api-utils_test.go | 2 +- go.mod | 2 +- go.sum | 4 +-- 4 files changed, 7 insertions(+), 70 deletions(-) diff --git a/cmd/object-api-utils.go b/cmd/object-api-utils.go index 804a52b0e..fabdda2f3 100644 --- a/cmd/object-api-utils.go +++ b/cmd/object-api-utils.go @@ -21,7 +21,6 @@ import ( "bytes" "context" "crypto/hmac" - "encoding/binary" "encoding/hex" "errors" "fmt" @@ -544,7 +543,7 @@ func getCompressedOffsets(oi ObjectInfo, offset int64, decrypt func([]byte) ([]b if err == nil { // Load Index var idx s2.Index - _, err := idx.Load(restoreIndexHeaders(dec)) + _, err := idx.Load(s2.RestoreIndexHeaders(dec)) // Find compressed/uncompressed offsets of our partskip compOff, uCompOff, err2 := idx.Find(partSkip) @@ -567,7 +566,7 @@ func getCompressedOffsets(oi ObjectInfo, offset int64, decrypt func([]byte) ([]b } else { // Not encrypted var idx s2.Index - _, err := idx.Load(restoreIndexHeaders(oi.Parts[firstPartIdx].Index)) + _, err := idx.Load(s2.RestoreIndexHeaders(oi.Parts[firstPartIdx].Index)) // Find compressed/uncompressed offsets of our partskip compOff, uCompOff, err2 := idx.Find(partSkip) @@ -1010,7 +1009,7 @@ func newS2CompressReader(r io.Reader, on int64) (rc io.ReadCloser, idx func() [] // If more than 8MB was written, generate index. if cn > 8<<20 { idx, err := comp.CloseIndex() - idx = removeIndexHeaders(idx) + idx = s2.RemoveIndexHeaders(idx) indexCh <- idx pw.CloseWithError(err) return @@ -1128,65 +1127,3 @@ func hasSpaceFor(di []*DiskInfo, size int64) bool { wantLeft := uint64(float64(total) * (1.0 - diskFillFraction)) return available > wantLeft } - -// removeIndexHeaders will trim all headers and trailers from a given index. -// This is expected to save 20 bytes. -// These can be restored using RestoreIndexHeaders. -// This removes a layer of security, but is the most compact representation. -// Returns nil if headers contains errors. -// The returned slice references the provided slice. -func removeIndexHeaders(b []byte) []byte { - const save = 4 + len(s2.S2IndexHeader) + len(s2.S2IndexTrailer) + 4 - if len(b) <= save { - return nil - } - if b[0] != s2.ChunkTypeIndex { - return nil - } - chunkLen := int(b[1]) | int(b[2])<<8 | int(b[3])<<16 - b = b[4:] - - // Validate we have enough... - if len(b) < chunkLen { - return nil - } - b = b[:chunkLen] - - if !bytes.Equal(b[:len(s2.S2IndexHeader)], []byte(s2.S2IndexHeader)) { - return nil - } - b = b[len(s2.S2IndexHeader):] - if !bytes.HasSuffix(b, []byte(s2.S2IndexTrailer)) { - return nil - } - b = bytes.TrimSuffix(b, []byte(s2.S2IndexTrailer)) - - if len(b) < 4 { - return nil - } - return b[:len(b)-4] -} - -// restoreIndexHeaders will index restore headers removed by RemoveIndexHeaders. -// No error checking is performed on the input. -func restoreIndexHeaders(in []byte) []byte { - if len(in) == 0 { - return nil - } - b := make([]byte, 0, 4+len(s2.S2IndexHeader)+len(in)+len(s2.S2IndexTrailer)+4) - b = append(b, s2.ChunkTypeIndex, 0, 0, 0) - b = append(b, []byte(s2.S2IndexHeader)...) - b = append(b, in...) - - var tmp [4]byte - binary.LittleEndian.PutUint32(tmp[:], uint32(len(b)+4+len(s2.S2IndexTrailer))) - b = append(b, tmp[:4]...) - // Trailer - b = append(b, []byte(s2.S2IndexTrailer)...) - - chunkLen := len(b) - 4 /*skippableFrameHeader*/ - b[1] = uint8(chunkLen >> 0) - b[2] = uint8(chunkLen >> 8) - b[3] = uint8(chunkLen >> 16) - return b -} diff --git a/cmd/object-api-utils_test.go b/cmd/object-api-utils_test.go index 7becb8198..3429727a9 100644 --- a/cmd/object-api-utils_test.go +++ b/cmd/object-api-utils_test.go @@ -642,7 +642,7 @@ func TestS2CompressReader(t *testing.T) { t.Errorf("no index returned") } var index s2.Index - _, err = index.Load(restoreIndexHeaders(idx)) + _, err = index.Load(s2.RestoreIndexHeaders(idx)) if err != nil { t.Errorf("error loading index: %v", err) } diff --git a/go.mod b/go.mod index 024d22dc8..7cb57e673 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/hashicorp/golang-lru v0.5.4 github.com/inconshreveable/mousetrap v1.0.0 github.com/json-iterator/go v1.1.12 - github.com/klauspost/compress v1.15.6 + github.com/klauspost/compress v1.15.8 github.com/klauspost/cpuid/v2 v2.0.14 github.com/klauspost/pgzip v1.2.5 github.com/klauspost/readahead v1.4.0 diff --git a/go.sum b/go.sum index 53b58e561..93e0f1eea 100644 --- a/go.sum +++ b/go.sum @@ -512,8 +512,8 @@ github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8 github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY= -github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.15.8 h1:JahtItbkWjf2jzm/T+qgMxkP9EMHsqEUA6vCMGmXvhA= +github.com/klauspost/compress v1.15.8/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=