mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
Upgrade compression package (#15284)
Includes mitigation for CVE-2022-30631 (Go should still be updated) Remove functions now available upstream.
This commit is contained in:
Klaus Post
GitHub
parent
788fd3df81
commit
697c9973a7
@ -21,7 +21,6 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
@ -544,7 +543,7 @@ func getCompressedOffsets(oi ObjectInfo, offset int64, decrypt func([]byte) ([]b
|
||||
if err == nil {
|
||||
// Load Index
|
||||
var idx s2.Index
|
||||
_, err := idx.Load(restoreIndexHeaders(dec))
|
||||
_, err := idx.Load(s2.RestoreIndexHeaders(dec))
|
||||
|
||||
// Find compressed/uncompressed offsets of our partskip
|
||||
compOff, uCompOff, err2 := idx.Find(partSkip)
|
||||
@ -567,7 +566,7 @@ func getCompressedOffsets(oi ObjectInfo, offset int64, decrypt func([]byte) ([]b
|
||||
} else {
|
||||
// Not encrypted
|
||||
var idx s2.Index
|
||||
_, err := idx.Load(restoreIndexHeaders(oi.Parts[firstPartIdx].Index))
|
||||
_, err := idx.Load(s2.RestoreIndexHeaders(oi.Parts[firstPartIdx].Index))
|
||||
|
||||
// Find compressed/uncompressed offsets of our partskip
|
||||
compOff, uCompOff, err2 := idx.Find(partSkip)
|
||||
@ -1010,7 +1009,7 @@ func newS2CompressReader(r io.Reader, on int64) (rc io.ReadCloser, idx func() []
|
||||
// If more than 8MB was written, generate index.
|
||||
if cn > 8<<20 {
|
||||
idx, err := comp.CloseIndex()
|
||||
idx = removeIndexHeaders(idx)
|
||||
idx = s2.RemoveIndexHeaders(idx)
|
||||
indexCh <- idx
|
||||
pw.CloseWithError(err)
|
||||
return
|
||||
@ -1128,65 +1127,3 @@ func hasSpaceFor(di []*DiskInfo, size int64) bool {
|
||||
wantLeft := uint64(float64(total) * (1.0 - diskFillFraction))
|
||||
return available > wantLeft
|
||||
}
|
||||
|
||||
// removeIndexHeaders will trim all headers and trailers from a given index.
|
||||
// This is expected to save 20 bytes.
|
||||
// These can be restored using RestoreIndexHeaders.
|
||||
// This removes a layer of security, but is the most compact representation.
|
||||
// Returns nil if headers contains errors.
|
||||
// The returned slice references the provided slice.
|
||||
func removeIndexHeaders(b []byte) []byte {
|
||||
const save = 4 + len(s2.S2IndexHeader) + len(s2.S2IndexTrailer) + 4
|
||||
if len(b) <= save {
|
||||
return nil
|
||||
}
|
||||
if b[0] != s2.ChunkTypeIndex {
|
||||
return nil
|
||||
}
|
||||
chunkLen := int(b[1]) | int(b[2])<<8 | int(b[3])<<16
|
||||
b = b[4:]
|
||||
|
||||
// Validate we have enough...
|
||||
if len(b) < chunkLen {
|
||||
return nil
|
||||
}
|
||||
b = b[:chunkLen]
|
||||
|
||||
if !bytes.Equal(b[:len(s2.S2IndexHeader)], []byte(s2.S2IndexHeader)) {
|
||||
return nil
|
||||
}
|
||||
b = b[len(s2.S2IndexHeader):]
|
||||
if !bytes.HasSuffix(b, []byte(s2.S2IndexTrailer)) {
|
||||
return nil
|
||||
}
|
||||
b = bytes.TrimSuffix(b, []byte(s2.S2IndexTrailer))
|
||||
|
||||
if len(b) < 4 {
|
||||
return nil
|
||||
}
|
||||
return b[:len(b)-4]
|
||||
}
|
||||
|
||||
// restoreIndexHeaders will index restore headers removed by RemoveIndexHeaders.
|
||||
// No error checking is performed on the input.
|
||||
func restoreIndexHeaders(in []byte) []byte {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
b := make([]byte, 0, 4+len(s2.S2IndexHeader)+len(in)+len(s2.S2IndexTrailer)+4)
|
||||
b = append(b, s2.ChunkTypeIndex, 0, 0, 0)
|
||||
b = append(b, []byte(s2.S2IndexHeader)...)
|
||||
b = append(b, in...)
|
||||
|
||||
var tmp [4]byte
|
||||
binary.LittleEndian.PutUint32(tmp[:], uint32(len(b)+4+len(s2.S2IndexTrailer)))
|
||||
b = append(b, tmp[:4]...)
|
||||
// Trailer
|
||||
b = append(b, []byte(s2.S2IndexTrailer)...)
|
||||
|
||||
chunkLen := len(b) - 4 /*skippableFrameHeader*/
|
||||
b[1] = uint8(chunkLen >> 0)
|
||||
b[2] = uint8(chunkLen >> 8)
|
||||
b[3] = uint8(chunkLen >> 16)
|
||||
return b
|
||||
}
|
||||
|
||||
@ -642,7 +642,7 @@ func TestS2CompressReader(t *testing.T) {
|
||||
t.Errorf("no index returned")
|
||||
}
|
||||
var index s2.Index
|
||||
_, err = index.Load(restoreIndexHeaders(idx))
|
||||
_, err = index.Load(s2.RestoreIndexHeaders(idx))
|
||||
if err != nil {
|
||||
t.Errorf("error loading index: %v", err)
|
||||
}
|
||||
|
||||
2
go.mod
2
go.mod
@ -35,7 +35,7 @@ require (
|
||||
github.com/hashicorp/golang-lru v0.5.4
|
||||
github.com/inconshreveable/mousetrap v1.0.0
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/klauspost/compress v1.15.6
|
||||
github.com/klauspost/compress v1.15.8
|
||||
github.com/klauspost/cpuid/v2 v2.0.14
|
||||
github.com/klauspost/pgzip v1.2.5
|
||||
github.com/klauspost/readahead v1.4.0
|
||||
|
||||
4
go.sum
4
go.sum
@ -512,8 +512,8 @@ github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8
|
||||
github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
|
||||
github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||
github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY=
|
||||
github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/compress v1.15.8 h1:JahtItbkWjf2jzm/T+qgMxkP9EMHsqEUA6vCMGmXvhA=
|
||||
github.com/klauspost/compress v1.15.8/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
|
||||
Loading…
Reference in New Issue
Block a user