MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 21:02:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

tests: Do not allow forced type asserts (#20905)

Browse Source
This commit is contained in:
Klaus Post
2025-02-18 08:25:55 -08:00
committed by GitHub
parent aeabac9181
commit 90f5e1e5f6
100 changed files with 371 additions and 358 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
internal/s3select/csv/reader.go
View File

@@ -27,25 +27,26 @@ import (
"unicode/utf8"
csv "github.com/minio/csvparser"
"github.com/minio/minio/internal/bpool"
"github.com/minio/minio/internal/s3select/sql"
)
// Reader - CSV record reader for S3Select.
type Reader struct {
args *ReaderArgs
readCloser io.ReadCloser // raw input
buf *bufio.Reader // input to the splitter
columnNames []string // names of columns
nameIndexMap map[string]int64 // name to column index
current [][]string // current block of results to be returned
recordsRead int // number of records read in current slice
input chan *queueItem // input for workers
queue chan *queueItem // output from workers in order
err error // global error state, only touched by Reader.Read
bufferPool sync.Pool // pool of []byte objects for input
csvDstPool sync.Pool // pool of [][]string used for output
close chan struct{} // used for shutting down the splitter before end of stream
readerWg sync.WaitGroup // used to keep track of async reader.
readCloser io.ReadCloser // raw input
buf *bufio.Reader // input to the splitter
columnNames []string // names of columns
nameIndexMap map[string]int64 // name to column index
current [][]string // current block of results to be returned
recordsRead int // number of records read in current slice
input chan *queueItem // input for workers
queue chan *queueItem // output from workers in order
err error // global error state, only touched by Reader.Read
bufferPool bpool.Pool[[]byte] // pool of []byte objects for input
csvDstPool bpool.Pool[[][]string] // pool of [][]string used for output
close chan struct{} // used for shutting down the splitter before end of stream
readerWg sync.WaitGroup // used to keep track of async reader.
}
// queueItem is an item in the queue.
@@ -69,7 +70,7 @@ func (r *Reader) Read(dst sql.Record) (sql.Record, error) {
r.err = io.EOF
return nil, r.err
}
//nolint:staticcheck // SA6002 Using pointer would allocate more since we would have to copy slice header before taking a pointer.
r.csvDstPool.Put(r.current)
r.current = <-item.dst
r.err = item.err
@@ -182,12 +183,12 @@ func (r *Reader) startReaders(newReader func(io.Reader) *csv.Reader) error {
}
}
r.bufferPool.New = func() interface{} {
r.bufferPool.New = func() []byte {
return make([]byte, csvSplitSize+1024)
}
// Return first block
next, nextErr := r.nextSplit(csvSplitSize, r.bufferPool.Get().([]byte))
next, nextErr := r.nextSplit(csvSplitSize, r.bufferPool.Get())
// Check if first block is valid.
if !utf8.Valid(next) {
return errInvalidTextEncodingError()
@@ -224,7 +225,7 @@ func (r *Reader) startReaders(newReader func(io.Reader) *csv.Reader) error {
// Exit on any error.
return
}
next, nextErr = r.nextSplit(csvSplitSize, r.bufferPool.Get().([]byte))
next, nextErr = r.nextSplit(csvSplitSize, r.bufferPool.Get())
}
}()
@@ -236,8 +237,8 @@ func (r *Reader) startReaders(newReader func(io.Reader) *csv.Reader) error {
in.dst <- nil
continue
}
dst, ok := r.csvDstPool.Get().([][]string)
if !ok {
dst := r.csvDstPool.Get()
if len(dst) < 1000 {
dst = make([][]string, 0, 1000)
}

34
internal/s3select/json/preader.go
View File

@@ -24,6 +24,7 @@ import (
"runtime"
"sync"
"github.com/minio/minio/internal/bpool"
"github.com/minio/minio/internal/s3select/jstream"
"github.com/minio/minio/internal/s3select/sql"
)
@@ -32,17 +33,17 @@ import (
// Operates concurrently on line-delimited JSON.
type PReader struct {
args *ReaderArgs
readCloser io.ReadCloser // raw input
buf *bufio.Reader // input to the splitter
current []jstream.KVS // current block of results to be returned
recordsRead int // number of records read in current slice
input chan *queueItem // input for workers
queue chan *queueItem // output from workers in order
err error // global error state, only touched by Reader.Read
bufferPool sync.Pool // pool of []byte objects for input
kvDstPool sync.Pool // pool of []jstream.KV used for output
close chan struct{} // used for shutting down the splitter before end of stream
readerWg sync.WaitGroup // used to keep track of async reader.
readCloser io.ReadCloser // raw input
buf *bufio.Reader // input to the splitter
current []jstream.KVS // current block of results to be returned
recordsRead int // number of records read in current slice
input chan *queueItem // input for workers
queue chan *queueItem // output from workers in order
err error // global error state, only touched by Reader.Read
bufferPool bpool.Pool[[]byte] // pool of []byte objects for input
kvDstPool bpool.Pool[[]jstream.KVS] // pool of []jstream.KVS used for output
close chan struct{} // used for shutting down the splitter before end of stream
readerWg sync.WaitGroup // used to keep track of async reader.
}
// queueItem is an item in the queue.
@@ -66,7 +67,6 @@ func (r *PReader) Read(dst sql.Record) (sql.Record, error) {
r.err = io.EOF
return nil, r.err
}
//nolint:staticcheck // SA6002 Using pointer would allocate more since we would have to copy slice header before taking a pointer.
r.kvDstPool.Put(r.current)
r.current = <-item.dst
r.err = item.err
@@ -133,7 +133,7 @@ const jsonSplitSize = 128 << 10
// and a number of workers based on GOMAXPROCS.
// If an error is returned no goroutines have been started and r.err will have been set.
func (r *PReader) startReaders() {
r.bufferPool.New = func() interface{} {
r.bufferPool.New = func() []byte {
return make([]byte, jsonSplitSize+1024)
}
@@ -148,7 +148,7 @@ func (r *PReader) startReaders() {
defer close(r.queue)
defer r.readerWg.Done()
for {
next, err := r.nextSplit(jsonSplitSize, r.bufferPool.Get().([]byte))
next, err := r.nextSplit(jsonSplitSize, r.bufferPool.Get())
q := queueItem{
input: next,
dst: make(chan []jstream.KVS, 1),
@@ -180,8 +180,8 @@ func (r *PReader) startReaders() {
in.dst <- nil
continue
}
dst, ok := r.kvDstPool.Get().([]jstream.KVS)
if !ok {
dst := r.kvDstPool.Get()
if len(dst) < 1000 {
dst = make([]jstream.KVS, 0, 1000)
}
@@ -193,7 +193,7 @@ func (r *PReader) startReaders() {
if mv.ValueType == jstream.Object {
// This is a JSON object type (that preserves key
// order)
kvs = mv.Value.(jstream.KVS)
kvs, _ = mv.Value.(jstream.KVS)
} else {
// To be AWS S3 compatible Select for JSON needs to
// output non-object JSON as single column value

2
internal/s3select/json/reader.go
View File

@@ -51,7 +51,7 @@ func (r *Reader) Read(dst sql.Record) (sql.Record, error) {
if v.ValueType == jstream.Object {
// This is a JSON object type (that preserves key
// order)
kvs = v.Value.(jstream.KVS)
kvs, _ = v.Value.(jstream.KVS)
} else {
// To be AWS S3 compatible Select for JSON needs to
// output non-object JSON as single column value

4
internal/s3select/message.go
View File

@@ -26,6 +26,8 @@ import (
"strconv"
"sync/atomic"
"time"
xhttp "github.com/minio/minio/internal/http"
)
// A message is in the format specified in
@@ -262,7 +264,7 @@ func (writer *messageWriter) write(data []byte) bool {
return false
}
writer.writer.(http.Flusher).Flush()
xhttp.Flush(writer.writer)
return true
}

1
internal/s3select/parquet/reader.go
View File

@@ -56,7 +56,6 @@ func (pr *Reader) Read(dst sql.Record) (rec sql.Record, rerr error) {
kvs := jstream.KVS{}
for _, col := range pr.r.Columns() {
var value interface{}
if v, ok := nextRow[col.FlatName()]; ok {
value, err = convertFromAnnotation(col.Element(), v)

13
internal/s3select/select.go
View File

@@ -32,6 +32,7 @@ import (
"github.com/klauspost/compress/s2"
"github.com/klauspost/compress/zstd"
gzip "github.com/klauspost/pgzip"
"github.com/minio/minio/internal/bpool"
"github.com/minio/minio/internal/config"
xioutil "github.com/minio/minio/internal/ioutil"
"github.com/minio/minio/internal/s3select/csv"
@@ -81,15 +82,15 @@ func init() {
parquetSupport = env.Get("MINIO_API_SELECT_PARQUET", config.EnableOff) == config.EnableOn
}
var bufPool = sync.Pool{
New: func() interface{} {
var bufPool = bpool.Pool[*bytes.Buffer]{
New: func() *bytes.Buffer {
// make a buffer with a reasonable capacity.
return bytes.NewBuffer(make([]byte, 0, maxRecordSize))
},
}
var bufioWriterPool = sync.Pool{
New: func() interface{} {
var bufioWriterPool = bpool.Pool[*bufio.Writer]{
New: func() *bufio.Writer {
// io.Discard is just used to create the writer. Actual destination
// writer is set later by Reset() before using it.
return bufio.NewWriter(xioutil.Discard)
@@ -468,7 +469,7 @@ func (s3Select *S3Select) marshal(buf *bytes.Buffer, record sql.Record) error {
switch s3Select.Output.format {
case csvFormat:
// Use bufio Writer to prevent csv.Writer from allocating a new buffer.
bufioWriter := bufioWriterPool.Get().(*bufio.Writer)
bufioWriter := bufioWriterPool.Get()
defer func() {
bufioWriter.Reset(xioutil.Discard)
bufioWriterPool.Put(bufioWriter)
@@ -530,7 +531,7 @@ func (s3Select *S3Select) Evaluate(w http.ResponseWriter) {
}
var err error
sendRecord := func() bool {
buf := bufPool.Get().(*bytes.Buffer)
buf := bufPool.Get()
buf.Reset()
for _, outputRecord := range outputQueue {

1
internal/s3select/sql/funceval.go
View File

@@ -107,7 +107,6 @@ func (e *FuncExpr) evalSQLFnNode(r Record, tableAlias string) (res *Value, err e
case sqlFnDateDiff:
return handleDateDiff(r, e.DateDiff, tableAlias)
}
// For all simple argument functions, we evaluate the arguments here

1
internal/s3select/sql/stringfuncs.go
View File

@@ -107,7 +107,6 @@ func evalSQLLike(text, pattern string, escape rune) (match bool, err error) {
default:
s = append(s, r)
}
}
if hasLeadingPercent {
return strings.HasSuffix(text, string(s)), nil

1
internal/s3select/sql/timestampfuncs.go
View File

@@ -175,7 +175,6 @@ func dateDiff(timePart string, ts1, ts2 time.Time) (*Value, error) {
seconds := duration / time.Second
return FromInt(int64(seconds)), nil
default:
}
return nil, errNotImplemented
}

9
internal/s3select/sql/value.go
View File

@@ -663,8 +663,13 @@ func inferTypeForArithOp(a *Value) error {
a.setFloat(f)
return nil
}
err := fmt.Errorf("Could not convert %q to a number", string(a.value.([]byte)))
var s string
if v, ok := a.value.([]byte); ok {
s = string(v)
} else {
s = fmt.Sprint(a.value)
}
err := fmt.Errorf("Could not convert %q to a number", s)
return errInvalidDataType(err)
}