MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-09 21:49:46 -05:00
Code Issues Packages Projects Releases Wiki Activity

add ruleguard support, fix all the reported issues (#10335)

Browse Source
This commit is contained in:
Harshavardhana
2020-08-24 12:11:20 -07:00
committed by GitHub
parent bc2ebe0021
commit caad314faa
46 changed files with 803 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/bucket/lifecycle/lifecycle_test.go
View File

@@ -241,7 +241,7 @@ func TestExpectedExpiryTime(t *testing.T) {
for i, tc := range testCases {
t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
got := expectedExpiryTime(tc.modTime, tc.days)
if got != tc.expected {
if !got.Equal(tc.expected) {
t.Fatalf("Expected %v to be equal to %v", got, tc.expected)
}
})

2
pkg/disk/obd.go
View File

@@ -84,7 +84,7 @@ func GetOBDInfo(ctx context.Context, drive, fsPath string) (Latency, Throughput,
return Latency{}, Throughput{}, fmt.Errorf("Expected to write %d, but only wrote %d", blockSize, n)
}
latencyInSecs := time.Since(startTime).Seconds()
latencies[i] = float64(latencyInSecs)
latencies[i] = latencyInSecs
}
// Sync every full writes fdatasync

14
pkg/disk/stat_linux.go
View File

@@ -1,4 +1,4 @@
// +build linux
// +build linux,!s390x,!arm,!386
/*
* MinIO Cloud Storage, (C) 2015, 2016, 2017 MinIO, Inc.
@@ -30,13 +30,13 @@ func GetInfo(path string) (info Info, err error) {
if err != nil {
return Info{}, err
}
reservedBlocks := uint64(s.Bfree) - uint64(s.Bavail)
reservedBlocks := s.Bfree - s.Bavail
info = Info{
Total: uint64(s.Frsize) * (uint64(s.Blocks) - reservedBlocks),
Free: uint64(s.Frsize) * uint64(s.Bavail),
Files: uint64(s.Files),
Ffree: uint64(s.Ffree),
FSType: getFSType(int64(s.Type)),
Total: uint64(s.Frsize) * (s.Blocks - reservedBlocks),
Free: uint64(s.Frsize) * s.Bavail,
Files: s.Files,
Ffree: s.Ffree,
FSType: getFSType(s.Type),
}
// Check for overflows.
// https://github.com/minio/minio/issues/8035

80
pkg/disk/stat_linux_32bit.go Normal file
View File

@@ -0,0 +1,80 @@
// +build linux,arm linux,386
/*
* MinIO Cloud Storage, (C) 2020 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package disk
import (
"fmt"
"strconv"
"syscall"
)
// fsType2StringMap - list of filesystems supported on linux
var fsType2StringMap = map[string]string{
"1021994": "TMPFS",
"137d": "EXT",
"4244": "HFS",
"4d44": "MSDOS",
"52654973": "REISERFS",
"5346544e": "NTFS",
"58465342": "XFS",
"61756673": "AUFS",
"6969": "NFS",
"ef51": "EXT2OLD",
"ef53": "EXT4",
"f15f": "ecryptfs",
"794c7630": "overlayfs",
"2fc12fc1": "zfs",
"ff534d42": "cifs",
"53464846": "wslfs",
}
// getFSType returns the filesystem type of the underlying mounted filesystem
func getFSType(ftype int32) string {
fsTypeHex := strconv.FormatInt(int64(ftype), 16)
fsTypeString, ok := fsType2StringMap[fsTypeHex]
if !ok {
return "UNKNOWN"
}
return fsTypeString
}
// GetInfo returns total and free bytes available in a directory, e.g. `/`.
func GetInfo(path string) (info Info, err error) {
s := syscall.Statfs_t{}
err = syscall.Statfs(path, &s)
if err != nil {
return Info{}, err
}
reservedBlocks := s.Bfree - s.Bavail
info = Info{
Total: uint64(s.Frsize) * (s.Blocks - reservedBlocks),
Free: uint64(s.Frsize) * s.Bavail,
Files: s.Files,
Ffree: s.Ffree,
FSType: getFSType(s.Type),
}
// Check for overflows.
// https://github.com/minio/minio/issues/8035
// XFS can show wrong values at times error out
// in such scenarios.
if info.Free > info.Total {
return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path)
}
return info, nil
}

80
pkg/disk/stat_linux_s390x.go Normal file
View File

@@ -0,0 +1,80 @@
// +build linux,s390x
/*
* MinIO Cloud Storage, (C) 2020 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package disk
import (
"fmt"
"strconv"
"syscall"
)
// fsType2StringMap - list of filesystems supported on linux
var fsType2StringMap = map[string]string{
"1021994": "TMPFS",
"137d": "EXT",
"4244": "HFS",
"4d44": "MSDOS",
"52654973": "REISERFS",
"5346544e": "NTFS",
"58465342": "XFS",
"61756673": "AUFS",
"6969": "NFS",
"ef51": "EXT2OLD",
"ef53": "EXT4",
"f15f": "ecryptfs",
"794c7630": "overlayfs",
"2fc12fc1": "zfs",
"ff534d42": "cifs",
"53464846": "wslfs",
}
// getFSType returns the filesystem type of the underlying mounted filesystem
func getFSType(ftype uint32) string {
fsTypeHex := strconv.FormatUint(uint64(ftype), 16)
fsTypeString, ok := fsType2StringMap[fsTypeHex]
if !ok {
return "UNKNOWN"
}
return fsTypeString
}
// GetInfo returns total and free bytes available in a directory, e.g. `/`.
func GetInfo(path string) (info Info, err error) {
s := syscall.Statfs_t{}
err = syscall.Statfs(path, &s)
if err != nil {
return Info{}, err
}
reservedBlocks := s.Bfree - s.Bavail
info = Info{
Total: uint64(s.Frsize) * (s.Blocks - reservedBlocks),
Free: uint64(s.Frsize) * s.Bavail,
Files: s.Files,
Ffree: s.Ffree,
FSType: getFSType(s.Type),
}
// Check for overflows.
// https://github.com/minio/minio/issues/8035
// XFS can show wrong values at times error out
// in such scenarios.
if info.Free > info.Total {
return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path)
}
return info, nil
}

2
pkg/disk/type_linux.go
View File

@@ -1,4 +1,4 @@
// +build linux
// +build linux,!s390x,!arm,!386
/*
* MinIO Cloud Storage, (C) 2017 MinIO, Inc.

2
pkg/dsync/drwmutex.go
View File

@@ -45,7 +45,7 @@ func log(format string, data ...interface{}) {
// DRWMutexAcquireTimeout - tolerance limit to wait for lock acquisition before.
const DRWMutexAcquireTimeout = 1 * time.Second // 1 second.
const drwMutexInfinite = time.Duration(1<<63 - 1)
const drwMutexInfinite = 1<<63 - 1
// A DRWMutex is a distributed mutual exclusion lock.
type DRWMutex struct {

4
pkg/lsync/lrwmutex.go
View File

@@ -46,7 +46,7 @@ func NewLRWMutex() *LRWMutex {
func (lm *LRWMutex) Lock() {
const isWriteLock = true
lm.lockLoop(context.Background(), lm.id, lm.source, time.Duration(math.MaxInt64), isWriteLock)
lm.lockLoop(context.Background(), lm.id, lm.source, math.MaxInt64, isWriteLock)
}
// GetLock tries to get a write lock on lm before the timeout occurs.
@@ -63,7 +63,7 @@ func (lm *LRWMutex) GetLock(ctx context.Context, id string, source string, timeo
func (lm *LRWMutex) RLock() {
const isWriteLock = false
lm.lockLoop(context.Background(), lm.id, lm.source, time.Duration(1<<63-1), isWriteLock)
lm.lockLoop(context.Background(), lm.id, lm.source, 1<<63-1, isWriteLock)
}
// GetRLock tries to get a read lock on lm before the timeout occurs.

4
pkg/madmin/retry.go
View File

@@ -81,11 +81,11 @@ func (adm AdminClient) newRetryTimer(ctx context.Context, maxRetry int, unit tim
}
//sleep = random_between(0, min(cap, base * 2 ** attempt))
sleep := unit * time.Duration(1<<uint(attempt))
sleep := unit * 1 << uint(attempt)
if sleep > cap {
sleep = cap
}
if jitter != NoJitter {
if jitter > NoJitter {
sleep -= time.Duration(adm.random.Float64() * float64(sleep) * jitter)
}
return sleep

4
pkg/retry/retry.go
View File

@@ -76,11 +76,11 @@ func NewTimerWithJitter(ctx context.Context, unit time.Duration, cap time.Durati
attempt = maxAttempt
}
//sleep = random_between(0, min(cap, base * 2 ** attempt))
sleep := unit * time.Duration(1<<uint(attempt))
sleep := unit * 1 << uint(attempt)
if sleep > cap {
sleep = cap
}
if jitter != NoJitter {
if jitter > NoJitter {
sleep -= time.Duration(rand.Float64() * float64(sleep) * jitter)
}
return sleep

6
pkg/s3select/internal/parquet-go/data/column.go
View File

@@ -186,11 +186,11 @@ func populate(columnDataMap map[string]*Column, input *jsonValue, tree *schema.T
}
var jsonData []byte
if jsonData, rerr = sjson.SetBytes([]byte{}, "key", key.Value()); err != nil {
if jsonData, rerr = sjson.SetBytes([]byte{}, "key", key.Value()); rerr != nil {
return false
}
if jsonData, rerr = sjson.SetBytes(jsonData, "value", value.Value()); err != nil {
if jsonData, rerr = sjson.SetBytes(jsonData, "value", value.Value()); rerr != nil {
return false
}
@@ -199,7 +199,7 @@ func populate(columnDataMap map[string]*Column, input *jsonValue, tree *schema.T
return false
}
if columnDataMap, rerr = populate(columnDataMap, jv, keyValueElement.Children, firstValueRL); err != nil {
if columnDataMap, rerr = populate(columnDataMap, jv, keyValueElement.Children, firstValueRL); rerr != nil {
return false
}

100
pkg/s3select/internal/parquet-go/gen-go/parquet/parquet.go
View File

@@ -922,7 +922,7 @@ func (p *Statistics) writeField3(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("null_count", thrift.I64, 3); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:null_count: ", p), err)
}
if err := oprot.WriteI64(int64(*p.NullCount)); err != nil {
if err := oprot.WriteI64(*p.NullCount); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.null_count (3) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -937,7 +937,7 @@ func (p *Statistics) writeField4(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("distinct_count", thrift.I64, 4); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:distinct_count: ", p), err)
}
if err := oprot.WriteI64(int64(*p.DistinctCount)); err != nil {
if err := oprot.WriteI64(*p.DistinctCount); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.distinct_count (4) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -1492,7 +1492,7 @@ func (p *DecimalType) writeField1(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("scale", thrift.I32, 1); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:scale: ", p), err)
}
if err := oprot.WriteI32(int32(p.Scale)); err != nil {
if err := oprot.WriteI32(p.Scale); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.scale (1) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -1505,7 +1505,7 @@ func (p *DecimalType) writeField2(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("precision", thrift.I32, 2); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:precision: ", p), err)
}
if err := oprot.WriteI32(int32(p.Precision)); err != nil {
if err := oprot.WriteI32(p.Precision); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.precision (2) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -2020,7 +2020,7 @@ func (p *TimestampType) writeField1(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("isAdjustedToUTC", thrift.BOOL, 1); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:isAdjustedToUTC: ", p), err)
}
if err := oprot.WriteBool(bool(p.IsAdjustedToUTC)); err != nil {
if err := oprot.WriteBool(p.IsAdjustedToUTC); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.isAdjustedToUTC (1) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -2171,7 +2171,7 @@ func (p *TimeType) writeField1(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("isAdjustedToUTC", thrift.BOOL, 1); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:isAdjustedToUTC: ", p), err)
}
if err := oprot.WriteBool(bool(p.IsAdjustedToUTC)); err != nil {
if err := oprot.WriteBool(p.IsAdjustedToUTC); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.isAdjustedToUTC (1) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -2277,7 +2277,7 @@ func (p *IntType) ReadField1(iprot thrift.TProtocol) error {
if v, err := iprot.ReadByte(); err != nil {
return thrift.PrependError("error reading field 1: ", err)
} else {
temp := int8(v)
temp := v
p.BitWidth = temp
}
return nil
@@ -2317,7 +2317,7 @@ func (p *IntType) writeField1(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("bitWidth", thrift.BYTE, 1); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:bitWidth: ", p), err)
}
if err := oprot.WriteByte(int8(p.BitWidth)); err != nil {
if err := oprot.WriteByte(p.BitWidth); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.bitWidth (1) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -2330,7 +2330,7 @@ func (p *IntType) writeField2(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("isSigned", thrift.BOOL, 2); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:isSigned: ", p), err)
}
if err := oprot.WriteBool(bool(p.IsSigned)); err != nil {
if err := oprot.WriteBool(p.IsSigned); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.isSigned (2) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -3558,7 +3558,7 @@ func (p *SchemaElement) writeField2(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("type_length", thrift.I32, 2); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:type_length: ", p), err)
}
if err := oprot.WriteI32(int32(*p.TypeLength)); err != nil {
if err := oprot.WriteI32(*p.TypeLength); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.type_length (2) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -3601,7 +3601,7 @@ func (p *SchemaElement) writeField5(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("num_children", thrift.I32, 5); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:num_children: ", p), err)
}
if err := oprot.WriteI32(int32(*p.NumChildren)); err != nil {
if err := oprot.WriteI32(*p.NumChildren); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.num_children (5) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -3631,7 +3631,7 @@ func (p *SchemaElement) writeField7(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("scale", thrift.I32, 7); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 7:scale: ", p), err)
}
if err := oprot.WriteI32(int32(*p.Scale)); err != nil {
if err := oprot.WriteI32(*p.Scale); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.scale (7) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -3646,7 +3646,7 @@ func (p *SchemaElement) writeField8(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("precision", thrift.I32, 8); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 8:precision: ", p), err)
}
if err := oprot.WriteI32(int32(*p.Precision)); err != nil {
if err := oprot.WriteI32(*p.Precision); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.precision (8) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -3661,7 +3661,7 @@ func (p *SchemaElement) writeField9(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("field_id", thrift.I32, 9); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 9:field_id: ", p), err)
}
if err := oprot.WriteI32(int32(*p.FieldID)); err != nil {
if err := oprot.WriteI32(*p.FieldID); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.field_id (9) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -3892,7 +3892,7 @@ func (p *DataPageHeader) writeField1(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("num_values", thrift.I32, 1); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:num_values: ", p), err)
}
if err := oprot.WriteI32(int32(p.NumValues)); err != nil {
if err := oprot.WriteI32(p.NumValues); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.num_values (1) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -4161,7 +4161,7 @@ func (p *DictionaryPageHeader) writeField1(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("num_values", thrift.I32, 1); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:num_values: ", p), err)
}
if err := oprot.WriteI32(int32(p.NumValues)); err != nil {
if err := oprot.WriteI32(p.NumValues); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.num_values (1) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -4188,7 +4188,7 @@ func (p *DictionaryPageHeader) writeField3(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("is_sorted", thrift.BOOL, 3); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:is_sorted: ", p), err)
}
if err := oprot.WriteBool(bool(*p.IsSorted)); err != nil {
if err := oprot.WriteBool(*p.IsSorted); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.is_sorted (3) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -4494,7 +4494,7 @@ func (p *DataPageHeaderV2) writeField1(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("num_values", thrift.I32, 1); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:num_values: ", p), err)
}
if err := oprot.WriteI32(int32(p.NumValues)); err != nil {
if err := oprot.WriteI32(p.NumValues); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.num_values (1) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -4507,7 +4507,7 @@ func (p *DataPageHeaderV2) writeField2(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("num_nulls", thrift.I32, 2); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:num_nulls: ", p), err)
}
if err := oprot.WriteI32(int32(p.NumNulls)); err != nil {
if err := oprot.WriteI32(p.NumNulls); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.num_nulls (2) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -4520,7 +4520,7 @@ func (p *DataPageHeaderV2) writeField3(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("num_rows", thrift.I32, 3); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:num_rows: ", p), err)
}
if err := oprot.WriteI32(int32(p.NumRows)); err != nil {
if err := oprot.WriteI32(p.NumRows); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.num_rows (3) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -4546,7 +4546,7 @@ func (p *DataPageHeaderV2) writeField5(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("definition_levels_byte_length", thrift.I32, 5); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:definition_levels_byte_length: ", p), err)
}
if err := oprot.WriteI32(int32(p.DefinitionLevelsByteLength)); err != nil {
if err := oprot.WriteI32(p.DefinitionLevelsByteLength); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.definition_levels_byte_length (5) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -4559,7 +4559,7 @@ func (p *DataPageHeaderV2) writeField6(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("repetition_levels_byte_length", thrift.I32, 6); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:repetition_levels_byte_length: ", p), err)
}
if err := oprot.WriteI32(int32(p.RepetitionLevelsByteLength)); err != nil {
if err := oprot.WriteI32(p.RepetitionLevelsByteLength); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.repetition_levels_byte_length (6) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -4573,7 +4573,7 @@ func (p *DataPageHeaderV2) writeField7(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("is_compressed", thrift.BOOL, 7); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 7:is_compressed: ", p), err)
}
if err := oprot.WriteBool(bool(p.IsCompressed)); err != nil {
if err := oprot.WriteBool(p.IsCompressed); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.is_compressed (7) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -4911,7 +4911,7 @@ func (p *PageHeader) writeField2(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("uncompressed_page_size", thrift.I32, 2); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:uncompressed_page_size: ", p), err)
}
if err := oprot.WriteI32(int32(p.UncompressedPageSize)); err != nil {
if err := oprot.WriteI32(p.UncompressedPageSize); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.uncompressed_page_size (2) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -4924,7 +4924,7 @@ func (p *PageHeader) writeField3(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("compressed_page_size", thrift.I32, 3); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:compressed_page_size: ", p), err)
}
if err := oprot.WriteI32(int32(p.CompressedPageSize)); err != nil {
if err := oprot.WriteI32(p.CompressedPageSize); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.compressed_page_size (3) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -4938,7 +4938,7 @@ func (p *PageHeader) writeField4(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("crc", thrift.I32, 4); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:crc: ", p), err)
}
if err := oprot.WriteI32(int32(*p.Crc)); err != nil {
if err := oprot.WriteI32(*p.Crc); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.crc (4) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -5302,7 +5302,7 @@ func (p *SortingColumn) writeField1(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("column_idx", thrift.I32, 1); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:column_idx: ", p), err)
}
if err := oprot.WriteI32(int32(p.ColumnIdx)); err != nil {
if err := oprot.WriteI32(p.ColumnIdx); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.column_idx (1) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -5315,7 +5315,7 @@ func (p *SortingColumn) writeField2(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("descending", thrift.BOOL, 2); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:descending: ", p), err)
}
if err := oprot.WriteBool(bool(p.Descending)); err != nil {
if err := oprot.WriteBool(p.Descending); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.descending (2) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -5328,7 +5328,7 @@ func (p *SortingColumn) writeField3(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("nulls_first", thrift.BOOL, 3); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:nulls_first: ", p), err)
}
if err := oprot.WriteBool(bool(p.NullsFirst)); err != nil {
if err := oprot.WriteBool(p.NullsFirst); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.nulls_first (3) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -5511,7 +5511,7 @@ func (p *PageEncodingStats) writeField3(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("count", thrift.I32, 3); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:count: ", p), err)
}
if err := oprot.WriteI32(int32(p.Count)); err != nil {
if err := oprot.WriteI32(p.Count); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.count (3) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -6071,7 +6071,7 @@ func (p *ColumnMetaData) writeField5(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("num_values", thrift.I64, 5); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:num_values: ", p), err)
}
if err := oprot.WriteI64(int64(p.NumValues)); err != nil {
if err := oprot.WriteI64(p.NumValues); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.num_values (5) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -6084,7 +6084,7 @@ func (p *ColumnMetaData) writeField6(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("total_uncompressed_size", thrift.I64, 6); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:total_uncompressed_size: ", p), err)
}
if err := oprot.WriteI64(int64(p.TotalUncompressedSize)); err != nil {
if err := oprot.WriteI64(p.TotalUncompressedSize); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.total_uncompressed_size (6) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -6097,7 +6097,7 @@ func (p *ColumnMetaData) writeField7(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("total_compressed_size", thrift.I64, 7); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 7:total_compressed_size: ", p), err)
}
if err := oprot.WriteI64(int64(p.TotalCompressedSize)); err != nil {
if err := oprot.WriteI64(p.TotalCompressedSize); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.total_compressed_size (7) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -6133,7 +6133,7 @@ func (p *ColumnMetaData) writeField9(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("data_page_offset", thrift.I64, 9); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 9:data_page_offset: ", p), err)
}
if err := oprot.WriteI64(int64(p.DataPageOffset)); err != nil {
if err := oprot.WriteI64(p.DataPageOffset); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.data_page_offset (9) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -6147,7 +6147,7 @@ func (p *ColumnMetaData) writeField10(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("index_page_offset", thrift.I64, 10); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 10:index_page_offset: ", p), err)
}
if err := oprot.WriteI64(int64(*p.IndexPageOffset)); err != nil {
if err := oprot.WriteI64(*p.IndexPageOffset); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.index_page_offset (10) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -6162,7 +6162,7 @@ func (p *ColumnMetaData) writeField11(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("dictionary_page_offset", thrift.I64, 11); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 11:dictionary_page_offset: ", p), err)
}
if err := oprot.WriteI64(int64(*p.DictionaryPageOffset)); err != nil {
if err := oprot.WriteI64(*p.DictionaryPageOffset); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.dictionary_page_offset (11) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -6505,7 +6505,7 @@ func (p *ColumnChunk) writeField2(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("file_offset", thrift.I64, 2); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:file_offset: ", p), err)
}
if err := oprot.WriteI64(int64(p.FileOffset)); err != nil {
if err := oprot.WriteI64(p.FileOffset); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.file_offset (2) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -6534,7 +6534,7 @@ func (p *ColumnChunk) writeField4(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("offset_index_offset", thrift.I64, 4); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:offset_index_offset: ", p), err)
}
if err := oprot.WriteI64(int64(*p.OffsetIndexOffset)); err != nil {
if err := oprot.WriteI64(*p.OffsetIndexOffset); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.offset_index_offset (4) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -6549,7 +6549,7 @@ func (p *ColumnChunk) writeField5(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("offset_index_length", thrift.I32, 5); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:offset_index_length: ", p), err)
}
if err := oprot.WriteI32(int32(*p.OffsetIndexLength)); err != nil {
if err := oprot.WriteI32(*p.OffsetIndexLength); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.offset_index_length (5) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -6564,7 +6564,7 @@ func (p *ColumnChunk) writeField6(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("column_index_offset", thrift.I64, 6); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:column_index_offset: ", p), err)
}
if err := oprot.WriteI64(int64(*p.ColumnIndexOffset)); err != nil {
if err := oprot.WriteI64(*p.ColumnIndexOffset); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.column_index_offset (6) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -6579,7 +6579,7 @@ func (p *ColumnChunk) writeField7(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("column_index_length", thrift.I32, 7); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 7:column_index_length: ", p), err)
}
if err := oprot.WriteI32(int32(*p.ColumnIndexLength)); err != nil {
if err := oprot.WriteI32(*p.ColumnIndexLength); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.column_index_length (7) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -6807,7 +6807,7 @@ func (p *RowGroup) writeField2(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("total_byte_size", thrift.I64, 2); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:total_byte_size: ", p), err)
}
if err := oprot.WriteI64(int64(p.TotalByteSize)); err != nil {
if err := oprot.WriteI64(p.TotalByteSize); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.total_byte_size (2) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -6820,7 +6820,7 @@ func (p *RowGroup) writeField3(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("num_rows", thrift.I64, 3); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:num_rows: ", p), err)
}
if err := oprot.WriteI64(int64(p.NumRows)); err != nil {
if err := oprot.WriteI64(p.NumRows); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.num_rows (3) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -7220,7 +7220,7 @@ func (p *PageLocation) writeField1(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("offset", thrift.I64, 1); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:offset: ", p), err)
}
if err := oprot.WriteI64(int64(p.Offset)); err != nil {
if err := oprot.WriteI64(p.Offset); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.offset (1) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -7233,7 +7233,7 @@ func (p *PageLocation) writeField2(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("compressed_page_size", thrift.I32, 2); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:compressed_page_size: ", p), err)
}
if err := oprot.WriteI32(int32(p.CompressedPageSize)); err != nil {
if err := oprot.WriteI32(p.CompressedPageSize); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.compressed_page_size (2) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -7246,7 +7246,7 @@ func (p *PageLocation) writeField3(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("first_row_index", thrift.I64, 3); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:first_row_index: ", p), err)
}
if err := oprot.WriteI64(int64(p.FirstRowIndex)); err != nil {
if err := oprot.WriteI64(p.FirstRowIndex); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.first_row_index (3) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -7646,7 +7646,7 @@ func (p *ColumnIndex) writeField1(oprot thrift.TProtocol) (err error) {
return thrift.PrependError("error writing list begin: ", err)
}
for _, v := range p.NullPages {
if err := oprot.WriteBool(bool(v)); err != nil {
if err := oprot.WriteBool(v); err != nil {
return thrift.PrependError(fmt.Sprintf("%T. (0) field write error: ", p), err)
}
}
@@ -7723,7 +7723,7 @@ func (p *ColumnIndex) writeField5(oprot thrift.TProtocol) (err error) {
return thrift.PrependError("error writing list begin: ", err)
}
for _, v := range p.NullCounts {
if err := oprot.WriteI64(int64(v)); err != nil {
if err := oprot.WriteI64(v); err != nil {
return thrift.PrependError(fmt.Sprintf("%T. (0) field write error: ", p), err)
}
}
@@ -8058,7 +8058,7 @@ func (p *FileMetaData) writeField1(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("version", thrift.I32, 1); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:version: ", p), err)
}
if err := oprot.WriteI32(int32(p.Version)); err != nil {
if err := oprot.WriteI32(p.Version); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.version (1) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {
@@ -8092,7 +8092,7 @@ func (p *FileMetaData) writeField3(oprot thrift.TProtocol) (err error) {
if err := oprot.WriteFieldBegin("num_rows", thrift.I64, 3); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:num_rows: ", p), err)
}
if err := oprot.WriteI64(int64(p.NumRows)); err != nil {
if err := oprot.WriteI64(p.NumRows); err != nil {
return thrift.PrependError(fmt.Sprintf("%T.num_rows (3) field write error: ", p), err)
}
if err := oprot.WriteFieldEnd(); err != nil {

2
pkg/s3select/parquet/reader.go
View File

@@ -64,7 +64,7 @@ func (r *Reader) Read(dst sql.Record) (rec sql.Record, rerr error) {
case parquetgen.Type_INT32:
value = int64(v.Value.(int32))
case parquetgen.Type_INT64:
value = int64(v.Value.(int64))
value = v.Value.(int64)
case parquetgen.Type_FLOAT:
value = float64(v.Value.(float32))
case parquetgen.Type_DOUBLE:

5
pkg/s3select/sql/evaluate.go
View File

@@ -246,6 +246,8 @@ func (e *ListExpr) evalNode(r Record) (*Value, error) {
return FromArray(res), nil
}
const floatCmpTolerance = 0.000001
func (e *In) evalInNode(r Record, lhs *Value) (*Value, error) {
// Compare two values in terms of in-ness.
var cmp func(a, b Value) bool
@@ -275,7 +277,8 @@ func (e *In) evalInNode(r Record, lhs *Value) (*Value, error) {
aF, aOK := a.ToFloat()
bF, bOK := b.ToFloat()
return aOK && bOK && aF == bF
diff := math.Abs(aF - bF)
return aOK && bOK && diff < floatCmpTolerance
}
var rhs Value

5
pkg/s3select/sql/value.go
View File

@@ -785,6 +785,7 @@ func intCompare(op string, left, right int64) bool {
}
func floatCompare(op string, left, right float64) bool {
diff := math.Abs(left - right)
switch op {
case opLt:
return left < right
@@ -795,9 +796,9 @@ func floatCompare(op string, left, right float64) bool {
case opGte:
return left >= right
case opEq:
return left == right
return diff < floatCmpTolerance
case opIneq:
return left != right
return diff > floatCmpTolerance
}
// This case does not happen
return false

3
pkg/s3select/sql/value_test.go
View File

@@ -559,7 +559,8 @@ func TestValue_bytesToFloat(t *testing.T) {
value: tt.fields.value,
}
got, got1 := v.bytesToFloat()
if got != tt.want {
diff := math.Abs(got - tt.want)
if diff > floatCmpTolerance {
t.Errorf("bytesToFloat() got = %v, want %v", got, tt.want)
}
if got1 != tt.wantOK {

9
pkg/sys/stats_linux.go
View File

@@ -1,4 +1,4 @@
// +build linux
// +build linux,!arm,!386
/*
* MinIO Cloud Storage, (C) 2016,2017 MinIO, Inc.
@@ -64,13 +64,12 @@ func getSysinfoMemoryLimit() (limit uint64, err error) {
// Some fields in syscall.Sysinfo_t have different integer sizes
// in different platform architectures. Cast all fields to uint64.
totalRAM := uint64(si.Totalram)
unit := uint64(si.Unit)
unit := si.Unit
totalRAM := si.Totalram
// Total RAM is always the multiplicative value
// of unit size and total ram.
limit = unit * totalRAM
return limit, nil
return uint64(unit) * totalRAM, nil
}
// GetStats - return system statistics, currently only

86
pkg/sys/stats_linux_32bit.go Normal file
View File

@@ -0,0 +1,86 @@
// +build linux,arm linux,386
/*
* MinIO Cloud Storage, (C) 2020 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package sys
import (
"os"
"syscall"
"github.com/minio/minio/pkg/cgroup"
)
// Get the final system memory limit chosen by the user.
// by default without any configuration on a vanilla Linux
// system you would see physical RAM limit. If cgroup
// is configured at some point in time this function
// would return the memory limit chosen for the given pid.
func getMemoryLimit() (sysLimit uint64, err error) {
if sysLimit, err = getSysinfoMemoryLimit(); err != nil {
// Physical memory info is not accessible, just exit here.
return 0, err
}
// Following code is deliberately ignoring the error.
cGroupLimit, gerr := cgroup.GetMemoryLimit(os.Getpid())
if gerr != nil {
// Upon error just return system limit.
return sysLimit, nil
}
// cgroup limit is lesser than system limit means
// user wants to limit the memory usage further
// treat cgroup limit as the system limit.
if cGroupLimit <= sysLimit {
sysLimit = cGroupLimit
}
// Final system limit.
return sysLimit, nil
}
// Get physical RAM size of the node.
func getSysinfoMemoryLimit() (limit uint64, err error) {
var si syscall.Sysinfo_t
if err = syscall.Sysinfo(&si); err != nil {
return 0, err
}
// Some fields in syscall.Sysinfo_t have different integer sizes
// in different platform architectures. Cast all fields to uint64.
unit := si.Unit
totalRAM := si.Totalram
// Total RAM is always the multiplicative value
// of unit size and total ram.
return uint64(unit) * uint64(totalRAM), nil
}
// GetStats - return system statistics, currently only
// supported value is TotalRAM.
func GetStats() (stats Stats, err error) {
var limit uint64
limit, err = getMemoryLimit()
if err != nil {
return Stats{}, err
}
stats.TotalRAM = limit
return stats, nil
}