diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 770a9b023..7ddf8dcc7 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -37,6 +37,8 @@ jobs: GO111MODULE: on MINIO_CI_CD: 1 run: | + sudo sysctl net.ipv6.conf.all.disable_ipv6=0 + sudo sysctl net.ipv6.conf.default.disable_ipv6=0 sudo apt-get install devscripts shellcheck nancy_version=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/sonatype-nexus-community/nancy/releases/latest | sed "s/https:\/\/github.com\/sonatype-nexus-community\/nancy\/releases\/tag\///") curl -L -o nancy https://github.com/sonatype-nexus-community/nancy/releases/download/${nancy_version}/nancy-linux.amd64-${nancy_version} && chmod +x nancy diff --git a/buildscripts/verify-build.sh b/buildscripts/verify-build.sh index d7b18c1c0..6967b3f2f 100755 --- a/buildscripts/verify-build.sh +++ b/buildscripts/verify-build.sh @@ -76,8 +76,8 @@ function start_minio_zone_erasure_sets_ipv6() export MINIO_ACCESS_KEY=$ACCESS_KEY export MINIO_SECRET_KEY=$SECRET_KEY - "${MINIO[@]}" server --address="[::1]:9000" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9000.log" 2>&1 & - "${MINIO[@]}" server --address="[::1]:9001" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9001.log" 2>&1 & + "${MINIO[@]}" server --address="[::1]:9000" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-ipv6-9000.log" 2>&1 & + "${MINIO[@]}" server --address="[::1]:9001" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-ipv6-9001.log" 2>&1 & sleep 40 } diff --git a/cmd/erasure-zones.go b/cmd/erasure-zones.go index fceee6bcc..e41d49947 100644 --- a/cmd/erasure-zones.go +++ b/cmd/erasure-zones.go @@ -1750,17 +1750,17 @@ func (z *erasureZones) Walk(ctx context.Context, bucket, prefix string, results return err } + var zoneDrivesPerSet []int + for _, zone := range z.zones { + zoneDrivesPerSet = append(zoneDrivesPerSet, zone.listTolerancePerSet-2) + } + if opts.WalkVersions { var zonesEntryChs [][]FileInfoVersionsCh for _, zone := range z.zones { zonesEntryChs = append(zonesEntryChs, zone.startMergeWalksVersions(ctx, bucket, prefix, "", true, ctx.Done())) } - var zoneDrivesPerSet []int - for _, zone := range z.zones { - zoneDrivesPerSet = append(zoneDrivesPerSet, zone.setDriveCount) - } - var zonesEntriesInfos [][]FileInfoVersions var zonesEntriesValid [][]bool for _, entryChs := range zonesEntryChs { @@ -1772,20 +1772,17 @@ func (z *erasureZones) Walk(ctx context.Context, bucket, prefix string, results defer close(results) for { - entry, quorumCount, zoneIndex, ok := lexicallySortedEntryZoneVersions(zonesEntryChs, zonesEntriesInfos, zonesEntriesValid) + entry, quorumCount, _, ok := lexicallySortedEntryZoneVersions(zonesEntryChs, zonesEntriesInfos, zonesEntriesValid) if !ok { // We have reached EOF across all entryChs, break the loop. return } - if quorumCount >= zoneDrivesPerSet[zoneIndex]/2 { - // Read quorum exists proceed + if quorumCount > 0 { for _, version := range entry.Versions { results <- version.ToObjectInfo(bucket, version.Name) } } - - // skip entries which do not have quorum } }() @@ -1793,10 +1790,8 @@ func (z *erasureZones) Walk(ctx context.Context, bucket, prefix string, results } zonesEntryChs := make([][]FileInfoCh, 0, len(z.zones)) - zoneDrivesPerSet := make([]int, 0, len(z.zones)) for _, zone := range z.zones { zonesEntryChs = append(zonesEntryChs, zone.startMergeWalks(ctx, bucket, prefix, "", true, ctx.Done())) - zoneDrivesPerSet = append(zoneDrivesPerSet, zone.setDriveCount) } zonesEntriesInfos := make([][]FileInfo, 0, len(zonesEntryChs)) @@ -1810,17 +1805,15 @@ func (z *erasureZones) Walk(ctx context.Context, bucket, prefix string, results defer close(results) for { - entry, quorumCount, zoneIndex, ok := lexicallySortedEntryZone(zonesEntryChs, zonesEntriesInfos, zonesEntriesValid) + entry, quorumCount, _, ok := lexicallySortedEntryZone(zonesEntryChs, zonesEntriesInfos, zonesEntriesValid) if !ok { // We have reached EOF across all entryChs, break the loop. return } - if quorumCount >= zoneDrivesPerSet[zoneIndex]/2 { - // Read quorum exists proceed + if quorumCount > 0 { results <- entry.ToObjectInfo(bucket, entry.Name) } - // skip entries which do not have quorum } }() @@ -1859,6 +1852,7 @@ func (z *erasureZones) HealObjects(ctx context.Context, bucket, prefix string, o if !ok { break } + // Indicate that first attempt was a success and subsequent loop // knows that its not our first attempt at 'prefix' err = nil @@ -1866,6 +1860,7 @@ func (z *erasureZones) HealObjects(ctx context.Context, bucket, prefix string, o if zoneIndex >= len(zoneDrivesPerSet) || zoneIndex < 0 { return fmt.Errorf("invalid zone index returned: %d", zoneIndex) } + if quorumCount == zoneDrivesPerSet[zoneIndex] && opts.ScanMode == madmin.HealNormalScan { // Skip good entries. continue diff --git a/cmd/net.go b/cmd/net.go index 89762c28a..133a6f093 100644 --- a/cmd/net.go +++ b/cmd/net.go @@ -189,21 +189,13 @@ func isHostIP(ipAddress string) bool { // Note: The check method tries to listen on given port and closes it. // It is possible to have a disconnected client in this tiny window of time. func checkPortAvailability(host, port string) (err error) { - network := []string{"tcp", "tcp4", "tcp6"} - for _, n := range network { - l, err := net.Listen(n, net.JoinHostPort(host, port)) - if err == nil { - // As we are able to listen on this network, the port is not in use. - // Close the listener and continue check other networks. - if err = l.Close(); err != nil { - return err - } - } else { - return err - } + l, err := net.Listen("tcp", net.JoinHostPort(host, port)) + if err != nil { + return err } - - return nil + // As we are able to listen on this network, the port is not in use. + // Close the listener and continue check other networks. + return l.Close() } // extractHostPort - extracts host/port from many address formats diff --git a/cmd/object-handlers_test.go b/cmd/object-handlers_test.go index 25c7d168c..6c89e8858 100644 --- a/cmd/object-handlers_test.go +++ b/cmd/object-handlers_test.go @@ -2788,7 +2788,8 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s if rec.Code == http.StatusOK { // Verify whether the bucket obtained object is same as the one created. if !bytes.Equal(testCase.expectedContent, actualContent) { - t.Errorf("Test %d : MinIO %s: Object content differs from expected value.", i+1, instanceType) + t.Errorf("Test %d : MinIO %s: CompleteMultipart response content differs from expected value. got %s, expecte %s", i+1, instanceType, + string(actualContent), string(testCase.expectedContent)) } continue } diff --git a/cmd/web-handlers_test.go b/cmd/web-handlers_test.go index d8eb7f598..54e742b37 100644 --- a/cmd/web-handlers_test.go +++ b/cmd/web-handlers_test.go @@ -935,9 +935,16 @@ func testWebHandlerDownloadZip(obj ObjectLayer, instanceType string, t TestErrHa t.Fatalf("%s : %s", instanceType, err) } - obj.PutObject(context.Background(), bucket, "a/one", mustGetPutObjReader(t, strings.NewReader(fileOne), int64(len(fileOne)), "", ""), opts) - obj.PutObject(context.Background(), bucket, "a/b/two", mustGetPutObjReader(t, strings.NewReader(fileTwo), int64(len(fileTwo)), "", ""), opts) - obj.PutObject(context.Background(), bucket, "a/c/three", mustGetPutObjReader(t, strings.NewReader(fileThree), int64(len(fileThree)), "", ""), opts) + for objName, value := range map[string]string{ + "a/one": fileOne, + "a/b/two": fileTwo, + "a/c/three": fileThree, + } { + _, err = obj.PutObject(context.Background(), bucket, objName, mustGetPutObjReader(t, strings.NewReader(value), int64(len(value)), "", ""), opts) + if err != nil { + t.Fatal(err) + } + } test := func(token string) (int, []byte) { rec := httptest.NewRecorder() diff --git a/pkg/disk/stat_windows.go b/pkg/disk/stat_windows.go index d95f1dcc7..f4acc59e9 100644 --- a/pkg/disk/stat_windows.go +++ b/pkg/disk/stat_windows.go @@ -23,16 +23,19 @@ import ( "os" "syscall" "unsafe" + + "golang.org/x/sys/windows" ) var ( - kernel32 = syscall.NewLazyDLL("kernel32.dll") + kernel32 = windows.NewLazySystemDLL("kernel32.dll") // GetDiskFreeSpaceEx - https://msdn.microsoft.com/en-us/library/windows/desktop/aa364937(v=vs.85).aspx // Retrieves information about the amount of space that is available on a disk volume, // which is the total amount of space, the total amount of free space, and the total // amount of free space available to the user that is associated with the calling thread. GetDiskFreeSpaceEx = kernel32.NewProc("GetDiskFreeSpaceExW") + // GetDiskFreeSpace - https://msdn.microsoft.com/en-us/library/windows/desktop/aa364935(v=vs.85).aspx // Retrieves information about the specified disk, including the amount of free space on the disk. GetDiskFreeSpace = kernel32.NewProc("GetDiskFreeSpaceW") @@ -63,10 +66,18 @@ func GetInfo(path string) (info Info, err error) { uintptr(unsafe.Pointer(&lpFreeBytesAvailable)), uintptr(unsafe.Pointer(&lpTotalNumberOfBytes)), uintptr(unsafe.Pointer(&lpTotalNumberOfFreeBytes))) - info = Info{} - info.Total = uint64(lpTotalNumberOfBytes) - info.Free = uint64(lpFreeBytesAvailable) - info.FSType = getFSType(path) + + if uint64(lpTotalNumberOfFreeBytes) > uint64(lpTotalNumberOfBytes) { + return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", + uint64(lpTotalNumberOfFreeBytes), uint64(lpTotalNumberOfBytes), path) + } + + info = Info{ + Total: uint64(lpTotalNumberOfBytes), + Free: uint64(lpTotalNumberOfFreeBytes), + Used: uint64(lpTotalNumberOfBytes) - uint64(lpTotalNumberOfFreeBytes), + FSType: getFSType(path), + } // Return values of GetDiskFreeSpace() lpSectorsPerCluster := uint32(0) @@ -91,9 +102,5 @@ func GetInfo(path string) (info Info, err error) { info.Files = uint64(lpTotalNumberOfClusters) info.Ffree = uint64(lpNumberOfFreeClusters) - if info.Free > info.Total { - return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path) - } - info.Used = info.Total - info.Free return info, nil } diff --git a/pkg/lock/lock_windows.go b/pkg/lock/lock_windows.go index bd7a6c3c0..f02794b56 100644 --- a/pkg/lock/lock_windows.go +++ b/pkg/lock/lock_windows.go @@ -24,10 +24,12 @@ import ( "path/filepath" "syscall" "unsafe" + + "golang.org/x/sys/windows" ) var ( - modkernel32 = syscall.NewLazyDLL("kernel32.dll") + modkernel32 = windows.NewLazySystemDLL("kernel32.dll") procLockFileEx = modkernel32.NewProc("LockFileEx") ) diff --git a/pkg/s3select/internal/parquet-go/gen-go/parquet/parquet-consts.go b/pkg/s3select/internal/parquet-go/gen-go/parquet/parquet-consts.go index 95d12c627..9177df914 100644 --- a/pkg/s3select/internal/parquet-go/gen-go/parquet/parquet-consts.go +++ b/pkg/s3select/internal/parquet-go/gen-go/parquet/parquet-consts.go @@ -6,6 +6,7 @@ package parquet import ( "bytes" "fmt" + "git.apache.org/thrift.git/lib/go/thrift" ) diff --git a/pkg/s3select/internal/parquet-go/gen-go/parquet/parquet.go b/pkg/s3select/internal/parquet-go/gen-go/parquet/parquet.go index 8e4e31820..2d624fbcf 100644 --- a/pkg/s3select/internal/parquet-go/gen-go/parquet/parquet.go +++ b/pkg/s3select/internal/parquet-go/gen-go/parquet/parquet.go @@ -8,6 +8,7 @@ import ( "database/sql/driver" "errors" "fmt" + "git.apache.org/thrift.git/lib/go/thrift" ) diff --git a/pkg/sys/stats_windows.go b/pkg/sys/stats_windows.go index e75612c00..e6ca73922 100644 --- a/pkg/sys/stats_windows.go +++ b/pkg/sys/stats_windows.go @@ -21,10 +21,12 @@ package sys import ( "syscall" "unsafe" + + "golang.org/x/sys/windows" ) var ( - modkernel32 = syscall.NewLazyDLL("kernel32.dll") + modkernel32 = windows.NewLazySystemDLL("kernel32.dll") procGlobalMemoryStatusEx = modkernel32.NewProc("GlobalMemoryStatusEx") )