mirror of
https://github.com/minio/minio.git
synced 2025-01-11 15:03:22 -05:00
add configurable VRF interface and user-timeout (#17108)
This commit is contained in:
parent
90e2cc3d4c
commit
9571b0825e
@ -212,6 +212,7 @@ var (
|
|||||||
globalTLSCerts *certs.Manager
|
globalTLSCerts *certs.Manager
|
||||||
|
|
||||||
globalHTTPServer *xhttp.Server
|
globalHTTPServer *xhttp.Server
|
||||||
|
globalTCPOptions xhttp.TCPOptions
|
||||||
globalHTTPServerErrorCh = make(chan error)
|
globalHTTPServerErrorCh = make(chan error)
|
||||||
globalOSSignalCh = make(chan os.Signal, 1)
|
globalOSSignalCh = make(chan os.Signal, 1)
|
||||||
|
|
||||||
|
13
cmd/net.go
13
cmd/net.go
@ -207,19 +207,6 @@ func isHostIP(ipAddress string) bool {
|
|||||||
return net.ParseIP(host) != nil
|
return net.ParseIP(host) != nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkPortAvailability - check if given host and port is already in use.
|
|
||||||
// Note: The check method tries to listen on given port and closes it.
|
|
||||||
// It is possible to have a disconnected client in this tiny window of time.
|
|
||||||
func checkPortAvailability(host, port string) (err error) {
|
|
||||||
l, err := net.Listen("tcp", net.JoinHostPort(host, port))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// As we are able to listen on this network, the port is not in use.
|
|
||||||
// Close the listener and continue check other networks.
|
|
||||||
return l.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
// extractHostPort - extracts host/port from many address formats
|
// extractHostPort - extracts host/port from many address formats
|
||||||
// such as, ":9000", "localhost:9000", "http://localhost:9000/"
|
// such as, ":9000", "localhost:9000", "http://localhost:9000/"
|
||||||
func extractHostPort(hostAddr string) (string, string, error) {
|
func extractHostPort(hostAddr string) (string, string, error) {
|
||||||
|
@ -20,9 +20,7 @@ package cmd
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net"
|
|
||||||
"reflect"
|
"reflect"
|
||||||
"runtime"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/minio/minio-go/v7/pkg/set"
|
"github.com/minio/minio-go/v7/pkg/set"
|
||||||
@ -180,61 +178,6 @@ func TestGetAPIEndpoints(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ask the kernel for a free open port.
|
|
||||||
func getFreePort() string {
|
|
||||||
addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
l, err := net.ListenTCP("tcp", addr)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
defer l.Close()
|
|
||||||
return fmt.Sprintf("%d", l.Addr().(*net.TCPAddr).Port)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Tests for port availability logic written for server startup sequence.
|
|
||||||
func TestCheckPortAvailability(t *testing.T) {
|
|
||||||
// Make a port is not available.
|
|
||||||
port := getFreePort()
|
|
||||||
listener, err := net.Listen("tcp", net.JoinHostPort("", port))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Unable to listen on port %v", port)
|
|
||||||
}
|
|
||||||
defer listener.Close()
|
|
||||||
|
|
||||||
testCases := []struct {
|
|
||||||
host string
|
|
||||||
port string
|
|
||||||
expectedErr error
|
|
||||||
}{
|
|
||||||
{"", port, fmt.Errorf("listen tcp :%v: bind: address already in use", port)},
|
|
||||||
{"127.0.0.1", port, fmt.Errorf("listen tcp 127.0.0.1:%v: bind: address already in use", port)},
|
|
||||||
{"", getFreePort(), nil},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, testCase := range testCases {
|
|
||||||
// On MS Windows and Mac, skip checking error case due to https://github.com/golang/go/issues/7598
|
|
||||||
if (runtime.GOOS == globalWindowsOSName || runtime.GOOS == globalMacOSName || runtime.GOOS == "solaris") && testCase.expectedErr != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
err := checkPortAvailability(testCase.host, testCase.port)
|
|
||||||
switch {
|
|
||||||
case testCase.expectedErr == nil:
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("error: expected = <nil>, got = %v", err)
|
|
||||||
}
|
|
||||||
case err == nil:
|
|
||||||
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
|
|
||||||
case testCase.expectedErr.Error() != err.Error():
|
|
||||||
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckLocalServerAddr(t *testing.T) {
|
func TestCheckLocalServerAddr(t *testing.T) {
|
||||||
testCases := []struct {
|
testCases := []struct {
|
||||||
serverAddr string
|
serverAddr string
|
||||||
|
@ -107,6 +107,19 @@ var ServerFlags = []cli.Flag{
|
|||||||
Value: 10 * time.Minute,
|
Value: 10 * time.Minute,
|
||||||
EnvVar: "MINIO_CONN_WRITE_DEADLINE",
|
EnvVar: "MINIO_CONN_WRITE_DEADLINE",
|
||||||
},
|
},
|
||||||
|
cli.DurationFlag{
|
||||||
|
Name: "conn-user-timeout",
|
||||||
|
Usage: "custom TCP_USER_TIMEOUT for socket buffers",
|
||||||
|
Hidden: true,
|
||||||
|
Value: 10 * time.Minute,
|
||||||
|
EnvVar: "MINIO_CONN_USER_TIMEOUT",
|
||||||
|
},
|
||||||
|
cli.StringFlag{
|
||||||
|
Name: "interface",
|
||||||
|
Usage: "bind to right VRF device for MinIO services",
|
||||||
|
Hidden: true,
|
||||||
|
EnvVar: "MINIO_INTERFACE",
|
||||||
|
},
|
||||||
cli.StringSliceFlag{
|
cli.StringSliceFlag{
|
||||||
Name: "ftp",
|
Name: "ftp",
|
||||||
Usage: "enable and configure an FTP(Secure) server",
|
Usage: "enable and configure an FTP(Secure) server",
|
||||||
@ -264,11 +277,16 @@ func serverHandleCmdArgs(ctx *cli.Context) {
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
|
globalTCPOptions = xhttp.TCPOptions{
|
||||||
|
UserTimeout: int(ctx.Duration("conn-user-timeout").Milliseconds()),
|
||||||
|
Interface: ctx.String("interface"),
|
||||||
|
}
|
||||||
|
|
||||||
// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
|
// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
|
||||||
// to IPv6 address ie minio will start listening on IPv6 address whereas another
|
// to IPv6 address ie minio will start listening on IPv6 address whereas another
|
||||||
// (non-)minio process is listening on IPv4 of given port.
|
// (non-)minio process is listening on IPv4 of given port.
|
||||||
// To avoid this error situation we check for port availability.
|
// To avoid this error situation we check for port availability.
|
||||||
logger.FatalIf(checkPortAvailability(globalMinioHost, globalMinioPort), "Unable to start the server")
|
logger.FatalIf(xhttp.CheckPortAvailability(globalMinioHost, globalMinioPort, globalTCPOptions), "Unable to start the server")
|
||||||
|
|
||||||
globalIsErasure = (setupType == ErasureSetupType)
|
globalIsErasure = (setupType == ErasureSetupType)
|
||||||
globalIsDistErasure = (setupType == DistErasureSetupType)
|
globalIsDistErasure = (setupType == DistErasureSetupType)
|
||||||
@ -570,7 +588,8 @@ func serverMain(ctx *cli.Context) {
|
|||||||
UseIdleTimeout(ctx.Duration("idle-timeout")).
|
UseIdleTimeout(ctx.Duration("idle-timeout")).
|
||||||
UseReadHeaderTimeout(ctx.Duration("read-header-timeout")).
|
UseReadHeaderTimeout(ctx.Duration("read-header-timeout")).
|
||||||
UseBaseContext(GlobalContext).
|
UseBaseContext(GlobalContext).
|
||||||
UseCustomLogger(log.New(io.Discard, "", 0)) // Turn-off random logging by Go stdlib
|
UseCustomLogger(log.New(io.Discard, "", 0)). // Turn-off random logging by Go stdlib
|
||||||
|
UseTCPOptions(globalTCPOptions)
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
globalHTTPServerErrorCh <- httpServer.Start(GlobalContext)
|
globalHTTPServerErrorCh <- httpServer.Start(GlobalContext)
|
||||||
|
@ -403,7 +403,7 @@ func parseReleaseData(data string) (sha256Sum []byte, releaseTime time.Time, rel
|
|||||||
func getUpdateTransport(timeout time.Duration) http.RoundTripper {
|
func getUpdateTransport(timeout time.Duration) http.RoundTripper {
|
||||||
var updateTransport http.RoundTripper = &http.Transport{
|
var updateTransport http.RoundTripper = &http.Transport{
|
||||||
Proxy: http.ProxyFromEnvironment,
|
Proxy: http.ProxyFromEnvironment,
|
||||||
DialContext: xhttp.NewCustomDialContext(timeout),
|
DialContext: xhttp.NewCustomDialContext(timeout, globalTCPOptions),
|
||||||
IdleConnTimeout: timeout,
|
IdleConnTimeout: timeout,
|
||||||
TLSHandshakeTimeout: timeout,
|
TLSHandshakeTimeout: timeout,
|
||||||
ExpectContinueTimeout: timeout,
|
ExpectContinueTimeout: timeout,
|
||||||
|
@ -574,6 +574,7 @@ func GetDefaultConnSettings() xhttp.ConnSettings {
|
|||||||
DNSCache: globalDNSCache,
|
DNSCache: globalDNSCache,
|
||||||
DialTimeout: rest.DefaultTimeout,
|
DialTimeout: rest.DefaultTimeout,
|
||||||
RootCAs: globalRootCAs,
|
RootCAs: globalRootCAs,
|
||||||
|
TCPOptions: globalTCPOptions,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -587,6 +588,7 @@ func NewInternodeHTTPTransport() func() http.RoundTripper {
|
|||||||
CipherSuites: fips.TLSCiphers(),
|
CipherSuites: fips.TLSCiphers(),
|
||||||
CurvePreferences: fips.TLSCurveIDs(),
|
CurvePreferences: fips.TLSCurveIDs(),
|
||||||
EnableHTTP2: false,
|
EnableHTTP2: false,
|
||||||
|
TCPOptions: globalTCPOptions,
|
||||||
}.NewInternodeHTTPTransport()
|
}.NewInternodeHTTPTransport()
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -600,6 +602,7 @@ func NewCustomHTTPProxyTransport() func() *http.Transport {
|
|||||||
CipherSuites: fips.TLSCiphers(),
|
CipherSuites: fips.TLSCiphers(),
|
||||||
CurvePreferences: fips.TLSCurveIDs(),
|
CurvePreferences: fips.TLSCurveIDs(),
|
||||||
EnableHTTP2: false,
|
EnableHTTP2: false,
|
||||||
|
TCPOptions: globalTCPOptions,
|
||||||
}.NewCustomHTTPProxyTransport()
|
}.NewCustomHTTPProxyTransport()
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -610,6 +613,7 @@ func NewHTTPTransportWithClientCerts(clientCert, clientKey string) *http.Transpo
|
|||||||
DNSCache: globalDNSCache,
|
DNSCache: globalDNSCache,
|
||||||
DialTimeout: defaultDialTimeout,
|
DialTimeout: defaultDialTimeout,
|
||||||
RootCAs: globalRootCAs,
|
RootCAs: globalRootCAs,
|
||||||
|
TCPOptions: globalTCPOptions,
|
||||||
EnableHTTP2: false,
|
EnableHTTP2: false,
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -643,6 +647,7 @@ func NewHTTPTransportWithTimeout(timeout time.Duration) *http.Transport {
|
|||||||
DNSCache: globalDNSCache,
|
DNSCache: globalDNSCache,
|
||||||
DialTimeout: defaultDialTimeout,
|
DialTimeout: defaultDialTimeout,
|
||||||
RootCAs: globalRootCAs,
|
RootCAs: globalRootCAs,
|
||||||
|
TCPOptions: globalTCPOptions,
|
||||||
EnableHTTP2: false,
|
EnableHTTP2: false,
|
||||||
}.NewHTTPTransportWithTimeout(timeout)
|
}.NewHTTPTransportWithTimeout(timeout)
|
||||||
}
|
}
|
||||||
@ -677,6 +682,7 @@ func NewRemoteTargetHTTPTransport() func() *http.Transport {
|
|||||||
DialContext: newCustomDialContext(),
|
DialContext: newCustomDialContext(),
|
||||||
DNSCache: globalDNSCache,
|
DNSCache: globalDNSCache,
|
||||||
RootCAs: globalRootCAs,
|
RootCAs: globalRootCAs,
|
||||||
|
TCPOptions: globalTCPOptions,
|
||||||
EnableHTTP2: false,
|
EnableHTTP2: false,
|
||||||
}.NewRemoteTargetHTTPTransport()
|
}.NewRemoteTargetHTTPTransport()
|
||||||
}
|
}
|
||||||
|
58
internal/http/check_port_linux.go
Normal file
58
internal/http/check_port_linux.go
Normal file
@ -0,0 +1,58 @@
|
|||||||
|
//go:build linux
|
||||||
|
// +build linux
|
||||||
|
|
||||||
|
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package http
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CheckPortAvailability - check if given host and port is already in use.
|
||||||
|
// Note: The check method tries to listen on given port and closes it.
|
||||||
|
// It is possible to have a disconnected client in this tiny window of time.
|
||||||
|
func CheckPortAvailability(host, port string, opts TCPOptions) (err error) {
|
||||||
|
lc := &net.ListenConfig{
|
||||||
|
Control: func(network, address string, c syscall.RawConn) error {
|
||||||
|
c.Control(func(fdPtr uintptr) {
|
||||||
|
if opts.Interface != "" {
|
||||||
|
// When interface is specified look for specifically port availability on
|
||||||
|
// the specified interface if any.
|
||||||
|
_ = syscall.SetsockoptString(int(fdPtr), syscall.SOL_SOCKET, syscall.SO_BINDTODEVICE, opts.Interface)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
l, err := lc.Listen(ctx, "tcp", net.JoinHostPort(host, port))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// As we are able to listen on this network, the port is not in use.
|
||||||
|
// Close the listener and continue check other networks.
|
||||||
|
return l.Close()
|
||||||
|
}
|
@ -1,7 +1,7 @@
|
|||||||
//go:build linux || darwin || dragonfly || freebsd || netbsd || openbsd || rumprun
|
//go:build !linux
|
||||||
// +build linux darwin dragonfly freebsd netbsd openbsd rumprun
|
// +build !linux
|
||||||
|
|
||||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||||
//
|
//
|
||||||
// This file is part of MinIO Object Storage stack
|
// This file is part of MinIO Object Storage stack
|
||||||
//
|
//
|
||||||
@ -21,10 +21,26 @@
|
|||||||
package http
|
package http
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"net"
|
"net"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Unix listener with special TCP options.
|
// CheckPortAvailability - check if given host and port is already in use.
|
||||||
var listenCfg = net.ListenConfig{
|
// Note: The check method tries to listen on given port and closes it.
|
||||||
Control: setTCPParameters,
|
// It is possible to have a disconnected client in this tiny window of time.
|
||||||
|
func CheckPortAvailability(host, port string, opts TCPOptions) (err error) {
|
||||||
|
lc := &net.ListenConfig{}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
l, err := lc.Listen(ctx, "tcp", net.JoinHostPort(host, port))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// As we are able to listen on this network, the port is not in use.
|
||||||
|
// Close the listener and continue check other networks.
|
||||||
|
return l.Close()
|
||||||
}
|
}
|
64
internal/http/check_port_test.go
Normal file
64
internal/http/check_port_test.go
Normal file
@ -0,0 +1,64 @@
|
|||||||
|
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package http
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"runtime"
|
||||||
|
"strconv"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Tests for port availability logic written for server startup sequence.
|
||||||
|
func TestCheckPortAvailability(t *testing.T) {
|
||||||
|
if runtime.GOOS != "linux" {
|
||||||
|
t.Skip()
|
||||||
|
}
|
||||||
|
|
||||||
|
l, err := net.Listen("tcp", "localhost:0") // ask kernel for a free port.
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer l.Close()
|
||||||
|
|
||||||
|
port := l.Addr().(*net.TCPAddr).Port
|
||||||
|
|
||||||
|
testCases := []struct {
|
||||||
|
host string
|
||||||
|
port int
|
||||||
|
expectedErr error
|
||||||
|
}{
|
||||||
|
{"", port, fmt.Errorf("listen tcp :%v: bind: address already in use", port)},
|
||||||
|
{"127.0.0.1", port, fmt.Errorf("listen tcp 127.0.0.1:%v: bind: address already in use", port)},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, testCase := range testCases {
|
||||||
|
err := CheckPortAvailability(testCase.host, strconv.Itoa(testCase.port), TCPOptions{})
|
||||||
|
switch {
|
||||||
|
case testCase.expectedErr == nil:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("error: expected = <nil>, got = %v", err)
|
||||||
|
}
|
||||||
|
case err == nil:
|
||||||
|
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
|
||||||
|
case testCase.expectedErr.Error() != err.Error():
|
||||||
|
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -1,7 +1,7 @@
|
|||||||
//go:build linux
|
//go:build linux
|
||||||
// +build linux
|
// +build linux
|
||||||
|
|
||||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||||
//
|
//
|
||||||
// This file is part of MinIO Object Storage stack
|
// This file is part of MinIO Object Storage stack
|
||||||
//
|
//
|
||||||
@ -29,71 +29,81 @@ import (
|
|||||||
"golang.org/x/sys/unix"
|
"golang.org/x/sys/unix"
|
||||||
)
|
)
|
||||||
|
|
||||||
func setTCPParameters(network, address string, c syscall.RawConn) error {
|
func setTCPParametersFn(opts TCPOptions) func(network, address string, c syscall.RawConn) error {
|
||||||
c.Control(func(fdPtr uintptr) {
|
return func(network, address string, c syscall.RawConn) error {
|
||||||
// got socket file descriptor to set parameters.
|
c.Control(func(fdPtr uintptr) {
|
||||||
fd := int(fdPtr)
|
// got socket file descriptor to set parameters.
|
||||||
|
fd := int(fdPtr)
|
||||||
|
|
||||||
_ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEADDR, 1)
|
_ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEADDR, 1)
|
||||||
|
|
||||||
_ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEPORT, 1)
|
_ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEPORT, 1)
|
||||||
|
|
||||||
// Enable TCP open
|
// Enable TCP open
|
||||||
// https://lwn.net/Articles/508865/ - 16k queue size.
|
// https://lwn.net/Articles/508865/ - 16k queue size.
|
||||||
_ = syscall.SetsockoptInt(fd, syscall.SOL_TCP, unix.TCP_FASTOPEN, 16*1024)
|
_ = syscall.SetsockoptInt(fd, syscall.SOL_TCP, unix.TCP_FASTOPEN, 16*1024)
|
||||||
|
|
||||||
// Enable TCP fast connect
|
// Enable TCP fast connect
|
||||||
// TCPFastOpenConnect sets the underlying socket to use
|
// TCPFastOpenConnect sets the underlying socket to use
|
||||||
// the TCP fast open connect. This feature is supported
|
// the TCP fast open connect. This feature is supported
|
||||||
// since Linux 4.11.
|
// since Linux 4.11.
|
||||||
_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, unix.TCP_FASTOPEN_CONNECT, 1)
|
_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, unix.TCP_FASTOPEN_CONNECT, 1)
|
||||||
|
|
||||||
// Enable TCP quick ACK, John Nagle says
|
// Enable TCP quick ACK, John Nagle says
|
||||||
// "Set TCP_QUICKACK. If you find a case where that makes things worse, let me know."
|
// "Set TCP_QUICKACK. If you find a case where that makes things worse, let me know."
|
||||||
_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, unix.TCP_QUICKACK, 1)
|
_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, unix.TCP_QUICKACK, 1)
|
||||||
|
|
||||||
// The time (in seconds) the connection needs to remain idle before
|
// The time (in seconds) the connection needs to remain idle before
|
||||||
// TCP starts sending keepalive probes
|
// TCP starts sending keepalive probes
|
||||||
_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPIDLE, 15)
|
_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPIDLE, 15)
|
||||||
|
|
||||||
// Number of probes.
|
// Number of probes.
|
||||||
// ~ cat /proc/sys/net/ipv4/tcp_keepalive_probes (defaults to 9, we reduce it to 5)
|
// ~ cat /proc/sys/net/ipv4/tcp_keepalive_probes (defaults to 9, we reduce it to 5)
|
||||||
_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPCNT, 5)
|
_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPCNT, 5)
|
||||||
|
|
||||||
// Wait time after successful probe in seconds.
|
// Wait time after successful probe in seconds.
|
||||||
// ~ cat /proc/sys/net/ipv4/tcp_keepalive_intvl (defaults to 75 secs, we reduce it to 15 secs)
|
// ~ cat /proc/sys/net/ipv4/tcp_keepalive_intvl (defaults to 75 secs, we reduce it to 15 secs)
|
||||||
_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, 15)
|
_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, 15)
|
||||||
|
|
||||||
// Set tcp user timeout in addition to the keep-alive - tcp-keepalive is not enough to close a socket
|
// Set tcp user timeout in addition to the keep-alive - tcp-keepalive is not enough to close a socket
|
||||||
// with dead end because tcp-keepalive is not fired when there is data in the socket buffer.
|
// with dead end because tcp-keepalive is not fired when there is data in the socket buffer.
|
||||||
// https://blog.cloudflare.com/when-tcp-sockets-refuse-to-die/
|
// https://blog.cloudflare.com/when-tcp-sockets-refuse-to-die/
|
||||||
// This is a sensitive configuration, it is better to set it to high values, > 60 secs since it can
|
// This is a sensitive configuration, it is better to set it to high values, > 60 secs since it can
|
||||||
// affect clients reading data with a very slow pace (disappropriate with socket buffer sizes)
|
// affect clients reading data with a very slow pace (disappropriate with socket buffer sizes)
|
||||||
_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT, 10*60*1000)
|
_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT, opts.UserTimeout)
|
||||||
})
|
|
||||||
return nil
|
if opts.Interface != "" {
|
||||||
|
// Create socket on specific vrf device.
|
||||||
|
// To catch all kinds of special cases this filters specifically for loopback networks.
|
||||||
|
if ip := net.ParseIP(address); ip != nil && !ip.IsLoopback() {
|
||||||
|
_ = syscall.SetsockoptString(fd, syscall.SOL_SOCKET, syscall.SO_BINDTODEVICE, opts.Interface)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// DialContext is a function to make custom Dial for internode communications
|
// DialContext is a function to make custom Dial for internode communications
|
||||||
type DialContext func(ctx context.Context, network, address string) (net.Conn, error)
|
type DialContext func(ctx context.Context, network, address string) (net.Conn, error)
|
||||||
|
|
||||||
// NewInternodeDialContext setups a custom dialer for internode communication
|
// NewInternodeDialContext setups a custom dialer for internode communication
|
||||||
func NewInternodeDialContext(dialTimeout time.Duration) DialContext {
|
func NewInternodeDialContext(dialTimeout time.Duration, opts TCPOptions) DialContext {
|
||||||
return func(ctx context.Context, network, addr string) (net.Conn, error) {
|
return func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||||
dialer := &net.Dialer{
|
dialer := &net.Dialer{
|
||||||
Timeout: dialTimeout,
|
Timeout: dialTimeout,
|
||||||
Control: setTCPParameters,
|
Control: setTCPParametersFn(opts),
|
||||||
}
|
}
|
||||||
return dialer.DialContext(ctx, network, addr)
|
return dialer.DialContext(ctx, network, addr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewCustomDialContext setups a custom dialer for any external communication and proxies.
|
// NewCustomDialContext setups a custom dialer for any external communication and proxies.
|
||||||
func NewCustomDialContext(dialTimeout time.Duration) DialContext {
|
func NewCustomDialContext(dialTimeout time.Duration, opts TCPOptions) DialContext {
|
||||||
return func(ctx context.Context, network, addr string) (net.Conn, error) {
|
return func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||||
dialer := &net.Dialer{
|
dialer := &net.Dialer{
|
||||||
Timeout: dialTimeout,
|
Timeout: dialTimeout,
|
||||||
Control: setTCPParameters,
|
Control: setTCPParametersFn(opts),
|
||||||
}
|
}
|
||||||
return dialer.DialContext(ctx, network, addr)
|
return dialer.DialContext(ctx, network, addr)
|
||||||
}
|
}
|
||||||
|
@ -30,8 +30,10 @@ import (
|
|||||||
// TODO: if possible implement for non-linux platforms, not a priority at the moment
|
// TODO: if possible implement for non-linux platforms, not a priority at the moment
|
||||||
//
|
//
|
||||||
//nolint:unused
|
//nolint:unused
|
||||||
func setTCPParameters(string, string, syscall.RawConn) error {
|
func setTCPParametersFn(opts TCPOptions) func(network, address string, c syscall.RawConn) error {
|
||||||
return nil
|
return func(network, address string, c syscall.RawConn) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// DialContext is a function to make custom Dial for internode communications
|
// DialContext is a function to make custom Dial for internode communications
|
||||||
@ -41,7 +43,7 @@ type DialContext func(ctx context.Context, network, address string) (net.Conn, e
|
|||||||
var NewInternodeDialContext = NewCustomDialContext
|
var NewInternodeDialContext = NewCustomDialContext
|
||||||
|
|
||||||
// NewCustomDialContext configures a custom dialer for internode communications
|
// NewCustomDialContext configures a custom dialer for internode communications
|
||||||
func NewCustomDialContext(dialTimeout time.Duration) DialContext {
|
func NewCustomDialContext(dialTimeout time.Duration, _ TCPOptions) DialContext {
|
||||||
return func(ctx context.Context, network, addr string) (net.Conn, error) {
|
return func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||||
dialer := &net.Dialer{
|
dialer := &net.Dialer{
|
||||||
Timeout: dialTimeout,
|
Timeout: dialTimeout,
|
||||||
|
@ -1,26 +0,0 @@
|
|||||||
//go:build windows || plan9 || solaris
|
|
||||||
// +build windows plan9 solaris
|
|
||||||
|
|
||||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
|
||||||
//
|
|
||||||
// This file is part of MinIO Object Storage stack
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
|
||||||
// the Free Software Foundation, either version 3 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
package http
|
|
||||||
|
|
||||||
import "net"
|
|
||||||
|
|
||||||
// Windows, plan9 specific listener.
|
|
||||||
var listenCfg = net.ListenConfig{}
|
|
@ -118,11 +118,17 @@ func (listener *httpListener) Addrs() (addrs []net.Addr) {
|
|||||||
return addrs
|
return addrs
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TCPOptions specify customizable TCP optimizations on raw socket
|
||||||
|
type TCPOptions struct {
|
||||||
|
UserTimeout int // this value is expected to be in milliseconds
|
||||||
|
Interface string // this is a VRF device passed via `--interface` flag
|
||||||
|
}
|
||||||
|
|
||||||
// newHTTPListener - creates new httpListener object which is interface compatible to net.Listener.
|
// newHTTPListener - creates new httpListener object which is interface compatible to net.Listener.
|
||||||
// httpListener is capable to
|
// httpListener is capable to
|
||||||
// * listen to multiple addresses
|
// * listen to multiple addresses
|
||||||
// * controls incoming connections only doing HTTP protocol
|
// * controls incoming connections only doing HTTP protocol
|
||||||
func newHTTPListener(ctx context.Context, serverAddrs []string) (listener *httpListener, err error) {
|
func newHTTPListener(ctx context.Context, serverAddrs []string, opts TCPOptions) (listener *httpListener, err error) {
|
||||||
var tcpListeners []*net.TCPListener
|
var tcpListeners []*net.TCPListener
|
||||||
|
|
||||||
// Close all opened listeners on error
|
// Close all opened listeners on error
|
||||||
@ -147,11 +153,16 @@ func newHTTPListener(ctx context.Context, serverAddrs []string) (listener *httpL
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Silently ignore failure to bind on DNS cached ipv6 loopback iff user specifies "localhost"
|
// Unix listener with special TCP options.
|
||||||
|
listenCfg := net.ListenConfig{
|
||||||
|
Control: setTCPParametersFn(opts),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Silently ignore failure to bind on DNS cached ipv6 loopback if user specifies "localhost"
|
||||||
for _, serverAddr := range serverAddrs {
|
for _, serverAddr := range serverAddrs {
|
||||||
var l net.Listener
|
var l net.Listener
|
||||||
if l, err = listenCfg.Listen(ctx, "tcp", serverAddr); err != nil {
|
if l, err = listenCfg.Listen(ctx, "tcp", serverAddr); err != nil {
|
||||||
if isLocalhost && strings.HasPrefix(serverAddr, "[::1]") {
|
if isLocalhost && strings.HasPrefix(serverAddr, "[::1") {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
return nil, err
|
return nil, err
|
||||||
@ -160,7 +171,7 @@ func newHTTPListener(ctx context.Context, serverAddrs []string) (listener *httpL
|
|||||||
tcpListener, ok := l.(*net.TCPListener)
|
tcpListener, ok := l.(*net.TCPListener)
|
||||||
if !ok {
|
if !ok {
|
||||||
err = fmt.Errorf("unexpected listener type found %v, expected net.TCPListener", l)
|
err = fmt.Errorf("unexpected listener type found %v, expected net.TCPListener", l)
|
||||||
if isLocalhost && strings.HasPrefix(serverAddr, "[::1]") {
|
if isLocalhost && strings.HasPrefix(serverAddr, "[::1") {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -153,6 +153,7 @@ func TestNewHTTPListener(t *testing.T) {
|
|||||||
for _, testCase := range testCases {
|
for _, testCase := range testCases {
|
||||||
listener, err := newHTTPListener(context.Background(),
|
listener, err := newHTTPListener(context.Background(),
|
||||||
testCase.serverAddrs,
|
testCase.serverAddrs,
|
||||||
|
TCPOptions{},
|
||||||
)
|
)
|
||||||
if !testCase.expectedErr {
|
if !testCase.expectedErr {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -189,6 +190,7 @@ func TestHTTPListenerStartClose(t *testing.T) {
|
|||||||
for i, testCase := range testCases {
|
for i, testCase := range testCases {
|
||||||
listener, err := newHTTPListener(context.Background(),
|
listener, err := newHTTPListener(context.Background(),
|
||||||
testCase.serverAddrs,
|
testCase.serverAddrs,
|
||||||
|
TCPOptions{},
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if strings.Contains(err.Error(), "The requested address is not valid in its context") {
|
if strings.Contains(err.Error(), "The requested address is not valid in its context") {
|
||||||
@ -239,6 +241,7 @@ func TestHTTPListenerAddr(t *testing.T) {
|
|||||||
for i, testCase := range testCases {
|
for i, testCase := range testCases {
|
||||||
listener, err := newHTTPListener(context.Background(),
|
listener, err := newHTTPListener(context.Background(),
|
||||||
testCase.serverAddrs,
|
testCase.serverAddrs,
|
||||||
|
TCPOptions{},
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if strings.Contains(err.Error(), "The requested address is not valid in its context") {
|
if strings.Contains(err.Error(), "The requested address is not valid in its context") {
|
||||||
@ -286,6 +289,7 @@ func TestHTTPListenerAddrs(t *testing.T) {
|
|||||||
for i, testCase := range testCases {
|
for i, testCase := range testCases {
|
||||||
listener, err := newHTTPListener(context.Background(),
|
listener, err := newHTTPListener(context.Background(),
|
||||||
testCase.serverAddrs,
|
testCase.serverAddrs,
|
||||||
|
TCPOptions{},
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if strings.Contains(err.Error(), "The requested address is not valid in its context") {
|
if strings.Contains(err.Error(), "The requested address is not valid in its context") {
|
||||||
|
@ -61,6 +61,7 @@ const (
|
|||||||
type Server struct {
|
type Server struct {
|
||||||
http.Server
|
http.Server
|
||||||
Addrs []string // addresses on which the server listens for new connection.
|
Addrs []string // addresses on which the server listens for new connection.
|
||||||
|
TCPOptions TCPOptions // all the configurable TCP conn specific configurable options.
|
||||||
ShutdownTimeout time.Duration // timeout used for graceful server shutdown.
|
ShutdownTimeout time.Duration // timeout used for graceful server shutdown.
|
||||||
listenerMutex sync.Mutex // to guard 'listener' field.
|
listenerMutex sync.Mutex // to guard 'listener' field.
|
||||||
listener *httpListener // HTTP listener for all 'Addrs' field.
|
listener *httpListener // HTTP listener for all 'Addrs' field.
|
||||||
@ -87,6 +88,7 @@ func (srv *Server) Start(ctx context.Context) (err error) {
|
|||||||
listener, err = newHTTPListener(
|
listener, err = newHTTPListener(
|
||||||
ctx,
|
ctx,
|
||||||
srv.Addrs,
|
srv.Addrs,
|
||||||
|
srv.TCPOptions,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@ -213,6 +215,12 @@ func (srv *Server) UseCustomLogger(l *log.Logger) *Server {
|
|||||||
return srv
|
return srv
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// UseTCPOptions use custom TCP options on raw socket
|
||||||
|
func (srv *Server) UseTCPOptions(opts TCPOptions) *Server {
|
||||||
|
srv.TCPOptions = opts
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
// NewServer - creates new HTTP server using given arguments.
|
// NewServer - creates new HTTP server using given arguments.
|
||||||
func NewServer(addrs []string) *Server {
|
func NewServer(addrs []string) *Server {
|
||||||
httpServer := &Server{
|
httpServer := &Server{
|
||||||
|
@ -49,12 +49,15 @@ type ConnSettings struct {
|
|||||||
|
|
||||||
// HTTP2
|
// HTTP2
|
||||||
EnableHTTP2 bool
|
EnableHTTP2 bool
|
||||||
|
|
||||||
|
// TCP Options
|
||||||
|
TCPOptions TCPOptions
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s ConnSettings) getDefaultTransport() *http.Transport {
|
func (s ConnSettings) getDefaultTransport() *http.Transport {
|
||||||
dialContext := s.DialContext
|
dialContext := s.DialContext
|
||||||
if dialContext == nil {
|
if dialContext == nil {
|
||||||
dialContext = DialContextWithDNSCache(s.DNSCache, NewInternodeDialContext(s.DialTimeout))
|
dialContext = DialContextWithDNSCache(s.DNSCache, NewInternodeDialContext(s.DialTimeout, s.TCPOptions))
|
||||||
}
|
}
|
||||||
|
|
||||||
tlsClientConfig := tls.Config{
|
tlsClientConfig := tls.Config{
|
||||||
|
Loading…
Reference in New Issue
Block a user