rename all remaining packages to internal/ (#12418)

This is to ensure that there are no projects
that try to import `minio/minio/pkg` into
their own repo. Any such common packages should
go to `https://github.com/minio/pkg`

internal/net/health.go

@@ -0,0 +1,97 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package net
+
+import (
+	"github.com/minio/madmin-go"
+	"github.com/montanaflynn/stats"
+)
+
+// ComputePerfStats takes arrays of Latency & Throughput to compute Statistics
+func ComputePerfStats(latencies, throughputs []float64) (madmin.NetLatency, madmin.NetThroughput, error) {
+	var avgLatency float64
+	var percentile50Latency float64
+	var percentile90Latency float64
+	var percentile99Latency float64
+	var minLatency float64
+	var maxLatency float64
+
+	var avgThroughput float64
+	var percentile50Throughput float64
+	var percentile90Throughput float64
+	var percentile99Throughput float64
+	var minThroughput float64
+	var maxThroughput float64
+	var err error
+
+	if avgLatency, err = stats.Mean(latencies); err != nil {
+		return madmin.NetLatency{}, madmin.NetThroughput{}, err
+	}
+	if percentile50Latency, err = stats.Percentile(latencies, 50); err != nil {
+		return madmin.NetLatency{}, madmin.NetThroughput{}, err
+	}
+	if percentile90Latency, err = stats.Percentile(latencies, 90); err != nil {
+		return madmin.NetLatency{}, madmin.NetThroughput{}, err
+	}
+	if percentile99Latency, err = stats.Percentile(latencies, 99); err != nil {
+		return madmin.NetLatency{}, madmin.NetThroughput{}, err
+	}
+	if maxLatency, err = stats.Max(latencies); err != nil {
+		return madmin.NetLatency{}, madmin.NetThroughput{}, err
+	}
+	if minLatency, err = stats.Min(latencies); err != nil {
+		return madmin.NetLatency{}, madmin.NetThroughput{}, err
+	}
+	l := madmin.NetLatency{
+		Avg:          avgLatency,
+		Percentile50: percentile50Latency,
+		Percentile90: percentile90Latency,
+		Percentile99: percentile99Latency,
+		Min:          minLatency,
+		Max:          maxLatency,
+	}
+
+	if avgThroughput, err = stats.Mean(throughputs); err != nil {
+		return madmin.NetLatency{}, madmin.NetThroughput{}, err
+	}
+	if percentile50Throughput, err = stats.Percentile(throughputs, 50); err != nil {
+		return madmin.NetLatency{}, madmin.NetThroughput{}, err
+	}
+	if percentile90Throughput, err = stats.Percentile(throughputs, 90); err != nil {
+		return madmin.NetLatency{}, madmin.NetThroughput{}, err
+	}
+	if percentile99Throughput, err = stats.Percentile(throughputs, 99); err != nil {
+		return madmin.NetLatency{}, madmin.NetThroughput{}, err
+	}
+	if maxThroughput, err = stats.Max(throughputs); err != nil {
+		return madmin.NetLatency{}, madmin.NetThroughput{}, err
+	}
+	if minThroughput, err = stats.Min(throughputs); err != nil {
+		return madmin.NetLatency{}, madmin.NetThroughput{}, err
+	}
+	t := madmin.NetThroughput{
+		Avg:          avgThroughput,
+		Percentile50: percentile50Throughput,
+		Percentile90: percentile90Throughput,
+		Percentile99: percentile99Throughput,
+		Min:          minThroughput,
+		Max:          maxThroughput,
+	}
+
+	return l, t, nil
+}

internal/net/host.go

@@ -0,0 +1,171 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package net
+
+import (
+	"encoding/json"
+	"errors"
+	"net"
+	"regexp"
+	"strings"
+)
+
+var hostLabelRegexp = regexp.MustCompile("^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$")
+
+// Host - holds network host IP/name and its port.
+type Host struct {
+	Name      string
+	Port      Port
+	IsPortSet bool
+}
+
+// IsEmpty - returns whether Host is empty or not
+func (host Host) IsEmpty() bool {
+	return host.Name == ""
+}
+
+// String - returns string representation of Host.
+func (host Host) String() string {
+	if !host.IsPortSet {
+		return host.Name
+	}
+
+	return net.JoinHostPort(host.Name, host.Port.String())
+}
+
+// Equal - checks whether given host is equal or not.
+func (host Host) Equal(compHost Host) bool {
+	return host.String() == compHost.String()
+}
+
+// MarshalJSON - converts Host into JSON data
+func (host Host) MarshalJSON() ([]byte, error) {
+	return json.Marshal(host.String())
+}
+
+// UnmarshalJSON - parses data into Host.
+func (host *Host) UnmarshalJSON(data []byte) (err error) {
+	var s string
+	if err = json.Unmarshal(data, &s); err != nil {
+		return err
+	}
+
+	// Allow empty string
+	if s == "" {
+		*host = Host{}
+		return nil
+	}
+
+	var h *Host
+	if h, err = ParseHost(s); err != nil {
+		return err
+	}
+
+	*host = *h
+	return nil
+}
+
+// ParseHost - parses string into Host
+func ParseHost(s string) (*Host, error) {
+	if s == "" {
+		return nil, errors.New("invalid argument")
+	}
+	isValidHost := func(host string) bool {
+		if host == "" {
+			return true
+		}
+
+		if ip := net.ParseIP(host); ip != nil {
+			return true
+		}
+
+		// host is not a valid IPv4 or IPv6 address
+		// host may be a hostname
+		// refer https://en.wikipedia.org/wiki/Hostname#Restrictions_on_valid_host_names
+		// why checks are done like below
+		if len(host) < 1 || len(host) > 253 {
+			return false
+		}
+
+		for _, label := range strings.Split(host, ".") {
+			if len(label) < 1 || len(label) > 63 {
+				return false
+			}
+
+			if !hostLabelRegexp.MatchString(label) {
+				return false
+			}
+		}
+
+		return true
+	}
+
+	var port Port
+	var isPortSet bool
+	host, portStr, err := net.SplitHostPort(s)
+	if err != nil {
+		if !strings.Contains(err.Error(), "missing port in address") {
+			return nil, err
+		}
+		host = s
+	} else {
+		if port, err = ParsePort(portStr); err != nil {
+			return nil, err
+		}
+
+		isPortSet = true
+	}
+
+	if host != "" {
+		host, err = trimIPv6(host)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// IPv6 requires a link-local address on every network interface.
+	// `%interface` should be preserved.
+	trimmedHost := host
+
+	if i := strings.LastIndex(trimmedHost, "%"); i > -1 {
+		// `%interface` can be skipped for validity check though.
+		trimmedHost = trimmedHost[:i]
+	}
+
+	if !isValidHost(trimmedHost) {
+		return nil, errors.New("invalid hostname")
+	}
+
+	return &Host{
+		Name:      host,
+		Port:      port,
+		IsPortSet: isPortSet,
+	}, nil
+}
+
+// IPv6 can be embedded with square brackets.
+func trimIPv6(host string) (string, error) {
+	// `missing ']' in host` error is already handled in `SplitHostPort`
+	if host[len(host)-1] == ']' {
+		if host[0] != '[' {
+			return "", errors.New("missing '[' in host")
+		}
+		return host[1:][:len(host)-2], nil
+	}
+	return host, nil
+}

internal/net/host_test.go

@@ -0,0 +1,251 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package net
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestHostIsEmpty(t *testing.T) {
+	testCases := []struct {
+		host           Host
+		expectedResult bool
+	}{
+		{Host{"", 0, false}, true},
+		{Host{"", 0, true}, true},
+		{Host{"play", 9000, false}, false},
+		{Host{"play", 9000, true}, false},
+	}
+
+	for i, testCase := range testCases {
+		result := testCase.host.IsEmpty()
+
+		if result != testCase.expectedResult {
+			t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result)
+		}
+	}
+}
+
+func TestHostString(t *testing.T) {
+	testCases := []struct {
+		host        Host
+		expectedStr string
+	}{
+		{Host{"", 0, false}, ""},
+		{Host{"", 0, true}, ":0"},
+		{Host{"play", 9000, false}, "play"},
+		{Host{"play", 9000, true}, "play:9000"},
+	}
+
+	for i, testCase := range testCases {
+		str := testCase.host.String()
+
+		if str != testCase.expectedStr {
+			t.Fatalf("test %v: string: expected: %v, got: %v", i+1, testCase.expectedStr, str)
+		}
+	}
+}
+
+func TestHostEqual(t *testing.T) {
+	testCases := []struct {
+		host           Host
+		compHost       Host
+		expectedResult bool
+	}{
+		{Host{"", 0, false}, Host{"", 0, true}, false},
+		{Host{"play", 9000, true}, Host{"play", 9000, false}, false},
+		{Host{"", 0, true}, Host{"", 0, true}, true},
+		{Host{"play", 9000, false}, Host{"play", 9000, false}, true},
+		{Host{"play", 9000, true}, Host{"play", 9000, true}, true},
+	}
+
+	for i, testCase := range testCases {
+		result := testCase.host.Equal(testCase.compHost)
+
+		if result != testCase.expectedResult {
+			t.Fatalf("test %v: string: expected: %v, got: %v", i+1, testCase.expectedResult, result)
+		}
+	}
+}
+
+func TestHostMarshalJSON(t *testing.T) {
+	testCases := []struct {
+		host         Host
+		expectedData []byte
+		expectErr    bool
+	}{
+		{Host{}, []byte(`""`), false},
+		{Host{"play", 0, false}, []byte(`"play"`), false},
+		{Host{"play", 0, true}, []byte(`"play:0"`), false},
+		{Host{"play", 9000, true}, []byte(`"play:9000"`), false},
+		{Host{"play.min.io", 0, false}, []byte(`"play.min.io"`), false},
+		{Host{"play.min.io", 9000, true}, []byte(`"play.min.io:9000"`), false},
+		{Host{"147.75.201.93", 0, false}, []byte(`"147.75.201.93"`), false},
+		{Host{"147.75.201.93", 9000, true}, []byte(`"147.75.201.93:9000"`), false},
+		{Host{"play12", 0, false}, []byte(`"play12"`), false},
+		{Host{"12play", 0, false}, []byte(`"12play"`), false},
+		{Host{"play-minio-io", 0, false}, []byte(`"play-minio-io"`), false},
+		{Host{"play--min.io", 0, false}, []byte(`"play--min.io"`), false},
+	}
+
+	for i, testCase := range testCases {
+		data, err := testCase.host.MarshalJSON()
+		expectErr := (err != nil)
+
+		if expectErr != testCase.expectErr {
+			t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
+		}
+
+		if !testCase.expectErr {
+			if !reflect.DeepEqual(data, testCase.expectedData) {
+				t.Fatalf("test %v: data: expected: %v, got: %v", i+1, string(testCase.expectedData), string(data))
+			}
+		}
+	}
+}
+
+func TestHostUnmarshalJSON(t *testing.T) {
+	testCases := []struct {
+		data         []byte
+		expectedHost *Host
+		expectErr    bool
+	}{
+		{[]byte(`""`), &Host{}, false},
+		{[]byte(`"play"`), &Host{"play", 0, false}, false},
+		{[]byte(`"play:0"`), &Host{"play", 0, true}, false},
+		{[]byte(`"play:9000"`), &Host{"play", 9000, true}, false},
+		{[]byte(`"play.min.io"`), &Host{"play.min.io", 0, false}, false},
+		{[]byte(`"play.min.io:9000"`), &Host{"play.min.io", 9000, true}, false},
+		{[]byte(`"147.75.201.93"`), &Host{"147.75.201.93", 0, false}, false},
+		{[]byte(`"147.75.201.93:9000"`), &Host{"147.75.201.93", 9000, true}, false},
+		{[]byte(`"play12"`), &Host{"play12", 0, false}, false},
+		{[]byte(`"12play"`), &Host{"12play", 0, false}, false},
+		{[]byte(`"play-minio-io"`), &Host{"play-minio-io", 0, false}, false},
+		{[]byte(`"play--min.io"`), &Host{"play--min.io", 0, false}, false},
+		{[]byte(`":9000"`), &Host{"", 9000, true}, false},
+		{[]byte(`"[fe80::8097:76eb:b397:e067%wlp2s0]"`), &Host{"fe80::8097:76eb:b397:e067%wlp2s0", 0, false}, false},
+		{[]byte(`"[fe80::8097:76eb:b397:e067]:9000"`), &Host{"fe80::8097:76eb:b397:e067", 9000, true}, false},
+		{[]byte(`"fe80::8097:76eb:b397:e067%wlp2s0"`), nil, true},
+		{[]byte(`"fe80::8097:76eb:b397:e067%wlp2s0]"`), nil, true},
+		{[]byte(`"[fe80::8097:76eb:b397:e067%wlp2s0"`), nil, true},
+		{[]byte(`"[[fe80::8097:76eb:b397:e067%wlp2s0]]"`), nil, true},
+		{[]byte(`"[[fe80::8097:76eb:b397:e067%wlp2s0"`), nil, true},
+		{[]byte(`"play:"`), nil, true},
+		{[]byte(`"play::"`), nil, true},
+		{[]byte(`"play:90000"`), nil, true},
+		{[]byte(`"play:-10"`), nil, true},
+		{[]byte(`"play-"`), nil, true},
+		{[]byte(`"play.minio..io"`), nil, true},
+		{[]byte(`":"`), nil, true},
+	}
+
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run("", func(t *testing.T) {
+			var host Host
+			err := host.UnmarshalJSON(testCase.data)
+			expectErr := (err != nil)
+
+			if expectErr != testCase.expectErr {
+				t.Errorf("error: expected: %v, got: %v", testCase.expectErr, expectErr)
+			}
+
+			if !testCase.expectErr {
+				if !reflect.DeepEqual(&host, testCase.expectedHost) {
+					t.Errorf("host: expected: %#v, got: %#v", testCase.expectedHost, host)
+				}
+			}
+		})
+	}
+}
+
+func TestParseHost(t *testing.T) {
+	testCases := []struct {
+		s            string
+		expectedHost *Host
+		expectErr    bool
+	}{
+		{"play", &Host{"play", 0, false}, false},
+		{"play:0", &Host{"play", 0, true}, false},
+		{"play:9000", &Host{"play", 9000, true}, false},
+		{"play.min.io", &Host{"play.min.io", 0, false}, false},
+		{"play.min.io:9000", &Host{"play.min.io", 9000, true}, false},
+		{"147.75.201.93", &Host{"147.75.201.93", 0, false}, false},
+		{"147.75.201.93:9000", &Host{"147.75.201.93", 9000, true}, false},
+		{"play12", &Host{"play12", 0, false}, false},
+		{"12play", &Host{"12play", 0, false}, false},
+		{"play-minio-io", &Host{"play-minio-io", 0, false}, false},
+		{"play--min.io", &Host{"play--min.io", 0, false}, false},
+		{":9000", &Host{"", 9000, true}, false},
+		{"play:", nil, true},
+		{"play::", nil, true},
+		{"play:90000", nil, true},
+		{"play:-10", nil, true},
+		{"play-", nil, true},
+		{"play.minio..io", nil, true},
+		{":", nil, true},
+		{"", nil, true},
+	}
+
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run("", func(t *testing.T) {
+			host, err := ParseHost(testCase.s)
+			expectErr := (err != nil)
+
+			if expectErr != testCase.expectErr {
+				t.Errorf("error: expected: %v, got: %v", testCase.expectErr, expectErr)
+			}
+
+			if !testCase.expectErr {
+				if !reflect.DeepEqual(host, testCase.expectedHost) {
+					t.Errorf("host: expected: %#v, got: %#v", testCase.expectedHost, host)
+				}
+			}
+		})
+	}
+}
+
+func TestTrimIPv6(t *testing.T) {
+	testCases := []struct {
+		IP         string
+		expectedIP string
+		expectErr  bool
+	}{
+		{"[fe80::8097:76eb:b397:e067%wlp2s0]", "fe80::8097:76eb:b397:e067%wlp2s0", false},
+		{"fe80::8097:76eb:b397:e067%wlp2s0]", "fe80::8097:76eb:b397:e067%wlp2s0", true},
+		{"[fe80::8097:76eb:b397:e067%wlp2s0]]", "fe80::8097:76eb:b397:e067%wlp2s0]", false},
+		{"[[fe80::8097:76eb:b397:e067%wlp2s0]]", "[fe80::8097:76eb:b397:e067%wlp2s0]", false},
+	}
+
+	for i, testCase := range testCases {
+		ip, err := trimIPv6(testCase.IP)
+		expectErr := (err != nil)
+
+		if expectErr != testCase.expectErr {
+			t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
+		}
+
+		if !testCase.expectErr {
+			if ip != testCase.expectedIP {
+				t.Fatalf("test %v: IP: expected: %#v, got: %#v", i+1, testCase.expectedIP, ip)
+			}
+		}
+	}
+}

internal/net/port.go

@@ -0,0 +1,51 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package net
+
+import (
+	"errors"
+	"strconv"
+)
+
+// Port - network port
+type Port uint16
+
+// String - returns string representation of port.
+func (p Port) String() string {
+	return strconv.Itoa(int(p))
+}
+
+// ParsePort - parses string into Port
+func ParsePort(s string) (p Port, err error) {
+	if s == "https" {
+		return Port(443), nil
+	} else if s == "http" {
+		return Port(80), nil
+	}
+
+	var i int
+	if i, err = strconv.Atoi(s); err != nil {
+		return p, errors.New("invalid port number")
+	}
+
+	if i < 0 || i > 65535 {
+		return p, errors.New("port must be between 0 to 65535")
+	}
+
+	return Port(i), nil
+}

internal/net/port_test.go

@@ -0,0 +1,75 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package net
+
+import (
+	"testing"
+)
+
+func TestPortString(t *testing.T) {
+	testCases := []struct {
+		port        Port
+		expectedStr string
+	}{
+		{Port(0), "0"},
+		{Port(9000), "9000"},
+		{Port(65535), "65535"},
+		{Port(1024), "1024"},
+	}
+
+	for i, testCase := range testCases {
+		str := testCase.port.String()
+
+		if str != testCase.expectedStr {
+			t.Fatalf("test %v: error: port: %v, got: %v", i+1, testCase.expectedStr, str)
+		}
+	}
+}
+
+func TestParsePort(t *testing.T) {
+	testCases := []struct {
+		s            string
+		expectedPort Port
+		expectErr    bool
+	}{
+		{"0", Port(0), false},
+		{"9000", Port(9000), false},
+		{"65535", Port(65535), false},
+		{"http", Port(80), false},
+		{"https", Port(443), false},
+		{"90000", Port(0), true},
+		{"-10", Port(0), true},
+		{"", Port(0), true},
+		{" 1024", Port(0), true},
+	}
+
+	for i, testCase := range testCases {
+		port, err := ParsePort(testCase.s)
+		expectErr := (err != nil)
+
+		if expectErr != testCase.expectErr {
+			t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
+		}
+
+		if !testCase.expectErr {
+			if port != testCase.expectedPort {
+				t.Fatalf("test %v: error: port: %v, got: %v", i+1, testCase.expectedPort, port)
+			}
+		}
+	}
+}

internal/net/url.go

@@ -0,0 +1,201 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package net
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"net/url"
+	"path"
+	"strings"
+)
+
+// URL - improved JSON friendly url.URL.
+type URL url.URL
+
+// IsEmpty - checks URL is empty or not.
+func (u URL) IsEmpty() bool {
+	return u.String() == ""
+}
+
+// String - returns string representation of URL.
+func (u URL) String() string {
+	// if port number 80 and 443, remove for http and https scheme respectively
+	if u.Host != "" {
+		host, err := ParseHost(u.Host)
+		if err != nil {
+			panic(err)
+		}
+		switch {
+		case u.Scheme == "http" && host.Port == 80:
+			fallthrough
+		case u.Scheme == "https" && host.Port == 443:
+			u.Host = host.Name
+		}
+	}
+
+	uu := url.URL(u)
+	return uu.String()
+}
+
+// MarshalJSON - converts to JSON string data.
+func (u URL) MarshalJSON() ([]byte, error) {
+	return json.Marshal(u.String())
+}
+
+// UnmarshalJSON - parses given data into URL.
+func (u *URL) UnmarshalJSON(data []byte) (err error) {
+	var s string
+	if err = json.Unmarshal(data, &s); err != nil {
+		return err
+	}
+
+	// Allow empty string
+	if s == "" {
+		*u = URL{}
+		return nil
+	}
+
+	var ru *URL
+	if ru, err = ParseURL(s); err != nil {
+		return err
+	}
+
+	*u = *ru
+	return nil
+}
+
+// ParseHTTPURL - parses a string into HTTP URL, string is
+// expected to be of form http:// or https://
+func ParseHTTPURL(s string) (u *URL, err error) {
+	u, err = ParseURL(s)
+	if err != nil {
+		return nil, err
+	}
+	switch u.Scheme {
+	default:
+		return nil, fmt.Errorf("unexpected scheme found %s", u.Scheme)
+	case "http", "https":
+		return u, nil
+	}
+}
+
+// ParseURL - parses string into URL.
+func ParseURL(s string) (u *URL, err error) {
+	var uu *url.URL
+	if uu, err = url.Parse(s); err != nil {
+		return nil, err
+	}
+
+	if uu.Hostname() == "" {
+		if uu.Scheme != "" {
+			return nil, errors.New("scheme appears with empty host")
+		}
+	} else {
+		portStr := uu.Port()
+		if portStr == "" {
+			switch uu.Scheme {
+			case "http":
+				portStr = "80"
+			case "https":
+				portStr = "443"
+			}
+		}
+		if _, err = ParseHost(net.JoinHostPort(uu.Hostname(), portStr)); err != nil {
+			return nil, err
+		}
+	}
+
+	// Clean path in the URL.
+	// Note: path.Clean() is used on purpose because in MS Windows filepath.Clean() converts
+	// `/` into `\` ie `/foo` becomes `\foo`
+	if uu.Path != "" {
+		uu.Path = path.Clean(uu.Path)
+	}
+
+	// path.Clean removes the trailing '/' and converts '//' to '/'.
+	if strings.HasSuffix(s, "/") && !strings.HasSuffix(uu.Path, "/") {
+		uu.Path += "/"
+	}
+
+	v := URL(*uu)
+	u = &v
+	return u, nil
+}
+
+// IsNetworkOrHostDown - if there was a network error or if the host is down.
+// expectTimeouts indicates that *context* timeouts are expected and does not
+// indicate a downed host. Other timeouts still returns down.
+func IsNetworkOrHostDown(err error, expectTimeouts bool) bool {
+	if err == nil {
+		return false
+	}
+
+	if errors.Is(err, context.Canceled) {
+		return false
+	}
+
+	if expectTimeouts && errors.Is(err, context.DeadlineExceeded) {
+		return false
+	}
+
+	// We need to figure if the error either a timeout
+	// or a non-temporary error.
+	urlErr := &url.Error{}
+	if errors.As(err, &urlErr) {
+		switch urlErr.Err.(type) {
+		case *net.DNSError, *net.OpError, net.UnknownNetworkError:
+			return true
+		}
+	}
+
+	var e net.Error
+	if errors.As(err, &e) {
+		if e.Timeout() {
+			return true
+		}
+	}
+
+	// Fallback to other mechanisms.
+	switch {
+	case strings.Contains(err.Error(), "Connection closed by foreign host"):
+		return true
+	case strings.Contains(err.Error(), "TLS handshake timeout"):
+		// If error is - tlsHandshakeTimeoutError.
+		return true
+	case strings.Contains(err.Error(), "i/o timeout"):
+		// If error is - tcp timeoutError.
+		return true
+	case strings.Contains(err.Error(), "connection timed out"):
+		// If err is a net.Dial timeout.
+		return true
+	case strings.Contains(err.Error(), "connection reset by peer"):
+		// IF err is a peer reset on a socket.
+		return true
+	case strings.Contains(err.Error(), "broken pipe"):
+		// IF err is a broken pipe on a socket.
+		return true
+	case strings.Contains(strings.ToLower(err.Error()), "503 service unavailable"):
+		// Denial errors
+		return true
+	}
+	return false
+}

internal/net/url_test.go

@@ -0,0 +1,205 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package net
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestURLIsEmpty(t *testing.T) {
+	testCases := []struct {
+		url            URL
+		expectedResult bool
+	}{
+		{URL{}, true},
+		{URL{Scheme: "http", Host: "play"}, false},
+		{URL{Path: "path/to/play"}, false},
+	}
+
+	for i, testCase := range testCases {
+		result := testCase.url.IsEmpty()
+
+		if result != testCase.expectedResult {
+			t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result)
+		}
+	}
+}
+
+func TestURLString(t *testing.T) {
+	testCases := []struct {
+		url         URL
+		expectedStr string
+	}{
+		{URL{}, ""},
+		{URL{Scheme: "http", Host: "play"}, "http://play"},
+		{URL{Scheme: "https", Host: "play:443"}, "https://play"},
+		{URL{Scheme: "https", Host: "play.min.io:80"}, "https://play.min.io:80"},
+		{URL{Scheme: "https", Host: "147.75.201.93:9000", Path: "/"}, "https://147.75.201.93:9000/"},
+		{URL{Scheme: "https", Host: "s3.amazonaws.com", Path: "/", RawQuery: "location"}, "https://s3.amazonaws.com/?location"},
+		{URL{Scheme: "http", Host: "myminio:10000", Path: "/mybucket/myobject"}, "http://myminio:10000/mybucket/myobject"},
+		{URL{Scheme: "ftp", Host: "myftp.server:10000", Path: "/myuser"}, "ftp://myftp.server:10000/myuser"},
+		{URL{Path: "path/to/play"}, "path/to/play"},
+	}
+
+	for i, testCase := range testCases {
+		str := testCase.url.String()
+
+		if str != testCase.expectedStr {
+			t.Fatalf("test %v: string: expected: %v, got: %v", i+1, testCase.expectedStr, str)
+		}
+	}
+}
+
+func TestURLMarshalJSON(t *testing.T) {
+	testCases := []struct {
+		url          URL
+		expectedData []byte
+		expectErr    bool
+	}{
+		{URL{}, []byte(`""`), false},
+		{URL{Scheme: "http", Host: "play"}, []byte(`"http://play"`), false},
+		{URL{Scheme: "https", Host: "play.min.io:0"}, []byte(`"https://play.min.io:0"`), false},
+		{URL{Scheme: "https", Host: "147.75.201.93:9000", Path: "/"}, []byte(`"https://147.75.201.93:9000/"`), false},
+		{URL{Scheme: "https", Host: "s3.amazonaws.com", Path: "/", RawQuery: "location"}, []byte(`"https://s3.amazonaws.com/?location"`), false},
+		{URL{Scheme: "http", Host: "myminio:10000", Path: "/mybucket/myobject"}, []byte(`"http://myminio:10000/mybucket/myobject"`), false},
+		{URL{Scheme: "ftp", Host: "myftp.server:10000", Path: "/myuser"}, []byte(`"ftp://myftp.server:10000/myuser"`), false},
+	}
+
+	for i, testCase := range testCases {
+		data, err := testCase.url.MarshalJSON()
+		expectErr := (err != nil)
+
+		if expectErr != testCase.expectErr {
+			t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
+		}
+
+		if !testCase.expectErr {
+			if !reflect.DeepEqual(data, testCase.expectedData) {
+				t.Fatalf("test %v: data: expected: %v, got: %v", i+1, string(testCase.expectedData), string(data))
+			}
+		}
+	}
+}
+
+func TestURLUnmarshalJSON(t *testing.T) {
+	testCases := []struct {
+		data        []byte
+		expectedURL *URL
+		expectErr   bool
+	}{
+		{[]byte(`""`), &URL{}, false},
+		{[]byte(`"http://play"`), &URL{Scheme: "http", Host: "play"}, false},
+		{[]byte(`"https://play.min.io:0"`), &URL{Scheme: "https", Host: "play.min.io:0"}, false},
+		{[]byte(`"https://147.75.201.93:9000/"`), &URL{Scheme: "https", Host: "147.75.201.93:9000", Path: "/"}, false},
+		{[]byte(`"https://s3.amazonaws.com/?location"`), &URL{Scheme: "https", Host: "s3.amazonaws.com", Path: "/", RawQuery: "location"}, false},
+		{[]byte(`"http://myminio:10000/mybucket/myobject//"`), &URL{Scheme: "http", Host: "myminio:10000", Path: "/mybucket/myobject/"}, false},
+		{[]byte(`"ftp://myftp.server:10000/myuser"`), &URL{Scheme: "ftp", Host: "myftp.server:10000", Path: "/myuser"}, false},
+		{[]byte(`"http://webhook.server:10000/mywebhook/"`), &URL{Scheme: "http", Host: "webhook.server:10000", Path: "/mywebhook/"}, false},
+		{[]byte(`"myserver:1000"`), nil, true},
+		{[]byte(`"http://:1000/mybucket"`), nil, true},
+		{[]byte(`"https://147.75.201.93:90000/"`), nil, true},
+		{[]byte(`"http:/play"`), nil, true},
+	}
+
+	for i, testCase := range testCases {
+		var url URL
+		err := url.UnmarshalJSON(testCase.data)
+		expectErr := (err != nil)
+
+		if expectErr != testCase.expectErr {
+			t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
+		}
+
+		if !testCase.expectErr {
+			if !reflect.DeepEqual(&url, testCase.expectedURL) {
+				t.Fatalf("test %v: host: expected: %#v, got: %#v", i+1, testCase.expectedURL, url)
+			}
+		}
+	}
+}
+
+func TestParseHTTPURL(t *testing.T) {
+	testCases := []struct {
+		s           string
+		expectedURL *URL
+		expectErr   bool
+	}{
+		{"http://play", &URL{Scheme: "http", Host: "play"}, false},
+		{"https://play.min.io:0", &URL{Scheme: "https", Host: "play.min.io:0"}, false},
+		{"https://147.75.201.93:9000/", &URL{Scheme: "https", Host: "147.75.201.93:9000", Path: "/"}, false},
+		{"https://s3.amazonaws.com/?location", &URL{Scheme: "https", Host: "s3.amazonaws.com", Path: "/", RawQuery: "location"}, false},
+		{"http://myminio:10000/mybucket//myobject/", &URL{Scheme: "http", Host: "myminio:10000", Path: "/mybucket/myobject/"}, false},
+		{"ftp://myftp.server:10000/myuser", nil, true},
+		{"https://my.server:10000000/myuser", nil, true},
+		{"myserver:1000", nil, true},
+		{"http://:1000/mybucket", nil, true},
+		{"https://147.75.201.93:90000/", nil, true},
+		{"http:/play", nil, true},
+	}
+
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run(testCase.s, func(t *testing.T) {
+			url, err := ParseHTTPURL(testCase.s)
+			expectErr := (err != nil)
+			if expectErr != testCase.expectErr {
+				t.Fatalf("error: expected: %v, got: %v", testCase.expectErr, expectErr)
+			}
+			if !testCase.expectErr {
+				if !reflect.DeepEqual(url, testCase.expectedURL) {
+					t.Fatalf("host: expected: %#v, got: %#v", testCase.expectedURL, url)
+				}
+			}
+		})
+	}
+}
+
+func TestParseURL(t *testing.T) {
+	testCases := []struct {
+		s           string
+		expectedURL *URL
+		expectErr   bool
+	}{
+		{"http://play", &URL{Scheme: "http", Host: "play"}, false},
+		{"https://play.min.io:0", &URL{Scheme: "https", Host: "play.min.io:0"}, false},
+		{"https://147.75.201.93:9000/", &URL{Scheme: "https", Host: "147.75.201.93:9000", Path: "/"}, false},
+		{"https://s3.amazonaws.com/?location", &URL{Scheme: "https", Host: "s3.amazonaws.com", Path: "/", RawQuery: "location"}, false},
+		{"http://myminio:10000/mybucket//myobject/", &URL{Scheme: "http", Host: "myminio:10000", Path: "/mybucket/myobject/"}, false},
+		{"ftp://myftp.server:10000/myuser", &URL{Scheme: "ftp", Host: "myftp.server:10000", Path: "/myuser"}, false},
+		{"myserver:1000", nil, true},
+		{"http://:1000/mybucket", nil, true},
+		{"https://147.75.201.93:90000/", nil, true},
+		{"http:/play", nil, true},
+	}
+
+	for i, testCase := range testCases {
+		url, err := ParseURL(testCase.s)
+		expectErr := (err != nil)
+
+		if expectErr != testCase.expectErr {
+			t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
+		}
+
+		if !testCase.expectErr {
+			if !reflect.DeepEqual(url, testCase.expectedURL) {
+				t.Fatalf("test %v: host: expected: %#v, got: %#v", i+1, testCase.expectedURL, url)
+			}
+		}
+	}
+}