mirror of
https://github.com/minio/minio.git
synced 2024-12-23 21:55:53 -05:00
Add extensive endpoints validation (#4019)
This commit is contained in:
parent
1b1b9e4801
commit
de204a0a52
@ -158,12 +158,7 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
|
||||
// Initialize boot time
|
||||
globalBootTime = UTCNow()
|
||||
|
||||
// Set globalEndpoints for a single node XL setup.
|
||||
for _, xlDir := range xlDirs {
|
||||
globalEndpoints = append(globalEndpoints, &url.URL{
|
||||
Path: xlDir,
|
||||
})
|
||||
}
|
||||
globalEndpoints = mustGetNewEndpointList(xlDirs...)
|
||||
|
||||
// Set globalIsXL to indicate that the setup uses an erasure code backend.
|
||||
globalIsXL = true
|
||||
@ -301,14 +296,8 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
|
||||
// Initialize admin peers to make admin RPC calls. Note: In a
|
||||
// single node setup, this degenerates to a simple function
|
||||
// call under the hood.
|
||||
eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse storage end point - %v", err)
|
||||
}
|
||||
|
||||
// Set globalMinioAddr to be able to distinguish local endpoints from remote.
|
||||
globalMinioAddr = eps[0].Host
|
||||
initGlobalAdminPeers(eps)
|
||||
globalMinioAddr = "127.0.0.1:9000"
|
||||
initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
|
||||
|
||||
// Setting up a go routine to simulate ServerMux's
|
||||
// handleServiceSignals for stop and restart commands.
|
||||
@ -367,14 +356,8 @@ func TestServiceSetCreds(t *testing.T) {
|
||||
// Initialize admin peers to make admin RPC calls. Note: In a
|
||||
// single node setup, this degenerates to a simple function
|
||||
// call under the hood.
|
||||
eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse storage end point - %v", err)
|
||||
}
|
||||
|
||||
// Set globalMinioAddr to be able to distinguish local endpoints from remote.
|
||||
globalMinioAddr = eps[0].Host
|
||||
initGlobalAdminPeers(eps)
|
||||
globalMinioAddr = "127.0.0.1:9000"
|
||||
initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
|
||||
|
||||
credentials := serverConfig.GetCredential()
|
||||
var body []byte
|
||||
@ -455,14 +438,8 @@ func TestListLocksHandler(t *testing.T) {
|
||||
defer adminTestBed.TearDown()
|
||||
|
||||
// Initialize admin peers to make admin RPC calls.
|
||||
eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse storage end point - %v", err)
|
||||
}
|
||||
|
||||
// Set globalMinioAddr to be able to distinguish local endpoints from remote.
|
||||
globalMinioAddr = eps[0].Host
|
||||
initGlobalAdminPeers(eps)
|
||||
globalMinioAddr = "127.0.0.1:9000"
|
||||
initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
|
||||
|
||||
testCases := []struct {
|
||||
bucket string
|
||||
@ -530,11 +507,7 @@ func TestClearLocksHandler(t *testing.T) {
|
||||
defer adminTestBed.TearDown()
|
||||
|
||||
// Initialize admin peers to make admin RPC calls.
|
||||
eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse storage end point - %v", err)
|
||||
}
|
||||
initGlobalAdminPeers(eps)
|
||||
initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
|
||||
|
||||
testCases := []struct {
|
||||
bucket string
|
||||
@ -1238,14 +1211,8 @@ func TestGetConfigHandler(t *testing.T) {
|
||||
defer adminTestBed.TearDown()
|
||||
|
||||
// Initialize admin peers to make admin RPC calls.
|
||||
eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse storage end point - %v", err)
|
||||
}
|
||||
|
||||
// Set globalMinioAddr to be able to distinguish local endpoints from remote.
|
||||
globalMinioAddr = eps[0].Host
|
||||
initGlobalAdminPeers(eps)
|
||||
globalMinioAddr = "127.0.0.1:9000"
|
||||
initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
|
||||
|
||||
// Prepare query params for get-config mgmt REST API.
|
||||
queryVal := url.Values{}
|
||||
@ -1273,14 +1240,8 @@ func TestSetConfigHandler(t *testing.T) {
|
||||
defer adminTestBed.TearDown()
|
||||
|
||||
// Initialize admin peers to make admin RPC calls.
|
||||
eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse storage end point - %v", err)
|
||||
}
|
||||
|
||||
// Set globalMinioAddr to be able to distinguish local endpoints from remote.
|
||||
globalMinioAddr = eps[0].Host
|
||||
initGlobalAdminPeers(eps)
|
||||
globalMinioAddr = "127.0.0.1:9000"
|
||||
initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
|
||||
|
||||
// SetConfigHandler restarts minio setup - need to start a
|
||||
// signal receiver to receive on globalServiceSignalCh.
|
||||
@ -1321,14 +1282,8 @@ func TestAdminServerInfo(t *testing.T) {
|
||||
defer adminTestBed.TearDown()
|
||||
|
||||
// Initialize admin peers to make admin RPC calls.
|
||||
eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse storage end point - %v", err)
|
||||
}
|
||||
|
||||
// Set globalMinioAddr to be able to distinguish local endpoints from remote.
|
||||
globalMinioAddr = eps[0].Host
|
||||
initGlobalAdminPeers(eps)
|
||||
globalMinioAddr = "127.0.0.1:9000"
|
||||
initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
|
||||
|
||||
// Prepare query params for set-config mgmt REST API.
|
||||
queryVal := url.Values{}
|
||||
|
@ -20,7 +20,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"net"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
@ -28,6 +28,8 @@ import (
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -211,52 +213,43 @@ type adminPeer struct {
|
||||
type adminPeers []adminPeer
|
||||
|
||||
// makeAdminPeers - helper function to construct a collection of adminPeer.
|
||||
func makeAdminPeers(eps []*url.URL) adminPeers {
|
||||
var servicePeers []adminPeer
|
||||
|
||||
// map to store peers that are already added to ret
|
||||
seenAddr := make(map[string]bool)
|
||||
|
||||
// add local (self) as peer in the array
|
||||
servicePeers = append(servicePeers, adminPeer{
|
||||
globalMinioAddr,
|
||||
func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
|
||||
thisPeer := globalMinioAddr
|
||||
if globalMinioHost == "" {
|
||||
thisPeer = net.JoinHostPort("localhost", globalMinioPort)
|
||||
}
|
||||
adminPeerList = append(adminPeerList, adminPeer{
|
||||
thisPeer,
|
||||
localAdminClient{},
|
||||
})
|
||||
seenAddr[globalMinioAddr] = true
|
||||
|
||||
serverCred := serverConfig.GetCredential()
|
||||
// iterate over endpoints to find new remote peers and add
|
||||
// them to ret.
|
||||
for _, ep := range eps {
|
||||
if ep.Host == "" {
|
||||
hostSet := set.CreateStringSet(globalMinioAddr)
|
||||
cred := serverConfig.GetCredential()
|
||||
serviceEndpoint := path.Join(minioReservedBucketPath, adminPath)
|
||||
for _, host := range GetRemotePeers(endpoints) {
|
||||
if hostSet.Contains(host) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if the remote host has been added already
|
||||
if !seenAddr[ep.Host] {
|
||||
cfg := authConfig{
|
||||
accessKey: serverCred.AccessKey,
|
||||
secretKey: serverCred.SecretKey,
|
||||
serverAddr: ep.Host,
|
||||
hostSet.Add(host)
|
||||
adminPeerList = append(adminPeerList, adminPeer{
|
||||
addr: host,
|
||||
cmdRunner: &remoteAdminClient{newAuthRPCClient(authConfig{
|
||||
accessKey: cred.AccessKey,
|
||||
secretKey: cred.SecretKey,
|
||||
serverAddr: host,
|
||||
serviceEndpoint: serviceEndpoint,
|
||||
secureConn: globalIsSSL,
|
||||
serviceEndpoint: path.Join(minioReservedBucketPath, adminPath),
|
||||
serviceName: "Admin",
|
||||
}
|
||||
|
||||
servicePeers = append(servicePeers, adminPeer{
|
||||
addr: ep.Host,
|
||||
cmdRunner: &remoteAdminClient{newAuthRPCClient(cfg)},
|
||||
})
|
||||
seenAddr[ep.Host] = true
|
||||
}
|
||||
})},
|
||||
})
|
||||
}
|
||||
|
||||
return servicePeers
|
||||
return adminPeerList
|
||||
}
|
||||
|
||||
// Initialize global adminPeer collection.
|
||||
func initGlobalAdminPeers(eps []*url.URL) {
|
||||
globalAdminPeers = makeAdminPeers(eps)
|
||||
func initGlobalAdminPeers(endpoints EndpointList) {
|
||||
globalAdminPeers = makeAdminPeers(endpoints)
|
||||
}
|
||||
|
||||
// invokeServiceCmd - Invoke Restart command.
|
||||
|
@ -18,7 +18,6 @@ package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@ -86,9 +85,7 @@ func TestReInitDisks(t *testing.T) {
|
||||
defer removeRoots(xlDirs)
|
||||
|
||||
// Set globalEndpoints for a single node XL setup.
|
||||
for _, xlDir := range xlDirs {
|
||||
globalEndpoints = append(globalEndpoints, &url.URL{Path: xlDir})
|
||||
}
|
||||
globalEndpoints = mustGetNewEndpointList(xlDirs...)
|
||||
|
||||
// Setup admin rpc server for an XL backend.
|
||||
globalIsXL = true
|
||||
|
@ -16,7 +16,11 @@
|
||||
|
||||
package cmd
|
||||
|
||||
import "strings"
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
)
|
||||
|
||||
// List of valid event types.
|
||||
var suppportedEventTypes = map[string]struct{}{
|
||||
@ -207,16 +211,14 @@ func validateQueueConfigs(queueConfigs []queueConfig) APIErrorCode {
|
||||
|
||||
// Check all the queue configs for any duplicates.
|
||||
func checkDuplicateQueueConfigs(configs []queueConfig) APIErrorCode {
|
||||
var queueConfigARNS []string
|
||||
queueConfigARNS := set.NewStringSet()
|
||||
|
||||
// Navigate through each configs and count the entries.
|
||||
for _, config := range configs {
|
||||
queueConfigARNS = append(queueConfigARNS, config.QueueARN)
|
||||
queueConfigARNS.Add(config.QueueARN)
|
||||
}
|
||||
|
||||
// Check if there are any duplicate counts.
|
||||
if err := checkDuplicateStrings(queueConfigARNS); err != nil {
|
||||
errorIf(err, "Invalid queue configs found.")
|
||||
if len(queueConfigARNS) != len(configs) {
|
||||
return ErrOverlappingConfigs
|
||||
}
|
||||
|
||||
|
@ -1,64 +0,0 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"net"
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// checkPortAvailability - check if given port is already in use.
|
||||
// Note: The check method tries to listen on given port and closes it.
|
||||
// It is possible to have a disconnected client in this tiny window of time.
|
||||
func checkPortAvailability(port string) error {
|
||||
network := [3]string{"tcp", "tcp4", "tcp6"}
|
||||
for _, n := range network {
|
||||
l, err := net.Listen(n, net.JoinHostPort("", port))
|
||||
if err != nil {
|
||||
if isAddrInUse(err) {
|
||||
// Return error if another process is listening on the
|
||||
// same port.
|
||||
return err
|
||||
}
|
||||
// Ignore any other error (ex. EAFNOSUPPORT)
|
||||
continue
|
||||
}
|
||||
|
||||
// look for error so we don't have dangling connection
|
||||
if err = l.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Return true if err is "address already in use" error.
|
||||
// syscall.EADDRINUSE is available on all OSes.
|
||||
func isAddrInUse(err error) bool {
|
||||
if opErr, ok := err.(*net.OpError); ok {
|
||||
if sysErr, ok := opErr.Err.(*os.SyscallError); ok {
|
||||
if errno, ok := sysErr.Err.(syscall.Errno); ok {
|
||||
if errno == syscall.EADDRINUSE {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
@ -1,54 +0,0 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"net"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Tests for port availability logic written for server startup sequence.
|
||||
func TestCheckPortAvailability(t *testing.T) {
|
||||
tests := []struct {
|
||||
port string
|
||||
}{
|
||||
{getFreePort()},
|
||||
{getFreePort()},
|
||||
}
|
||||
for _, test := range tests {
|
||||
// This test should pass if the ports are available
|
||||
err := checkPortAvailability(test.port)
|
||||
if err != nil {
|
||||
t.Fatalf("checkPortAvailability test failed for port: %s. Error: %v", test.port, err)
|
||||
}
|
||||
|
||||
// Now use the ports and check again
|
||||
ln, err := net.Listen("tcp", net.JoinHostPort("", test.port))
|
||||
if err != nil {
|
||||
t.Fail()
|
||||
}
|
||||
defer ln.Close()
|
||||
|
||||
err = checkPortAvailability(test.port)
|
||||
|
||||
// Skip if the os is windows due to https://github.com/golang/go/issues/7598
|
||||
if err == nil && runtime.GOOS != globalWindowsOSName {
|
||||
t.Fatalf("checkPortAvailability should fail for port: %s. Error: %v", test.port, err)
|
||||
}
|
||||
}
|
||||
}
|
382
cmd/endpoint.go
Normal file
382
cmd/endpoint.go
Normal file
@ -0,0 +1,382 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"path"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
)
|
||||
|
||||
// EndpointType - enum for endpoint type.
|
||||
type EndpointType int
|
||||
|
||||
const (
|
||||
// PathEndpointType - path style endpoint type enum.
|
||||
PathEndpointType EndpointType = iota + 1
|
||||
|
||||
// URLEndpointType - URL style endpoint type enum.
|
||||
URLEndpointType
|
||||
)
|
||||
|
||||
// Endpoint - any type of endpoint.
|
||||
type Endpoint struct {
|
||||
*url.URL
|
||||
IsLocal bool
|
||||
}
|
||||
|
||||
func (endpoint Endpoint) String() string {
|
||||
if endpoint.Host == "" {
|
||||
return endpoint.Path
|
||||
}
|
||||
|
||||
return endpoint.URL.String()
|
||||
}
|
||||
|
||||
// Type - returns type of endpoint.
|
||||
func (endpoint Endpoint) Type() EndpointType {
|
||||
if endpoint.Host == "" {
|
||||
return PathEndpointType
|
||||
}
|
||||
|
||||
return URLEndpointType
|
||||
}
|
||||
|
||||
// SetHTTPS - sets secure http for URLEndpointType.
|
||||
func (endpoint Endpoint) SetHTTPS() {
|
||||
if endpoint.Host != "" {
|
||||
endpoint.Scheme = "https"
|
||||
}
|
||||
}
|
||||
|
||||
// SetHTTP - sets insecure http for URLEndpointType.
|
||||
func (endpoint Endpoint) SetHTTP() {
|
||||
if endpoint.Host != "" {
|
||||
endpoint.Scheme = "http"
|
||||
}
|
||||
}
|
||||
|
||||
// NewEndpoint - returns new endpoint based on given arguments.
|
||||
func NewEndpoint(arg string) (Endpoint, error) {
|
||||
// isEmptyPath - check whether given path is not empty.
|
||||
isEmptyPath := func(path string) bool {
|
||||
return path == "" || path == "." || path == "/" || path == `\`
|
||||
}
|
||||
|
||||
if isEmptyPath(arg) {
|
||||
return Endpoint{}, fmt.Errorf("empty or root endpoint is not supported")
|
||||
}
|
||||
|
||||
var isLocal bool
|
||||
u, err := url.Parse(arg)
|
||||
if err == nil && u.Host != "" {
|
||||
// URL style of endpoint.
|
||||
// Valid URL style endpoint is
|
||||
// - Scheme field must contain "http" or "https"
|
||||
// - All field should be empty except Host and Path.
|
||||
if !((u.Scheme == "http" || u.Scheme == "https") &&
|
||||
u.User == nil && u.Opaque == "" && u.ForceQuery == false && u.RawQuery == "" && u.Fragment == "") {
|
||||
return Endpoint{}, fmt.Errorf("invalid URL endpoint format")
|
||||
}
|
||||
|
||||
host, port, err := net.SplitHostPort(u.Host)
|
||||
if err != nil {
|
||||
if !strings.Contains(err.Error(), "missing port in address") {
|
||||
return Endpoint{}, fmt.Errorf("invalid URL endpoint format: %s", err)
|
||||
}
|
||||
|
||||
host = u.Host
|
||||
} else {
|
||||
var p int
|
||||
p, err = strconv.Atoi(port)
|
||||
if err != nil {
|
||||
return Endpoint{}, fmt.Errorf("invalid URL endpoint format: invalid port number")
|
||||
} else if p < 1 || p > 65535 {
|
||||
return Endpoint{}, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535")
|
||||
}
|
||||
}
|
||||
|
||||
if host == "" {
|
||||
return Endpoint{}, fmt.Errorf("invalid URL endpoint format: empty host name")
|
||||
}
|
||||
|
||||
// As this is path in the URL, we should use path package, not filepath package.
|
||||
// On MS Windows, filepath.Clean() converts into Windows path style ie `/foo` becomes `\foo`
|
||||
u.Path = path.Clean(u.Path)
|
||||
if isEmptyPath(u.Path) {
|
||||
return Endpoint{}, fmt.Errorf("empty or root path is not supported in URL endpoint")
|
||||
}
|
||||
|
||||
// Get IPv4 address of the host.
|
||||
hostIPs, err := getHostIP4(host)
|
||||
if err != nil {
|
||||
return Endpoint{}, err
|
||||
}
|
||||
|
||||
// If intersection of two IP sets is not empty, then the host is local host.
|
||||
isLocal = !localIP4.Intersection(hostIPs).IsEmpty()
|
||||
} else {
|
||||
u = &url.URL{Path: path.Clean(arg)}
|
||||
isLocal = true
|
||||
}
|
||||
|
||||
return Endpoint{
|
||||
URL: u,
|
||||
IsLocal: isLocal,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// EndpointList - list of same type of endpoint.
|
||||
type EndpointList []Endpoint
|
||||
|
||||
// Swap - helper method for sorting.
|
||||
func (endpoints EndpointList) Swap(i, j int) {
|
||||
endpoints[i], endpoints[j] = endpoints[j], endpoints[i]
|
||||
}
|
||||
|
||||
// Len - helper method for sorting.
|
||||
func (endpoints EndpointList) Len() int {
|
||||
return len(endpoints)
|
||||
}
|
||||
|
||||
// Less - helper method for sorting.
|
||||
func (endpoints EndpointList) Less(i, j int) bool {
|
||||
return endpoints[i].String() < endpoints[j].String()
|
||||
}
|
||||
|
||||
// SetHTTPS - sets secure http for URLEndpointType.
|
||||
func (endpoints EndpointList) SetHTTPS() {
|
||||
for i := range endpoints {
|
||||
endpoints[i].SetHTTPS()
|
||||
}
|
||||
}
|
||||
|
||||
// SetHTTP - sets insecure http for URLEndpointType.
|
||||
func (endpoints EndpointList) SetHTTP() {
|
||||
for i := range endpoints {
|
||||
endpoints[i].SetHTTP()
|
||||
}
|
||||
}
|
||||
|
||||
// NewEndpointList - returns new endpoint list based on input args.
|
||||
func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
|
||||
// isValidDistribution - checks whether given count is a valid distribution for erasure coding.
|
||||
isValidDistribution := func(count int) bool {
|
||||
return (count >= 4 && count <= 16 && count%2 == 0)
|
||||
}
|
||||
|
||||
// Check whether no. of args are valid for XL distribution.
|
||||
if !isValidDistribution(len(args)) {
|
||||
return nil, fmt.Errorf("total endpoints %d found. For XL/Distribute, it should be 4, 6, 8, 10, 12, 14 or 16", len(args))
|
||||
}
|
||||
|
||||
var endpointType EndpointType
|
||||
var scheme string
|
||||
|
||||
uniqueArgs := set.NewStringSet()
|
||||
// Loop through args and adds to endpoint list.
|
||||
for i, arg := range args {
|
||||
endpoint, err := NewEndpoint(arg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("'%s': %s", arg, err.Error())
|
||||
}
|
||||
|
||||
// All endpoints have to be same type and scheme if applicable.
|
||||
if i == 0 {
|
||||
endpointType = endpoint.Type()
|
||||
scheme = endpoint.Scheme
|
||||
} else if endpoint.Type() != endpointType {
|
||||
return nil, fmt.Errorf("mixed style endpoints are not supported")
|
||||
} else if endpoint.Scheme != scheme {
|
||||
return nil, fmt.Errorf("mixed scheme is not supported")
|
||||
}
|
||||
|
||||
arg = endpoint.String()
|
||||
if uniqueArgs.Contains(arg) {
|
||||
return nil, fmt.Errorf("duplicate endpoints found")
|
||||
}
|
||||
uniqueArgs.Add(arg)
|
||||
|
||||
endpoints = append(endpoints, endpoint)
|
||||
}
|
||||
|
||||
sort.Sort(endpoints)
|
||||
|
||||
return endpoints, nil
|
||||
}
|
||||
|
||||
// CreateEndpoints - validates and creates new endpoints for given args.
|
||||
func CreateEndpoints(serverAddr string, args ...string) (string, EndpointList, SetupType, error) {
|
||||
var endpoints EndpointList
|
||||
var setupType SetupType
|
||||
var err error
|
||||
|
||||
// Check whether serverAddr is valid for this host.
|
||||
if err = CheckLocalServerAddr(serverAddr); err != nil {
|
||||
return serverAddr, endpoints, setupType, err
|
||||
}
|
||||
|
||||
_, serverAddrPort := mustSplitHostPort(serverAddr)
|
||||
|
||||
// For single arg, return FS setup.
|
||||
if len(args) == 1 {
|
||||
var endpoint Endpoint
|
||||
endpoint, err = NewEndpoint(args[0])
|
||||
if err != nil {
|
||||
return serverAddr, endpoints, setupType, err
|
||||
}
|
||||
|
||||
if endpoint.Type() != PathEndpointType {
|
||||
return serverAddr, endpoints, setupType, fmt.Errorf("use path style endpoint for FS setup")
|
||||
}
|
||||
|
||||
endpoints = append(endpoints, endpoint)
|
||||
setupType = FSSetupType
|
||||
return serverAddr, endpoints, setupType, nil
|
||||
}
|
||||
|
||||
// Convert args to endpoints
|
||||
if endpoints, err = NewEndpointList(args...); err != nil {
|
||||
return serverAddr, endpoints, setupType, err
|
||||
}
|
||||
|
||||
// Return XL setup when all endpoints are path style.
|
||||
if endpoints[0].Type() == PathEndpointType {
|
||||
setupType = XLSetupType
|
||||
return serverAddr, endpoints, setupType, nil
|
||||
}
|
||||
|
||||
// Here all endpoints are URL style.
|
||||
endpointPathSet := set.NewStringSet()
|
||||
localEndpointCount := 0
|
||||
localServerAddrSet := set.NewStringSet()
|
||||
localPortSet := set.NewStringSet()
|
||||
for _, endpoint := range endpoints {
|
||||
endpointPathSet.Add(endpoint.Path)
|
||||
if endpoint.IsLocal {
|
||||
localServerAddrSet.Add(endpoint.Host)
|
||||
|
||||
var port string
|
||||
_, port, err = net.SplitHostPort(endpoint.Host)
|
||||
if err != nil {
|
||||
port = serverAddrPort
|
||||
}
|
||||
|
||||
localPortSet.Add(port)
|
||||
|
||||
localEndpointCount++
|
||||
}
|
||||
}
|
||||
|
||||
// No local endpoint found.
|
||||
if localEndpointCount == 0 {
|
||||
return serverAddr, endpoints, setupType, fmt.Errorf("no endpoint found for this host")
|
||||
}
|
||||
|
||||
// Check whether same path is not used in endpoints of a host.
|
||||
{
|
||||
pathIPMap := make(map[string]set.StringSet)
|
||||
for _, endpoint := range endpoints {
|
||||
var host string
|
||||
host, _, err = net.SplitHostPort(endpoint.Host)
|
||||
if err != nil {
|
||||
host = endpoint.Host
|
||||
}
|
||||
hostIPSet, _ := getHostIP4(host)
|
||||
if IPSet, ok := pathIPMap[endpoint.Path]; ok {
|
||||
if !IPSet.Intersection(hostIPSet).IsEmpty() {
|
||||
err = fmt.Errorf("path '%s' can not be served from different address/port", endpoint.Path)
|
||||
return serverAddr, endpoints, setupType, err
|
||||
}
|
||||
} else {
|
||||
pathIPMap[endpoint.Path] = hostIPSet
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check whether serverAddrPort matches at least in one of port used in local endpoints.
|
||||
{
|
||||
if !localPortSet.Contains(serverAddrPort) {
|
||||
if len(localPortSet) > 1 {
|
||||
err = fmt.Errorf("port number in server address must match with one of the port in local endpoints")
|
||||
} else {
|
||||
err = fmt.Errorf("server address and local endpoint have different ports")
|
||||
}
|
||||
|
||||
return serverAddr, endpoints, setupType, err
|
||||
}
|
||||
}
|
||||
|
||||
// If all endpoints are pointing to local host and having same port number, then this is XL setup using URL style endpoints.
|
||||
if len(endpoints) == localEndpointCount && len(localPortSet) == 1 {
|
||||
if len(localServerAddrSet) > 1 {
|
||||
// TODO: Eventhough all endpoints are local, the local host is referred by different IP/name.
|
||||
// eg '172.0.0.1', 'localhost' and 'mylocalhostname' point to same local host.
|
||||
//
|
||||
// In this case, we bind to 0.0.0.0 ie to all interfaces.
|
||||
// The actual way to do is bind to only IPs in uniqueLocalHosts.
|
||||
serverAddr = net.JoinHostPort("", serverAddrPort)
|
||||
}
|
||||
|
||||
endpointPaths := endpointPathSet.ToSlice()
|
||||
endpoints, _ = NewEndpointList(endpointPaths...)
|
||||
setupType = XLSetupType
|
||||
return serverAddr, endpoints, setupType, nil
|
||||
}
|
||||
|
||||
// Add missing port in all endpoints.
|
||||
for i := range endpoints {
|
||||
_, port, err := net.SplitHostPort(endpoints[i].Host)
|
||||
if err != nil {
|
||||
endpoints[i].Host = net.JoinHostPort(endpoints[i].Host, serverAddrPort)
|
||||
} else if endpoints[i].IsLocal && serverAddrPort != port {
|
||||
// If endpoint is local, but port is different than serverAddrPort, then make it as remote.
|
||||
endpoints[i].IsLocal = false
|
||||
}
|
||||
}
|
||||
|
||||
// This is DistXL setup.
|
||||
setupType = DistXLSetupType
|
||||
return serverAddr, endpoints, setupType, nil
|
||||
}
|
||||
|
||||
// GetRemotePeers - get hosts information other than this minio service.
|
||||
func GetRemotePeers(endpoints EndpointList) []string {
|
||||
peerSet := set.NewStringSet()
|
||||
for _, endpoint := range endpoints {
|
||||
if endpoint.Type() != URLEndpointType {
|
||||
continue
|
||||
}
|
||||
|
||||
peer := endpoint.Host
|
||||
if endpoint.IsLocal {
|
||||
if _, port := mustSplitHostPort(peer); port == globalMinioPort {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
peerSet.Add(peer)
|
||||
}
|
||||
|
||||
return peerSet.ToSlice()
|
||||
}
|
312
cmd/endpoint_test.go
Normal file
312
cmd/endpoint_test.go
Normal file
@ -0,0 +1,312 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewEndpoint(t *testing.T) {
|
||||
u1, _ := url.Parse("http://localhost/path")
|
||||
u2, _ := url.Parse("https://example.org/path")
|
||||
u3, _ := url.Parse("http://127.0.0.1:8080/path")
|
||||
u4, _ := url.Parse("http://192.168.253.200/path")
|
||||
|
||||
errMsg := ": no such host"
|
||||
if runtime.GOOS == "windows" {
|
||||
errMsg = ": No such host is known."
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
arg string
|
||||
expectedEndpoint Endpoint
|
||||
expectedType EndpointType
|
||||
expectedErr error
|
||||
}{
|
||||
{"foo", Endpoint{URL: &url.URL{Path: "foo"}, IsLocal: true}, PathEndpointType, nil},
|
||||
{"/foo", Endpoint{URL: &url.URL{Path: "/foo"}, IsLocal: true}, PathEndpointType, nil},
|
||||
{`\foo`, Endpoint{URL: &url.URL{Path: `\foo`}, IsLocal: true}, PathEndpointType, nil},
|
||||
{"C", Endpoint{URL: &url.URL{Path: `C`}, IsLocal: true}, PathEndpointType, nil},
|
||||
{"C:", Endpoint{URL: &url.URL{Path: `C:`}, IsLocal: true}, PathEndpointType, nil},
|
||||
{"C:/", Endpoint{URL: &url.URL{Path: "C:"}, IsLocal: true}, PathEndpointType, nil},
|
||||
{`C:\`, Endpoint{URL: &url.URL{Path: `C:\`}, IsLocal: true}, PathEndpointType, nil},
|
||||
{`C:\foo`, Endpoint{URL: &url.URL{Path: `C:\foo`}, IsLocal: true}, PathEndpointType, nil},
|
||||
{"C:/foo", Endpoint{URL: &url.URL{Path: "C:/foo"}, IsLocal: true}, PathEndpointType, nil},
|
||||
{`C:\\foo`, Endpoint{URL: &url.URL{Path: `C:\\foo`}, IsLocal: true}, PathEndpointType, nil},
|
||||
{"http:path", Endpoint{URL: &url.URL{Path: "http:path"}, IsLocal: true}, PathEndpointType, nil},
|
||||
{"http:/path", Endpoint{URL: &url.URL{Path: "http:/path"}, IsLocal: true}, PathEndpointType, nil},
|
||||
{"http:///path", Endpoint{URL: &url.URL{Path: "http:/path"}, IsLocal: true}, PathEndpointType, nil},
|
||||
{"http://localhost/path", Endpoint{URL: u1, IsLocal: true}, URLEndpointType, nil},
|
||||
{"http://localhost/path//", Endpoint{URL: u1, IsLocal: true}, URLEndpointType, nil},
|
||||
{"https://example.org/path", Endpoint{URL: u2}, URLEndpointType, nil},
|
||||
{"http://127.0.0.1:8080/path", Endpoint{URL: u3, IsLocal: true}, URLEndpointType, nil},
|
||||
{"http://192.168.253.200/path", Endpoint{URL: u4}, URLEndpointType, nil},
|
||||
{"", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
|
||||
{".", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
|
||||
{"/", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
|
||||
{`\`, Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
|
||||
{"c://foo", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format")},
|
||||
{"ftp://foo", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format")},
|
||||
{"http://server/path?location", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format")},
|
||||
{"http://:/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: invalid port number")},
|
||||
{"http://:8080/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: empty host name")},
|
||||
{"http://server:/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: invalid port number")},
|
||||
{"https://93.184.216.34:808080/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535")},
|
||||
{"http://server:8080//", Endpoint{}, -1, fmt.Errorf("empty or root path is not supported in URL endpoint")},
|
||||
{"http://server:8080/", Endpoint{}, -1, fmt.Errorf("empty or root path is not supported in URL endpoint")},
|
||||
{"http://server/path", Endpoint{}, -1, fmt.Errorf("lookup server" + errMsg)},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
endpoint, err := NewEndpoint(testCase.arg)
|
||||
if testCase.expectedErr == nil {
|
||||
if err != nil {
|
||||
t.Fatalf("error: expected = <nil>, got = %v", err)
|
||||
}
|
||||
} else if err == nil {
|
||||
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
|
||||
} else {
|
||||
match := false
|
||||
if strings.HasSuffix(testCase.expectedErr.Error(), errMsg) {
|
||||
match = strings.HasSuffix(err.Error(), errMsg)
|
||||
} else {
|
||||
match = (testCase.expectedErr.Error() == err.Error())
|
||||
}
|
||||
if !match {
|
||||
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err == nil && !reflect.DeepEqual(testCase.expectedEndpoint, endpoint) {
|
||||
t.Fatalf("endpoint: expected = %+v, got = %+v", testCase.expectedEndpoint, endpoint)
|
||||
}
|
||||
|
||||
if err == nil && testCase.expectedType != endpoint.Type() {
|
||||
t.Fatalf("type: expected = %+v, got = %+v", testCase.expectedType, endpoint.Type())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewEndpointList(t *testing.T) {
|
||||
testCases := []struct {
|
||||
args []string
|
||||
expectedErr error
|
||||
}{
|
||||
{[]string{"d1", "d2", "d3", "d4"}, nil},
|
||||
{[]string{"/d1", "/d2", "/d3", "/d4"}, nil},
|
||||
{[]string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}, nil},
|
||||
{[]string{"http://example.org/d1", "http://example.com/d1", "http://example.net/d1", "http://example.edu/d1"}, nil},
|
||||
{[]string{"http://localhost/d1", "http://localhost/d2", "http://example.org/d1", "http://example.org/d2"}, nil},
|
||||
{[]string{"https://localhost:9000/d1", "https://localhost:9001/d2", "https://localhost:9002/d3", "https://localhost:9003/d4"}, nil},
|
||||
// // It is valid WRT endpoint list that same path is expected with different port on same server.
|
||||
{[]string{"https://127.0.0.1:9000/d1", "https://127.0.0.1:9001/d1", "https://127.0.0.1:9002/d1", "https://127.0.0.1:9003/d1"}, nil},
|
||||
{[]string{"d1", "d2", "d3", "d1"}, fmt.Errorf("duplicate endpoints found")},
|
||||
{[]string{"d1", "d2", "d3", "./d1"}, fmt.Errorf("duplicate endpoints found")},
|
||||
{[]string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d1", "http://localhost/d4"}, fmt.Errorf("duplicate endpoints found")},
|
||||
{[]string{"d1", "d2", "d3", "d4", "d5"}, fmt.Errorf("total endpoints 5 found. For XL/Distribute, it should be 4, 6, 8, 10, 12, 14 or 16")},
|
||||
{[]string{"ftp://server/d1", "http://server/d2", "http://server/d3", "http://server/d4"}, fmt.Errorf("'ftp://server/d1': invalid URL endpoint format")},
|
||||
{[]string{"d1", "http://localhost/d2", "d3", "d4"}, fmt.Errorf("mixed style endpoints are not supported")},
|
||||
{[]string{"http://example.org/d1", "https://example.com/d1", "http://example.net/d1", "https://example.edut/d1"}, fmt.Errorf("mixed scheme is not supported")},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
_, err := NewEndpointList(testCase.args...)
|
||||
if testCase.expectedErr == nil {
|
||||
if err != nil {
|
||||
t.Fatalf("error: expected = <nil>, got = %v", err)
|
||||
}
|
||||
} else if err == nil {
|
||||
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
|
||||
} else if testCase.expectedErr.Error() != err.Error() {
|
||||
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateEndpoints(t *testing.T) {
|
||||
case1u1, _ := url.Parse("http://example.com:10000/d4")
|
||||
case1u2, _ := url.Parse("http://example.org:10000/d3")
|
||||
case1u3, _ := url.Parse("http://localhost:10000/d1")
|
||||
case1u4, _ := url.Parse("http://localhost:10000/d2")
|
||||
|
||||
case2u1, _ := url.Parse("http://example.com:10000/d4")
|
||||
case2u2, _ := url.Parse("http://example.org:10000/d3")
|
||||
case2u3, _ := url.Parse("http://localhost:10000/d1")
|
||||
case2u4, _ := url.Parse("http://localhost:9000/d2")
|
||||
|
||||
case3u1, _ := url.Parse("http://example.com:80/d3")
|
||||
case3u2, _ := url.Parse("http://example.net:80/d4")
|
||||
case3u3, _ := url.Parse("http://example.org:9000/d2")
|
||||
case3u4, _ := url.Parse("http://localhost:80/d1")
|
||||
|
||||
case4u1, _ := url.Parse("http://example.com:9000/d3")
|
||||
case4u2, _ := url.Parse("http://example.net:9000/d4")
|
||||
case4u3, _ := url.Parse("http://example.org:9000/d2")
|
||||
case4u4, _ := url.Parse("http://localhost:9000/d1")
|
||||
|
||||
case5u1, _ := url.Parse("http://localhost:9000/d1")
|
||||
case5u2, _ := url.Parse("http://localhost:9001/d2")
|
||||
case5u3, _ := url.Parse("http://localhost:9002/d3")
|
||||
case5u4, _ := url.Parse("http://localhost:9003/d4")
|
||||
|
||||
case6u1, _ := url.Parse("http://10.0.0.1:9000/export")
|
||||
case6u2, _ := url.Parse("http://10.0.0.2:9000/export")
|
||||
case6u3, _ := url.Parse("http://10.0.0.3:9000/export")
|
||||
case6u4, _ := url.Parse("http://localhost:9001/export")
|
||||
|
||||
testCases := []struct {
|
||||
serverAddr string
|
||||
args []string
|
||||
expectedServerAddr string
|
||||
expectedEndpoints EndpointList
|
||||
expectedSetupType SetupType
|
||||
expectedErr error
|
||||
}{
|
||||
{"localhost", []string{}, "", EndpointList{}, -1, fmt.Errorf("missing port in address localhost")},
|
||||
|
||||
// FS Setup
|
||||
{"localhost:9000", []string{"http://localhost/d1"}, "", EndpointList{}, -1, fmt.Errorf("use path style endpoint for FS setup")},
|
||||
{":443", []string{"d1"}, ":443", EndpointList{Endpoint{URL: &url.URL{Path: "d1"}, IsLocal: true}}, FSSetupType, nil},
|
||||
{"localhost:10000", []string{"/d1"}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true}}, FSSetupType, nil},
|
||||
{"localhost:10000", []string{"./d1"}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: "d1"}, IsLocal: true}}, FSSetupType, nil},
|
||||
{"localhost:10000", []string{`\d1`}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: `\d1`}, IsLocal: true}}, FSSetupType, nil},
|
||||
{"localhost:10000", []string{`.\d1`}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: `.\d1`}, IsLocal: true}}, FSSetupType, nil},
|
||||
{":8080", []string{"https://example.org/d1", "https://example.org/d2", "https://example.org/d3", "https://example.org/d4"}, "", EndpointList{}, -1, fmt.Errorf("no endpoint found for this host")},
|
||||
{":8080", []string{"https://example.org/d1", "https://example.com/d2", "https://example.net:8000/d3", "https://example.edu/d1"}, "", EndpointList{}, -1, fmt.Errorf("no endpoint found for this host")},
|
||||
{"localhost:9000", []string{"https://127.0.0.1:9000/d1", "https://localhost:9001/d1", "https://example.com/d1", "https://example.com/d2"}, "", EndpointList{}, -1, fmt.Errorf("path '/d1' can not be served from different address/port")},
|
||||
{"localhost:9000", []string{"https://127.0.0.1:8000/d1", "https://localhost:9001/d2", "https://example.com/d1", "https://example.com/d2"}, "", EndpointList{}, -1, fmt.Errorf("port number in server address must match with one of the port in local endpoints")},
|
||||
{"localhost:10000", []string{"https://127.0.0.1:8000/d1", "https://localhost:8000/d2", "https://example.com/d1", "https://example.com/d2"}, "", EndpointList{}, -1, fmt.Errorf("server address and local endpoint have different ports")},
|
||||
|
||||
// XL Setup with PathEndpointType
|
||||
{":1234", []string{"/d1", "/d2", "d3", "d4"}, ":1234",
|
||||
EndpointList{
|
||||
Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Path: "/d2"}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Path: "d3"}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Path: "d4"}, IsLocal: true},
|
||||
}, XLSetupType, nil},
|
||||
// XL Setup with URLEndpointType
|
||||
{":9000", []string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}, ":9000", EndpointList{
|
||||
Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Path: "/d2"}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Path: "/d3"}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Path: "/d4"}, IsLocal: true},
|
||||
}, XLSetupType, nil},
|
||||
// XL Setup with URLEndpointType having mixed naming to local host.
|
||||
{"127.0.0.1:10000", []string{"http://localhost/d1", "http://localhost/d2", "http://127.0.0.1/d3", "http://127.0.0.1/d4"}, ":10000", EndpointList{
|
||||
Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Path: "/d2"}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Path: "/d3"}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Path: "/d4"}, IsLocal: true},
|
||||
}, XLSetupType, nil},
|
||||
|
||||
// DistXL type
|
||||
{"127.0.0.1:10000", []string{"http://localhost/d1", "http://localhost/d2", "http://example.org/d3", "http://example.com/d4"}, "127.0.0.1:10000", EndpointList{
|
||||
Endpoint{URL: case1u1, IsLocal: false},
|
||||
Endpoint{URL: case1u2, IsLocal: false},
|
||||
Endpoint{URL: case1u3, IsLocal: true},
|
||||
Endpoint{URL: case1u4, IsLocal: true},
|
||||
}, DistXLSetupType, nil},
|
||||
{"127.0.0.1:10000", []string{"http://localhost/d1", "http://localhost:9000/d2", "http://example.org/d3", "http://example.com/d4"}, "127.0.0.1:10000", EndpointList{
|
||||
Endpoint{URL: case2u1, IsLocal: false},
|
||||
Endpoint{URL: case2u2, IsLocal: false},
|
||||
Endpoint{URL: case2u3, IsLocal: true},
|
||||
Endpoint{URL: case2u4, IsLocal: false},
|
||||
}, DistXLSetupType, nil},
|
||||
{":80", []string{"http://localhost/d1", "http://example.org:9000/d2", "http://example.com/d3", "http://example.net/d4"}, ":80", EndpointList{
|
||||
Endpoint{URL: case3u1, IsLocal: false},
|
||||
Endpoint{URL: case3u2, IsLocal: false},
|
||||
Endpoint{URL: case3u3, IsLocal: false},
|
||||
Endpoint{URL: case3u4, IsLocal: true},
|
||||
}, DistXLSetupType, nil},
|
||||
{":9000", []string{"http://localhost/d1", "http://example.org/d2", "http://example.com/d3", "http://example.net/d4"}, ":9000", EndpointList{
|
||||
Endpoint{URL: case4u1, IsLocal: false},
|
||||
Endpoint{URL: case4u2, IsLocal: false},
|
||||
Endpoint{URL: case4u3, IsLocal: false},
|
||||
Endpoint{URL: case4u4, IsLocal: true},
|
||||
}, DistXLSetupType, nil},
|
||||
{":9000", []string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://localhost:9002/d3", "http://localhost:9003/d4"}, ":9000", EndpointList{
|
||||
Endpoint{URL: case5u1, IsLocal: true},
|
||||
Endpoint{URL: case5u2, IsLocal: false},
|
||||
Endpoint{URL: case5u3, IsLocal: false},
|
||||
Endpoint{URL: case5u4, IsLocal: false},
|
||||
}, DistXLSetupType, nil},
|
||||
|
||||
{":9001", []string{"http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export", "http://localhost:9001/export", "http://10.0.0.3:9000/export"}, ":9001", EndpointList{
|
||||
Endpoint{URL: case6u1, IsLocal: false},
|
||||
Endpoint{URL: case6u2, IsLocal: false},
|
||||
Endpoint{URL: case6u3, IsLocal: false},
|
||||
Endpoint{URL: case6u4, IsLocal: true},
|
||||
}, DistXLSetupType, nil},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
serverAddr, endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, testCase.args...)
|
||||
|
||||
if err == nil {
|
||||
if testCase.expectedErr != nil {
|
||||
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
|
||||
} else {
|
||||
if serverAddr != testCase.expectedServerAddr {
|
||||
t.Fatalf("serverAddr: expected = %v, got = %v", testCase.expectedServerAddr, serverAddr)
|
||||
}
|
||||
if !reflect.DeepEqual(endpoints, testCase.expectedEndpoints) {
|
||||
t.Fatalf("endpoints: expected = %v, got = %v", testCase.expectedEndpoints, endpoints)
|
||||
}
|
||||
if setupType != testCase.expectedSetupType {
|
||||
t.Fatalf("setupType: expected = %v, got = %v", testCase.expectedSetupType, setupType)
|
||||
}
|
||||
}
|
||||
} else if testCase.expectedErr == nil {
|
||||
t.Fatalf("error: expected = <nil>, got = %v", err)
|
||||
} else if err.Error() != testCase.expectedErr.Error() {
|
||||
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetRemotePeers(t *testing.T) {
|
||||
tempGlobalMinioPort := globalMinioPort
|
||||
defer func() {
|
||||
globalMinioPort = tempGlobalMinioPort
|
||||
}()
|
||||
globalMinioPort = "9000"
|
||||
|
||||
testCases := []struct {
|
||||
endpointArgs []string
|
||||
expectedResult []string
|
||||
}{
|
||||
{[]string{"/d1", "/d2", "d3", "d4"}, []string{}},
|
||||
{[]string{"http://localhost:9000/d1", "http://localhost:9000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000"}},
|
||||
{[]string{"http://localhost:9000/d1", "http://localhost:10000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000", "localhost:10000"}},
|
||||
{[]string{"http://localhost:9000/d1", "http://example.org:9000/d2", "http://example.com:9000/d3", "http://example.net:9000/d4"}, []string{"example.com:9000", "example.net:9000", "example.org:9000"}},
|
||||
{[]string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://localhost:9002/d3", "http://localhost:9003/d4"}, []string{"localhost:9001", "localhost:9002", "localhost:9003"}},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
endpoints, _ := NewEndpointList(testCase.endpointArgs...)
|
||||
remotePeers := GetRemotePeers(endpoints)
|
||||
if !reflect.DeepEqual(remotePeers, testCase.expectedResult) {
|
||||
t.Fatalf("expected: %v, got: %v", testCase.expectedResult, remotePeers)
|
||||
}
|
||||
}
|
||||
}
|
@ -194,11 +194,7 @@ func TestErasureReadUtils(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
objLayer, _, err := initObjectLayer(endpoints)
|
||||
objLayer, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
|
||||
if err != nil {
|
||||
removeRoots(disks)
|
||||
t.Fatal(err)
|
||||
|
@ -94,6 +94,21 @@ type eventData struct {
|
||||
// New notification event constructs a new notification event message from
|
||||
// input request metadata which completed successfully.
|
||||
func newNotificationEvent(event eventData) NotificationEvent {
|
||||
getResponseOriginEndpointKey := func() string {
|
||||
host := globalMinioHost
|
||||
if host == "" {
|
||||
// FIXME: Send FQDN or hostname of this machine than sending IP address.
|
||||
host = localIP4.ToSlice()[0]
|
||||
}
|
||||
|
||||
scheme := httpScheme
|
||||
if globalIsSSL {
|
||||
scheme = httpsScheme
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s://%s:%s", scheme, host, globalMinioPort)
|
||||
}
|
||||
|
||||
// Fetch the region.
|
||||
region := serverConfig.GetRegion()
|
||||
|
||||
@ -103,14 +118,6 @@ func newNotificationEvent(event eventData) NotificationEvent {
|
||||
// Time when Minio finished processing the request.
|
||||
eventTime := UTCNow()
|
||||
|
||||
// API endpoint is captured here to be returned back
|
||||
// to the client for it to differentiate from which
|
||||
// server the request came from.
|
||||
var apiEndpoint string
|
||||
if len(globalAPIEndpoints) >= 1 {
|
||||
apiEndpoint = globalAPIEndpoints[0]
|
||||
}
|
||||
|
||||
// Fetch a hexadecimal representation of event time in nano seconds.
|
||||
uniqueID := mustGetRequestID(eventTime)
|
||||
|
||||
@ -131,7 +138,7 @@ func newNotificationEvent(event eventData) NotificationEvent {
|
||||
responseRequestIDKey: uniqueID,
|
||||
// Following is a custom response element to indicate
|
||||
// event origin server endpoint.
|
||||
responseOriginEndpointKey: apiEndpoint,
|
||||
responseOriginEndpointKey: getResponseOriginEndpointKey(),
|
||||
},
|
||||
S3: eventMeta{
|
||||
SchemaVersion: eventSchemaVersion,
|
||||
|
@ -19,7 +19,6 @@ package cmd
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
@ -40,11 +39,7 @@ func TestInitEventNotifierFaultyDisks(t *testing.T) {
|
||||
t.Fatal("Unable to create directories for FS backend. ", err)
|
||||
}
|
||||
defer removeRoots(disks)
|
||||
endpoints, err := parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
|
||||
if err != nil {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
@ -97,11 +92,7 @@ func TestInitEventNotifierWithPostgreSQL(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal("Unable to create directories for FS backend. ", err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fs, _, err := initObjectLayer(endpoints)
|
||||
fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
|
||||
if err != nil {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
@ -128,11 +119,7 @@ func TestInitEventNotifierWithNATS(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal("Unable to create directories for FS backend. ", err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fs, _, err := initObjectLayer(endpoints)
|
||||
fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
|
||||
if err != nil {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
@ -159,11 +146,7 @@ func TestInitEventNotifierWithWebHook(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal("Unable to create directories for FS backend. ", err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fs, _, err := initObjectLayer(endpoints)
|
||||
fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
|
||||
if err != nil {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
@ -190,11 +173,7 @@ func TestInitEventNotifierWithAMQP(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal("Unable to create directories for FS backend. ", err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fs, _, err := initObjectLayer(endpoints)
|
||||
fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
|
||||
if err != nil {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
@ -221,11 +200,7 @@ func TestInitEventNotifierWithElasticSearch(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal("Unable to create directories for FS backend. ", err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fs, _, err := initObjectLayer(endpoints)
|
||||
fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
|
||||
if err != nil {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
@ -252,11 +227,7 @@ func TestInitEventNotifierWithRedis(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal("Unable to create directories for FS backend. ", err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fs, _, err := initObjectLayer(endpoints)
|
||||
fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
|
||||
if err != nil {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
@ -276,15 +247,10 @@ func (s *TestPeerRPCServerData) Setup(t *testing.T) {
|
||||
s.testServer = StartTestPeersRPCServer(t, s.serverType)
|
||||
|
||||
// setup port and minio addr
|
||||
host, port, err := net.SplitHostPort(s.testServer.Server.Listener.Addr().String())
|
||||
if err != nil {
|
||||
t.Fatalf("Initialisation error: %v", err)
|
||||
}
|
||||
host, port := mustSplitHostPort(s.testServer.Server.Listener.Addr().String())
|
||||
globalMinioHost = host
|
||||
globalMinioPort = port
|
||||
globalMinioAddr = getLocalAddress(
|
||||
s.testServer.SrvCmdCfg,
|
||||
)
|
||||
globalMinioAddr = getEndpointsLocalAddr(s.testServer.endpoints)
|
||||
|
||||
// initialize the peer client(s)
|
||||
initGlobalS3Peers(s.testServer.Disks)
|
||||
|
@ -273,12 +273,8 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Create an instance of xl backend.
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
@ -309,12 +305,8 @@ func TestFormatXLHealFreshDisksErrorExpected(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Create an instance of xl backend.
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
@ -596,12 +588,8 @@ func TestInitFormatXLErrors(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer removeRoots(fsDirs)
|
||||
endpoints, err := parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Create an instance of xl backend.
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -702,12 +690,8 @@ func TestLoadFormatXLErrs(t *testing.T) {
|
||||
}
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
endpoints, err := parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Create an instance of xl backend.
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -733,11 +717,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
|
||||
}
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -761,11 +741,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
|
||||
}
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -787,11 +763,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
|
||||
}
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -820,13 +792,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err := parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Everything is fine, should return nil
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -842,13 +809,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Disks 0..15 are nil
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -866,13 +828,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// One disk returns Faulty Disk
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -892,13 +849,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// One disk is not found, heal corrupted disks should return nil
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -914,13 +866,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Remove format.json of all disks
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -940,13 +887,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Corrupted format json in one disk
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -976,13 +918,8 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err := parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Everything is fine, should return nil
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -997,13 +934,8 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Disks 0..15 are nil
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -1021,13 +953,8 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// One disk returns Faulty Disk
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -1047,13 +974,8 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// One disk is not found, heal corrupted disks should return nil
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -1069,13 +991,8 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Remove format.json of all disks
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
@ -187,9 +187,6 @@ func gatewayMain(ctx *cli.Context) {
|
||||
fatalIf(aerr, "Failed to start minio server")
|
||||
}()
|
||||
|
||||
apiEndPoints, err := finalizeAPIEndpoints(apiServer.Addr)
|
||||
fatalIf(err, "Unable to finalize API endpoints for %s", apiServer.Addr)
|
||||
|
||||
// Once endpoints are finalized, initialize the new object api.
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = newObject
|
||||
@ -202,7 +199,8 @@ func gatewayMain(ctx *cli.Context) {
|
||||
mode = globalMinioModeGatewayAzure
|
||||
}
|
||||
checkUpdate(mode)
|
||||
printGatewayStartupMessage(apiEndPoints, accessKey, secretKey, backendType)
|
||||
apiEndpoints := getAPIEndpoints(apiServer.Addr)
|
||||
printGatewayStartupMessage(apiEndpoints, accessKey, secretKey, backendType)
|
||||
}
|
||||
|
||||
<-globalServiceDoneCh
|
||||
|
@ -18,7 +18,6 @@ package cmd
|
||||
|
||||
import (
|
||||
"crypto/x509"
|
||||
"net/url"
|
||||
"runtime"
|
||||
"time"
|
||||
|
||||
@ -85,9 +84,6 @@ var (
|
||||
// Holds the host that was passed using --address
|
||||
globalMinioHost = ""
|
||||
|
||||
// Holds the list of API endpoints for a given server.
|
||||
globalAPIEndpoints = []string{}
|
||||
|
||||
// Peer communication struct
|
||||
globalS3Peers = s3Peers{}
|
||||
|
||||
@ -103,8 +99,7 @@ var (
|
||||
// Minio server user agent string.
|
||||
globalServerUserAgent = "Minio/" + ReleaseTag + " (" + runtime.GOOS + "; " + runtime.GOARCH + ")"
|
||||
|
||||
// url.URL endpoints of disks that belong to the object storage.
|
||||
globalEndpoints = []*url.URL{}
|
||||
globalEndpoints EndpointList
|
||||
|
||||
// Global server's network statistics
|
||||
globalConnStats = newConnStats()
|
||||
|
@ -1,69 +0,0 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// byLastOctetValue implements sort.Interface used in sorting a list
|
||||
// of ip address by their last octet value.
|
||||
type byLastOctetValue []net.IP
|
||||
|
||||
func (n byLastOctetValue) Len() int { return len(n) }
|
||||
func (n byLastOctetValue) Swap(i, j int) { n[i], n[j] = n[j], n[i] }
|
||||
func (n byLastOctetValue) Less(i, j int) bool {
|
||||
return []byte(n[i].To4())[3] < []byte(n[j].To4())[3]
|
||||
}
|
||||
|
||||
// getInterfaceIPv4s is synonymous to net.InterfaceAddrs()
|
||||
// returns net.IP IPv4 only representation of the net.Addr.
|
||||
// Additionally the returned list is sorted by their last
|
||||
// octet value.
|
||||
//
|
||||
// [The logic to sort by last octet is implemented to
|
||||
// prefer CIDRs with higher octects, this in-turn skips the
|
||||
// localhost/loopback address to be not preferred as the
|
||||
// first ip on the list. Subsequently this list helps us print
|
||||
// a user friendly message with appropriate values].
|
||||
func getInterfaceIPv4s() ([]net.IP, error) {
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Unable to determine network interface address. %s", err)
|
||||
}
|
||||
// Go through each return network address and collate IPv4 addresses.
|
||||
var nips []net.IP
|
||||
for _, addr := range addrs {
|
||||
if addr.Network() == "ip+net" {
|
||||
var nip net.IP
|
||||
// Attempt to parse the addr through CIDR.
|
||||
nip, _, err = net.ParseCIDR(addr.String())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Unable to parse addrss %s, error %s", addr, err)
|
||||
}
|
||||
// Collect only IPv4 addrs.
|
||||
if nip.To4() != nil {
|
||||
nips = append(nips, nip)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Sort the list of IPs by their last octet value.
|
||||
sort.Sort(sort.Reverse(byLastOctetValue(nips)))
|
||||
return nips, nil
|
||||
}
|
@ -1,31 +0,0 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestGetInterfaceIPv4s(t *testing.T) {
|
||||
ipv4s, err := getInterfaceIPv4s()
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error %s", err)
|
||||
}
|
||||
for _, ip := range ipv4s {
|
||||
if ip.To4() == nil {
|
||||
t.Fatalf("Unexpected expecting only IPv4 addresses only %s", ip)
|
||||
}
|
||||
}
|
||||
}
|
@ -90,9 +90,9 @@ func startLockMaintainence(lockServers []*lockServer) {
|
||||
}
|
||||
|
||||
// Register distributed NS lock handlers.
|
||||
func registerDistNSLockRouter(mux *router.Router, serverConfig serverCmdConfig) error {
|
||||
func registerDistNSLockRouter(mux *router.Router, endpoints EndpointList) error {
|
||||
// Initialize a new set of lock servers.
|
||||
lockServers := newLockServers(serverConfig)
|
||||
lockServers := newLockServers(endpoints)
|
||||
|
||||
// Start lock maintenance from all lock servers.
|
||||
startLockMaintainence(lockServers)
|
||||
@ -102,19 +102,18 @@ func registerDistNSLockRouter(mux *router.Router, serverConfig serverCmdConfig)
|
||||
}
|
||||
|
||||
// Create one lock server for every local storage rpc server.
|
||||
func newLockServers(srvConfig serverCmdConfig) (lockServers []*lockServer) {
|
||||
for _, ep := range srvConfig.endpoints {
|
||||
func newLockServers(endpoints EndpointList) (lockServers []*lockServer) {
|
||||
for _, endpoint := range endpoints {
|
||||
// Initialize new lock server for each local node.
|
||||
if isLocalStorage(ep) {
|
||||
// Create handler for lock RPCs
|
||||
locker := &lockServer{
|
||||
serviceEndpoint: getPath(ep),
|
||||
if endpoint.IsLocal {
|
||||
lockServers = append(lockServers, &lockServer{
|
||||
serviceEndpoint: endpoint.Path,
|
||||
mutex: sync.Mutex{},
|
||||
lockMap: make(map[string][]lockRequesterInfo),
|
||||
}
|
||||
lockServers = append(lockServers, locker)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return lockServers
|
||||
}
|
||||
|
||||
|
@ -17,7 +17,6 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"runtime"
|
||||
"sync"
|
||||
"testing"
|
||||
@ -433,66 +432,46 @@ func TestLockServers(t *testing.T) {
|
||||
globalIsDistXL = currentIsDistXL
|
||||
}()
|
||||
|
||||
case1Endpoints := mustGetNewEndpointList(
|
||||
"http://localhost:9000/mnt/disk1",
|
||||
"http://1.1.1.2:9000/mnt/disk2",
|
||||
"http://1.1.2.1:9000/mnt/disk3",
|
||||
"http://1.1.2.2:9000/mnt/disk4",
|
||||
)
|
||||
for i := range case1Endpoints {
|
||||
if case1Endpoints[i].Host == "localhost:9000" {
|
||||
case1Endpoints[i].IsLocal = true
|
||||
}
|
||||
}
|
||||
|
||||
case2Endpoints := mustGetNewEndpointList(
|
||||
"http://localhost:9000/mnt/disk1",
|
||||
"http://localhost:9000/mnt/disk2",
|
||||
"http://1.1.2.1:9000/mnt/disk3",
|
||||
"http://1.1.2.2:9000/mnt/disk4",
|
||||
)
|
||||
for i := range case2Endpoints {
|
||||
if case2Endpoints[i].Host == "localhost:9000" {
|
||||
case2Endpoints[i].IsLocal = true
|
||||
}
|
||||
}
|
||||
|
||||
globalMinioHost = ""
|
||||
testCases := []struct {
|
||||
isDistXL bool
|
||||
srvCmdConfig serverCmdConfig
|
||||
endpoints EndpointList
|
||||
totalLockServers int
|
||||
}{
|
||||
// Test - 1 one lock server initialized.
|
||||
{
|
||||
isDistXL: true,
|
||||
srvCmdConfig: serverCmdConfig{
|
||||
endpoints: []*url.URL{{
|
||||
Scheme: httpScheme,
|
||||
Host: "localhost:9000",
|
||||
Path: "/mnt/disk1",
|
||||
}, {
|
||||
Scheme: httpScheme,
|
||||
Host: "1.1.1.2:9000",
|
||||
Path: "/mnt/disk2",
|
||||
}, {
|
||||
Scheme: httpScheme,
|
||||
Host: "1.1.2.1:9000",
|
||||
Path: "/mnt/disk3",
|
||||
}, {
|
||||
Scheme: httpScheme,
|
||||
Host: "1.1.2.2:9000",
|
||||
Path: "/mnt/disk4",
|
||||
}},
|
||||
},
|
||||
totalLockServers: 1,
|
||||
},
|
||||
{true, case1Endpoints, 1},
|
||||
// Test - 2 two servers possible.
|
||||
{
|
||||
isDistXL: true,
|
||||
srvCmdConfig: serverCmdConfig{
|
||||
endpoints: []*url.URL{{
|
||||
Scheme: httpScheme,
|
||||
Host: "localhost:9000",
|
||||
Path: "/mnt/disk1",
|
||||
}, {
|
||||
Scheme: httpScheme,
|
||||
Host: "localhost:9000",
|
||||
Path: "/mnt/disk2",
|
||||
}, {
|
||||
Scheme: httpScheme,
|
||||
Host: "1.1.2.1:9000",
|
||||
Path: "/mnt/disk3",
|
||||
}, {
|
||||
Scheme: httpScheme,
|
||||
Host: "1.1.2.2:9000",
|
||||
Path: "/mnt/disk4",
|
||||
}},
|
||||
},
|
||||
totalLockServers: 2,
|
||||
},
|
||||
{true, case2Endpoints, 2},
|
||||
}
|
||||
|
||||
// Validates lock server initialization.
|
||||
for i, testCase := range testCases {
|
||||
globalIsDistXL = testCase.isDistXL
|
||||
lockServers := newLockServers(testCase.srvCmdConfig)
|
||||
lockServers := newLockServers(testCase.endpoints)
|
||||
if len(lockServers) != testCase.totalLockServers {
|
||||
t.Fatalf("Test %d: Expected total %d, got %d", i+1, testCase.totalLockServers, len(lockServers))
|
||||
}
|
||||
|
@ -42,19 +42,16 @@ func initDsyncNodes() error {
|
||||
// Initialize rpc lock client information only if this instance is a distributed setup.
|
||||
clnts := make([]dsync.NetLocker, len(globalEndpoints))
|
||||
myNode := -1
|
||||
for index, ep := range globalEndpoints {
|
||||
if ep == nil {
|
||||
return errInvalidArgument
|
||||
}
|
||||
for index, endpoint := range globalEndpoints {
|
||||
clnts[index] = newLockRPCClient(authConfig{
|
||||
accessKey: cred.AccessKey,
|
||||
secretKey: cred.SecretKey,
|
||||
serverAddr: ep.Host,
|
||||
serverAddr: endpoint.Host,
|
||||
secureConn: globalIsSSL,
|
||||
serviceEndpoint: pathutil.Join(minioReservedBucketPath, lockServicePath, getPath(ep)),
|
||||
serviceEndpoint: pathutil.Join(minioReservedBucketPath, lockServicePath, endpoint.Path),
|
||||
serviceName: lockServiceName,
|
||||
})
|
||||
if isLocalStorage(ep) && myNode == -1 {
|
||||
if endpoint.IsLocal && myNode == -1 {
|
||||
myNode = index
|
||||
}
|
||||
}
|
||||
|
167
cmd/net.go
Normal file
167
cmd/net.go
Normal file
@ -0,0 +1,167 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"syscall"
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
)
|
||||
|
||||
// IPv4 addresses of local host.
|
||||
var localIP4 = mustGetLocalIP4()
|
||||
|
||||
// mustSplitHostPort is a wrapper to net.SplitHostPort() where error is assumed to be a fatal.
|
||||
func mustSplitHostPort(hostPort string) (host, port string) {
|
||||
host, port, err := net.SplitHostPort(hostPort)
|
||||
fatalIf(err, "Unable to split host port %s", hostPort)
|
||||
return host, port
|
||||
}
|
||||
|
||||
// mustGetLocalIP4 returns IPv4 addresses of local host. It panics on error.
|
||||
func mustGetLocalIP4() (ipList set.StringSet) {
|
||||
ipList = set.NewStringSet()
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
fatalIf(err, "Unable to get IP addresses of this host.")
|
||||
|
||||
for _, addr := range addrs {
|
||||
var ip net.IP
|
||||
switch v := addr.(type) {
|
||||
case *net.IPNet:
|
||||
ip = v.IP
|
||||
case *net.IPAddr:
|
||||
ip = v.IP
|
||||
}
|
||||
|
||||
if ip.To4() != nil {
|
||||
ipList.Add(ip.String())
|
||||
}
|
||||
}
|
||||
|
||||
return ipList
|
||||
}
|
||||
|
||||
// getHostIP4 returns IPv4 address of given host.
|
||||
func getHostIP4(host string) (ipList set.StringSet, err error) {
|
||||
ipList = set.NewStringSet()
|
||||
ips, err := net.LookupIP(host)
|
||||
if err != nil {
|
||||
return ipList, err
|
||||
}
|
||||
|
||||
for _, ip := range ips {
|
||||
if ip.To4() != nil {
|
||||
ipList.Add(ip.String())
|
||||
}
|
||||
}
|
||||
|
||||
return ipList, err
|
||||
}
|
||||
|
||||
func getAPIEndpoints(serverAddr string) (apiEndpoints []string) {
|
||||
host, port := mustSplitHostPort(serverAddr)
|
||||
|
||||
var ipList []string
|
||||
if host == "" {
|
||||
ipList = localIP4.ToSlice()
|
||||
} else {
|
||||
ipList = []string{host}
|
||||
}
|
||||
|
||||
sort.Strings(ipList)
|
||||
|
||||
scheme := httpScheme
|
||||
if globalIsSSL {
|
||||
scheme = httpsScheme
|
||||
}
|
||||
|
||||
for _, ip := range ipList {
|
||||
apiEndpoints = append(apiEndpoints, fmt.Sprintf("%s://%s:%s", scheme, ip, port))
|
||||
}
|
||||
|
||||
return apiEndpoints
|
||||
}
|
||||
|
||||
// checkPortAvailability - check if given port is already in use.
|
||||
// Note: The check method tries to listen on given port and closes it.
|
||||
// It is possible to have a disconnected client in this tiny window of time.
|
||||
func checkPortAvailability(port string) (err error) {
|
||||
// Return true if err is "address already in use" error.
|
||||
isAddrInUseErr := func(err error) (b bool) {
|
||||
if opErr, ok := err.(*net.OpError); ok {
|
||||
if sysErr, ok := opErr.Err.(*os.SyscallError); ok {
|
||||
if errno, ok := sysErr.Err.(syscall.Errno); ok {
|
||||
b = (errno == syscall.EADDRINUSE)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return b
|
||||
}
|
||||
|
||||
network := []string{"tcp", "tcp4", "tcp6"}
|
||||
for _, n := range network {
|
||||
l, err := net.Listen(n, net.JoinHostPort("", port))
|
||||
if err == nil {
|
||||
// As we are able to listen on this network, the port is not in use.
|
||||
// Close the listener and continue check other networks.
|
||||
if err = l.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if isAddrInUseErr(err) {
|
||||
// As we got EADDRINUSE error, the port is in use by other process.
|
||||
// Return the error.
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// CheckLocalServerAddr - checks if serverAddr is valid and local host.
|
||||
func CheckLocalServerAddr(serverAddr string) error {
|
||||
host, port, err := net.SplitHostPort(serverAddr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check whether port is a valid port number.
|
||||
p, err := strconv.Atoi(port)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid port number")
|
||||
} else if p < 1 || p > 65535 {
|
||||
return fmt.Errorf("port number must be between 1 to 65535")
|
||||
}
|
||||
|
||||
if host != "" {
|
||||
hostIPs, err := getHostIP4(host)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if localIP4.Intersection(hostIPs).IsEmpty() {
|
||||
return fmt.Errorf("host in server address should be this server")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
183
cmd/net_test.go
Normal file
183
cmd/net_test.go
Normal file
@ -0,0 +1,183 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
)
|
||||
|
||||
func TestMustSplitHostPort(t *testing.T) {
|
||||
testCases := []struct {
|
||||
hostPort string
|
||||
expectedHost string
|
||||
expectedPort string
|
||||
}{
|
||||
{":54321", "", "54321"},
|
||||
{"server:54321", "server", "54321"},
|
||||
{":", "", ""},
|
||||
{":0", "", "0"},
|
||||
{":-10", "", "-10"},
|
||||
{"server:100000000", "server", "100000000"},
|
||||
{"server:https", "server", "https"},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
host, port := mustSplitHostPort(testCase.hostPort)
|
||||
if testCase.expectedHost != host {
|
||||
t.Fatalf("host: expected = %v, got = %v", testCase.expectedHost, host)
|
||||
}
|
||||
|
||||
if testCase.expectedPort != port {
|
||||
t.Fatalf("port: expected = %v, got = %v", testCase.expectedPort, port)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMustGetLocalIP4(t *testing.T) {
|
||||
testCases := []struct {
|
||||
expectedIPList set.StringSet
|
||||
}{
|
||||
{set.CreateStringSet("127.0.0.1")},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
ipList := mustGetLocalIP4()
|
||||
if testCase.expectedIPList != nil && testCase.expectedIPList.Intersection(ipList).IsEmpty() {
|
||||
t.Fatalf("host: expected = %v, got = %v", testCase.expectedIPList, ipList)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetHostIP(t *testing.T) {
|
||||
_, err := getHostIP4("myserver")
|
||||
testCases := []struct {
|
||||
host string
|
||||
expectedIPList set.StringSet
|
||||
expectedErr error
|
||||
}{
|
||||
{"localhost", set.CreateStringSet("127.0.0.1"), nil},
|
||||
{"example.org", set.CreateStringSet("93.184.216.34"), nil},
|
||||
{"myserver", nil, err},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
ipList, err := getHostIP4(testCase.host)
|
||||
if testCase.expectedErr == nil {
|
||||
if err != nil {
|
||||
t.Fatalf("error: expected = <nil>, got = %v", err)
|
||||
}
|
||||
} else if err == nil {
|
||||
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
|
||||
} else if testCase.expectedErr.Error() != err.Error() {
|
||||
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
|
||||
}
|
||||
|
||||
if testCase.expectedIPList != nil && testCase.expectedIPList.Intersection(ipList).IsEmpty() {
|
||||
t.Fatalf("host: expected = %v, got = %v", testCase.expectedIPList, ipList)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Tests finalize api endpoints.
|
||||
func TestGetAPIEndpoints(t *testing.T) {
|
||||
testCases := []struct {
|
||||
serverAddr string
|
||||
expectedResult string
|
||||
}{
|
||||
{":80", "http://127.0.0.1:80"},
|
||||
{"127.0.0.1:80", "http://127.0.0.1:80"},
|
||||
{"localhost:80", "http://localhost:80"},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
apiEndpoints := getAPIEndpoints(testCase.serverAddr)
|
||||
apiEndpointSet := set.CreateStringSet(apiEndpoints...)
|
||||
if !apiEndpointSet.Contains(testCase.expectedResult) {
|
||||
t.Fatalf("test %d: expected: Found, got: Not Found", i+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Tests for port availability logic written for server startup sequence.
|
||||
func TestCheckPortAvailability(t *testing.T) {
|
||||
// Make a port is not available.
|
||||
port := getFreePort()
|
||||
listener, err := net.Listen("tcp", net.JoinHostPort("", port))
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to listen on port %v", port)
|
||||
}
|
||||
defer listener.Close()
|
||||
|
||||
testCases := []struct {
|
||||
port string
|
||||
expectedErr error
|
||||
}{
|
||||
{port, fmt.Errorf("listen tcp :%v: bind: address already in use", port)},
|
||||
{getFreePort(), nil},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
// On MS Windows, skip checking error case due to https://github.com/golang/go/issues/7598
|
||||
if runtime.GOOS == globalWindowsOSName && testCase.expectedErr != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
err := checkPortAvailability(testCase.port)
|
||||
if testCase.expectedErr == nil {
|
||||
if err != nil {
|
||||
t.Fatalf("error: expected = <nil>, got = %v", err)
|
||||
}
|
||||
} else if err == nil {
|
||||
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
|
||||
} else if testCase.expectedErr.Error() != err.Error() {
|
||||
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckLocalServerAddr(t *testing.T) {
|
||||
testCases := []struct {
|
||||
serverAddr string
|
||||
expectedErr error
|
||||
}{
|
||||
{":54321", nil},
|
||||
{"localhost:54321", nil},
|
||||
{"", fmt.Errorf("missing port in address")},
|
||||
{"localhost", fmt.Errorf("missing port in address localhost")},
|
||||
{"example.org:54321", fmt.Errorf("host in server address should be this server")},
|
||||
{":0", fmt.Errorf("port number must be between 1 to 65535")},
|
||||
{":-10", fmt.Errorf("port number must be between 1 to 65535")},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
err := CheckLocalServerAddr(testCase.serverAddr)
|
||||
if testCase.expectedErr == nil {
|
||||
if err != nil {
|
||||
t.Fatalf("error: expected = <nil>, got = %v", err)
|
||||
}
|
||||
} else if err == nil {
|
||||
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
|
||||
} else if testCase.expectedErr.Error() != err.Error() {
|
||||
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
|
||||
}
|
||||
}
|
||||
}
|
@ -17,9 +17,6 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/url"
|
||||
"runtime"
|
||||
"sync"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
@ -132,90 +129,13 @@ func houseKeeping(storageDisks []StorageAPI) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check if a network path is local to this node.
|
||||
func isLocalStorage(ep *url.URL) bool {
|
||||
if ep.Host == "" {
|
||||
return true
|
||||
}
|
||||
if globalMinioHost != "" && globalMinioPort != "" {
|
||||
// if --address host:port was specified for distXL we short
|
||||
// circuit only the endPoint that matches host:port
|
||||
return net.JoinHostPort(globalMinioHost, globalMinioPort) == ep.Host
|
||||
}
|
||||
// Split host to extract host information.
|
||||
host, _, err := net.SplitHostPort(ep.Host)
|
||||
if err != nil {
|
||||
errorIf(err, "Cannot split host port")
|
||||
return false
|
||||
}
|
||||
// Resolve host to address to check if the IP is loopback.
|
||||
// If address resolution fails, assume it's a non-local host.
|
||||
addrs, err := net.LookupHost(host)
|
||||
if err != nil {
|
||||
errorIf(err, "Failed to lookup host")
|
||||
return false
|
||||
}
|
||||
for _, addr := range addrs {
|
||||
if ip := net.ParseIP(addr); ip.IsLoopback() {
|
||||
return true
|
||||
}
|
||||
}
|
||||
iaddrs, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to list interface addresses")
|
||||
return false
|
||||
}
|
||||
for _, addr := range addrs {
|
||||
for _, iaddr := range iaddrs {
|
||||
ip, _, err := net.ParseCIDR(iaddr.String())
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to parse CIDR")
|
||||
return false
|
||||
}
|
||||
if ip.String() == addr {
|
||||
return true
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Fetch the path component from *url.URL*.
|
||||
func getPath(ep *url.URL) string {
|
||||
if ep == nil {
|
||||
return ""
|
||||
}
|
||||
var diskPath string
|
||||
// For windows ep.Path is usually empty
|
||||
if runtime.GOOS == globalWindowsOSName {
|
||||
switch ep.Scheme {
|
||||
case "":
|
||||
// Eg. "minio server .\export"
|
||||
diskPath = ep.Path
|
||||
case httpScheme, httpsScheme:
|
||||
// For full URLs windows drive is part of URL path.
|
||||
// Eg: http://ip:port/C:\mydrive
|
||||
// For windows trim off the preceding "/".
|
||||
diskPath = ep.Path[1:]
|
||||
default:
|
||||
// For the rest url splits drive letter into
|
||||
// Scheme contruct the disk path back.
|
||||
diskPath = ep.Scheme + ":" + ep.Opaque
|
||||
}
|
||||
} else {
|
||||
// For other operating systems ep.Path is non empty.
|
||||
diskPath = ep.Path
|
||||
}
|
||||
return diskPath
|
||||
}
|
||||
|
||||
// Depending on the disk type network or local, initialize storage API.
|
||||
func newStorageAPI(ep *url.URL) (storage StorageAPI, err error) {
|
||||
if isLocalStorage(ep) {
|
||||
return newPosix(getPath(ep))
|
||||
func newStorageAPI(endpoint Endpoint) (storage StorageAPI, err error) {
|
||||
if endpoint.IsLocal {
|
||||
return newPosix(endpoint.Path)
|
||||
}
|
||||
return newStorageRPC(ep)
|
||||
|
||||
return newStorageRPC(endpoint), nil
|
||||
}
|
||||
|
||||
var initMetaVolIgnoredErrs = append(baseIgnoredErrs, errVolumeExists)
|
||||
|
@ -17,7 +17,6 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
@ -98,53 +97,3 @@ func TestHouseKeeping(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test getPath() - the path that needs to be passed to newPosix()
|
||||
func TestGetPath(t *testing.T) {
|
||||
globalMinioHost = ""
|
||||
var testCases []struct {
|
||||
epStr string
|
||||
path string
|
||||
}
|
||||
if runtime.GOOS == globalWindowsOSName {
|
||||
testCases = []struct {
|
||||
epStr string
|
||||
path string
|
||||
}{
|
||||
{"\\export", "\\export"},
|
||||
{"D:\\export", "d:\\export"},
|
||||
{"D:\\", "d:\\"},
|
||||
{"D:", "d:"},
|
||||
{"\\", "\\"},
|
||||
{"http://127.0.0.1/d:/export", "d:/export"},
|
||||
{"https://127.0.0.1/d:/export", "d:/export"},
|
||||
}
|
||||
} else {
|
||||
testCases = []struct {
|
||||
epStr string
|
||||
path string
|
||||
}{
|
||||
{"/export", "/export"},
|
||||
{"http://127.0.0.1/export", "/export"},
|
||||
{"https://127.0.0.1/export", "/export"},
|
||||
}
|
||||
}
|
||||
testCasesCommon := []struct {
|
||||
epStr string
|
||||
path string
|
||||
}{
|
||||
{"export", "export"},
|
||||
}
|
||||
testCases = append(testCases, testCasesCommon...)
|
||||
for i, test := range testCases {
|
||||
eps, err := parseStorageEndpoints([]string{test.epStr})
|
||||
if err != nil {
|
||||
t.Errorf("Test %d: %s - %s", i+1, test.epStr, err)
|
||||
continue
|
||||
}
|
||||
path := getPath(eps[0])
|
||||
if path != test.path {
|
||||
t.Errorf("Test %d: For endpoing %s, getPath() failed, got: %s, expected: %s,", i+1, test.epStr, path, test.path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -18,8 +18,6 @@ package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"sync"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
@ -52,45 +50,11 @@ func printOnceFn() printOnceFunc {
|
||||
}
|
||||
|
||||
// Prints custom message when healing is required for XL and Distributed XL backend.
|
||||
func printHealMsg(endpoints []*url.URL, storageDisks []StorageAPI, fn printOnceFunc) {
|
||||
func printHealMsg(endpoints EndpointList, storageDisks []StorageAPI, fn printOnceFunc) {
|
||||
msg := getHealMsg(endpoints, storageDisks)
|
||||
fn(msg)
|
||||
}
|
||||
|
||||
// Heal endpoint constructs the final endpoint URL for control heal command.
|
||||
// Disk heal endpoint needs to be just a URL and no special paths.
|
||||
// This function constructs the right endpoint under various conditions
|
||||
// for single node XL, distributed XL and when minio server is bound
|
||||
// to a specific ip:port.
|
||||
func getHealEndpoint(tls bool, firstEndpoint *url.URL) (cEndpoint *url.URL) {
|
||||
scheme := httpScheme
|
||||
if tls {
|
||||
scheme = httpsScheme
|
||||
}
|
||||
cEndpoint = &url.URL{
|
||||
Scheme: scheme,
|
||||
}
|
||||
// Bind to `--address host:port` was specified.
|
||||
if globalMinioHost != "" {
|
||||
cEndpoint.Host = net.JoinHostPort(globalMinioHost, globalMinioPort)
|
||||
return cEndpoint
|
||||
}
|
||||
// For distributed XL setup.
|
||||
if firstEndpoint.Host != "" {
|
||||
cEndpoint.Host = firstEndpoint.Host
|
||||
return cEndpoint
|
||||
}
|
||||
// For single node XL setup, we need to find the endpoint.
|
||||
cEndpoint.Host = globalMinioAddr
|
||||
// Fetch all the listening ips. For single node XL we
|
||||
// just use the first host.
|
||||
hosts, _, err := getListenIPs(cEndpoint.Host)
|
||||
if err == nil {
|
||||
cEndpoint.Host = net.JoinHostPort(hosts[0], globalMinioPort)
|
||||
}
|
||||
return cEndpoint
|
||||
}
|
||||
|
||||
// Disks offline and online strings..
|
||||
const (
|
||||
diskOffline = "offline"
|
||||
@ -100,7 +64,7 @@ const (
|
||||
// Constructs a formatted heal message, when cluster is found to be in state where it requires healing.
|
||||
// healing is optional, server continues to initialize object layer after printing this message.
|
||||
// it is upto the end user to perform a heal if needed.
|
||||
func getHealMsg(endpoints []*url.URL, storageDisks []StorageAPI) string {
|
||||
func getHealMsg(endpoints EndpointList, storageDisks []StorageAPI) string {
|
||||
healFmtCmd := `"mc admin heal myminio"`
|
||||
msg := fmt.Sprintf("New disk(s) were found, format them by running - %s\n",
|
||||
healFmtCmd)
|
||||
@ -126,13 +90,13 @@ func getHealMsg(endpoints []*url.URL, storageDisks []StorageAPI) string {
|
||||
}
|
||||
|
||||
// Prints regular message when we have sufficient disks to start the cluster.
|
||||
func printRegularMsg(endpoints []*url.URL, storageDisks []StorageAPI, fn printOnceFunc) {
|
||||
func printRegularMsg(endpoints EndpointList, storageDisks []StorageAPI, fn printOnceFunc) {
|
||||
msg := getStorageInitMsg("\nInitializing data volume.", endpoints, storageDisks)
|
||||
fn(msg)
|
||||
}
|
||||
|
||||
// Constructs a formatted regular message when we have sufficient disks to start the cluster.
|
||||
func getStorageInitMsg(titleMsg string, endpoints []*url.URL, storageDisks []StorageAPI) string {
|
||||
func getStorageInitMsg(titleMsg string, endpoints EndpointList, storageDisks []StorageAPI) string {
|
||||
msg := colorBlue(titleMsg)
|
||||
disksInfo, _, _ := getDisksInfo(storageDisks)
|
||||
for i, info := range disksInfo {
|
||||
@ -156,7 +120,7 @@ func getStorageInitMsg(titleMsg string, endpoints []*url.URL, storageDisks []Sto
|
||||
}
|
||||
|
||||
// Prints initialization message when cluster is being initialized for the first time.
|
||||
func printFormatMsg(endpoints []*url.URL, storageDisks []StorageAPI, fn printOnceFunc) {
|
||||
func printFormatMsg(endpoints EndpointList, storageDisks []StorageAPI, fn printOnceFunc) {
|
||||
msg := getStorageInitMsg("\nInitializing data volume for the first time.", endpoints, storageDisks)
|
||||
fn(msg)
|
||||
}
|
||||
|
@ -17,60 +17,10 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"reflect"
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Tests and validates the output for heal endpoint.
|
||||
func TestGetHealEndpoint(t *testing.T) {
|
||||
// Test for a SSL scheme.
|
||||
tls := true
|
||||
hURL := getHealEndpoint(tls, &url.URL{
|
||||
Scheme: httpScheme,
|
||||
Host: "localhost:9000",
|
||||
})
|
||||
sHURL := &url.URL{
|
||||
Scheme: httpsScheme,
|
||||
Host: "localhost:9000",
|
||||
}
|
||||
if !reflect.DeepEqual(hURL, sHURL) {
|
||||
t.Fatalf("Expected %#v, but got %#v", sHURL, hURL)
|
||||
}
|
||||
|
||||
// Test a non-TLS scheme.
|
||||
tls = false
|
||||
hURL = getHealEndpoint(tls, &url.URL{
|
||||
Scheme: httpsScheme,
|
||||
Host: "localhost:9000",
|
||||
})
|
||||
sHURL = &url.URL{
|
||||
Scheme: httpScheme,
|
||||
Host: "localhost:9000",
|
||||
}
|
||||
if !reflect.DeepEqual(hURL, sHURL) {
|
||||
t.Fatalf("Expected %#v, but got %#v", sHURL, hURL)
|
||||
}
|
||||
|
||||
// FIXME(GLOBAL): purposefully Host is left empty because
|
||||
// we need to bring in safe handling on global values
|
||||
// add a proper test case here once that happens.
|
||||
/*
|
||||
tls = false
|
||||
hURL = getHealEndpoint(tls, &url.URL{
|
||||
Path: "/export",
|
||||
})
|
||||
sHURL = &url.URL{
|
||||
Scheme: httpScheme,
|
||||
Host: "",
|
||||
}
|
||||
globalMinioAddr = ""
|
||||
if !reflect.DeepEqual(hURL, sHURL) {
|
||||
t.Fatalf("Expected %#v, but got %#v", sHURL, hURL)
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
||||
// Tests heal message to be correct and properly formatted.
|
||||
func TestHealMsg(t *testing.T) {
|
||||
rootPath, err := newTestConfig(globalMinioDefaultRegion)
|
||||
@ -85,39 +35,26 @@ func TestHealMsg(t *testing.T) {
|
||||
nilDisks[5] = nil
|
||||
authErrs := make([]error, len(storageDisks))
|
||||
authErrs[5] = errAuthentication
|
||||
endpointURL, err := url.Parse("http://10.1.10.1:9000")
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected error:", err)
|
||||
}
|
||||
endpointURLs := make([]*url.URL, len(storageDisks))
|
||||
for idx := 0; idx < len(endpointURLs); idx++ {
|
||||
endpointURLs[idx] = endpointURL
|
||||
|
||||
args := []string{}
|
||||
for i := range storageDisks {
|
||||
args = append(args, fmt.Sprintf("http://10.1.10.%d:9000/d1", i+1))
|
||||
}
|
||||
endpoints := mustGetNewEndpointList(args...)
|
||||
|
||||
testCases := []struct {
|
||||
endPoints []*url.URL
|
||||
endPoints EndpointList
|
||||
storageDisks []StorageAPI
|
||||
serrs []error
|
||||
}{
|
||||
// Test - 1 for valid disks and errors.
|
||||
{
|
||||
endPoints: endpointURLs,
|
||||
storageDisks: storageDisks,
|
||||
serrs: errs,
|
||||
},
|
||||
{endpoints, storageDisks, errs},
|
||||
// Test - 2 for one of the disks is nil.
|
||||
{
|
||||
endPoints: endpointURLs,
|
||||
storageDisks: nilDisks,
|
||||
serrs: errs,
|
||||
},
|
||||
{endpoints, nilDisks, errs},
|
||||
// Test - 3 for one of the errs is authentication.
|
||||
{
|
||||
endPoints: endpointURLs,
|
||||
storageDisks: nilDisks,
|
||||
serrs: authErrs,
|
||||
},
|
||||
{endpoints, nilDisks, authErrs},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
msg := getHealMsg(testCase.endPoints, testCase.storageDisks)
|
||||
if msg == "" {
|
||||
|
@ -18,7 +18,6 @@ package cmd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/minio/mc/pkg/console"
|
||||
@ -191,7 +190,7 @@ func printRetryMsg(sErrs []error, storageDisks []StorageAPI) {
|
||||
|
||||
// Implements a jitter backoff loop for formatting all disks during
|
||||
// initialization of the server.
|
||||
func retryFormattingXLDisks(firstDisk bool, endpoints []*url.URL, storageDisks []StorageAPI) error {
|
||||
func retryFormattingXLDisks(firstDisk bool, endpoints EndpointList, storageDisks []StorageAPI) error {
|
||||
if len(endpoints) == 0 {
|
||||
return errInvalidArgument
|
||||
}
|
||||
@ -276,16 +275,13 @@ func retryFormattingXLDisks(firstDisk bool, endpoints []*url.URL, storageDisks [
|
||||
}
|
||||
|
||||
// Initialize storage disks based on input arguments.
|
||||
func initStorageDisks(endpoints []*url.URL) ([]StorageAPI, error) {
|
||||
func initStorageDisks(endpoints EndpointList) ([]StorageAPI, error) {
|
||||
// Bootstrap disks.
|
||||
storageDisks := make([]StorageAPI, len(endpoints))
|
||||
for index, ep := range endpoints {
|
||||
if ep == nil {
|
||||
return nil, errInvalidArgument
|
||||
}
|
||||
for index, endpoint := range endpoints {
|
||||
// Intentionally ignore disk not found errors. XL is designed
|
||||
// to handle these errors internally.
|
||||
storage, err := newStorageAPI(ep)
|
||||
storage, err := newStorageAPI(endpoint)
|
||||
if err != nil && err != errDiskNotFound {
|
||||
return nil, err
|
||||
}
|
||||
@ -295,14 +291,10 @@ func initStorageDisks(endpoints []*url.URL) ([]StorageAPI, error) {
|
||||
}
|
||||
|
||||
// Format disks before initialization of object layer.
|
||||
func waitForFormatXLDisks(firstDisk bool, endpoints []*url.URL, storageDisks []StorageAPI) (formattedDisks []StorageAPI, err error) {
|
||||
func waitForFormatXLDisks(firstDisk bool, endpoints EndpointList, storageDisks []StorageAPI) (formattedDisks []StorageAPI, err error) {
|
||||
if len(endpoints) == 0 {
|
||||
return nil, errInvalidArgument
|
||||
}
|
||||
firstEndpoint := endpoints[0]
|
||||
if firstEndpoint == nil {
|
||||
return nil, errInvalidArgument
|
||||
}
|
||||
if storageDisks == nil {
|
||||
return nil, errInvalidArgument
|
||||
}
|
||||
|
@ -30,15 +30,15 @@ func newObjectLayerFn() (layer ObjectLayer) {
|
||||
}
|
||||
|
||||
// Composed function registering routers for only distributed XL setup.
|
||||
func registerDistXLRouters(mux *router.Router, srvCmdConfig serverCmdConfig) error {
|
||||
func registerDistXLRouters(mux *router.Router, endpoints EndpointList) error {
|
||||
// Register storage rpc router only if its a distributed setup.
|
||||
err := registerStorageRPCRouters(mux, srvCmdConfig)
|
||||
err := registerStorageRPCRouters(mux, endpoints)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Register distributed namespace lock.
|
||||
err = registerDistNSLockRouter(mux, srvCmdConfig)
|
||||
err = registerDistNSLockRouter(mux, endpoints)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -54,14 +54,14 @@ func registerDistXLRouters(mux *router.Router, srvCmdConfig serverCmdConfig) err
|
||||
}
|
||||
|
||||
// configureServer handler returns final handler for the http server.
|
||||
func configureServerHandler(srvCmdConfig serverCmdConfig) (http.Handler, error) {
|
||||
func configureServerHandler(endpoints EndpointList) (http.Handler, error) {
|
||||
// Initialize router. `SkipClean(true)` stops gorilla/mux from
|
||||
// normalizing URL path minio/minio#3256
|
||||
mux := router.NewRouter().SkipClean(true)
|
||||
|
||||
// Initialize distributed NS lock.
|
||||
if globalIsDistXL {
|
||||
registerDistXLRouters(mux, srvCmdConfig)
|
||||
registerDistXLRouters(mux, endpoints)
|
||||
}
|
||||
|
||||
// Add Admin RPC router
|
||||
|
@ -19,9 +19,11 @@ package cmd
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"net"
|
||||
"path"
|
||||
"sync"
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
)
|
||||
|
||||
// s3Peer structs contains the address of a peer in the cluster, and
|
||||
@ -39,53 +41,44 @@ type s3Peers []s3Peer
|
||||
// makeS3Peers makes an s3Peers struct value from the given urls
|
||||
// slice. The urls slice is assumed to be non-empty and free of nil
|
||||
// values.
|
||||
func makeS3Peers(eps []*url.URL) s3Peers {
|
||||
var ret []s3Peer
|
||||
|
||||
// map to store peers that are already added to ret
|
||||
seenAddr := make(map[string]bool)
|
||||
|
||||
// add local (self) as peer in the array
|
||||
ret = append(ret, s3Peer{
|
||||
globalMinioAddr,
|
||||
func makeS3Peers(endpoints EndpointList) (s3PeerList s3Peers) {
|
||||
thisPeer := globalMinioAddr
|
||||
if globalMinioHost == "" {
|
||||
thisPeer = net.JoinHostPort("localhost", globalMinioPort)
|
||||
}
|
||||
s3PeerList = append(s3PeerList, s3Peer{
|
||||
thisPeer,
|
||||
&localBucketMetaState{ObjectAPI: newObjectLayerFn},
|
||||
})
|
||||
seenAddr[globalMinioAddr] = true
|
||||
|
||||
serverCred := serverConfig.GetCredential()
|
||||
// iterate over endpoints to find new remote peers and add
|
||||
// them to ret.
|
||||
for _, ep := range eps {
|
||||
if ep.Host == "" {
|
||||
hostSet := set.CreateStringSet(globalMinioAddr)
|
||||
cred := serverConfig.GetCredential()
|
||||
serviceEndpoint := path.Join(minioReservedBucketPath, s3Path)
|
||||
for _, host := range GetRemotePeers(endpoints) {
|
||||
if hostSet.Contains(host) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if the remote host has been added already
|
||||
if !seenAddr[ep.Host] {
|
||||
cfg := authConfig{
|
||||
accessKey: serverCred.AccessKey,
|
||||
secretKey: serverCred.SecretKey,
|
||||
serverAddr: ep.Host,
|
||||
serviceEndpoint: path.Join(minioReservedBucketPath, s3Path),
|
||||
hostSet.Add(host)
|
||||
s3PeerList = append(s3PeerList, s3Peer{
|
||||
addr: host,
|
||||
bmsClient: &remoteBucketMetaState{newAuthRPCClient(authConfig{
|
||||
accessKey: cred.AccessKey,
|
||||
secretKey: cred.SecretKey,
|
||||
serverAddr: host,
|
||||
serviceEndpoint: serviceEndpoint,
|
||||
secureConn: globalIsSSL,
|
||||
serviceName: "S3",
|
||||
}
|
||||
|
||||
ret = append(ret, s3Peer{
|
||||
addr: ep.Host,
|
||||
bmsClient: &remoteBucketMetaState{newAuthRPCClient(cfg)},
|
||||
})
|
||||
seenAddr[ep.Host] = true
|
||||
}
|
||||
})},
|
||||
})
|
||||
}
|
||||
|
||||
return ret
|
||||
return s3PeerList
|
||||
}
|
||||
|
||||
// initGlobalS3Peers - initialize globalS3Peers by passing in
|
||||
// endpoints - intended to be called early in program start-up.
|
||||
func initGlobalS3Peers(eps []*url.URL) {
|
||||
globalS3Peers = makeS3Peers(eps)
|
||||
func initGlobalS3Peers(endpoints EndpointList) {
|
||||
globalS3Peers = makeS3Peers(endpoints)
|
||||
}
|
||||
|
||||
// GetPeerClient - fetch BucketMetaState interface by peer address
|
||||
|
@ -17,7 +17,6 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
@ -35,12 +34,12 @@ func TestMakeS3Peers(t *testing.T) {
|
||||
// test cases
|
||||
testCases := []struct {
|
||||
gMinioAddr string
|
||||
eps []*url.URL
|
||||
eps EndpointList
|
||||
peers []string
|
||||
}{
|
||||
{":9000", []*url.URL{{Path: "/mnt/disk1"}}, []string{":9000"}},
|
||||
{":9000", []*url.URL{{Host: "localhost:9001"}}, []string{":9000", "localhost:9001"}},
|
||||
{"m1:9000", []*url.URL{{Host: "m1:9000"}, {Host: "m2:9000"}, {Host: "m3:9000"}}, []string{"m1:9000", "m2:9000", "m3:9000"}},
|
||||
{"127.0.0.1:9000", mustGetNewEndpointList("/mnt/disk1"), []string{"127.0.0.1:9000"}},
|
||||
{"127.0.0.1:9000", mustGetNewEndpointList("http://localhost:9001/d1"), []string{"127.0.0.1:9000", "localhost:9001"}},
|
||||
{"example.org:9000", mustGetNewEndpointList("http://example.org:9000/d1", "http://example.com:9000/d1", "http://example.net:9000/d1", "http://example.edu:9000/d1"), []string{"example.org:9000", "example.com:9000", "example.edu:9000", "example.net:9000"}},
|
||||
}
|
||||
|
||||
getPeersHelper := func(s3p s3Peers) []string {
|
||||
|
@ -18,19 +18,12 @@ package cmd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"runtime"
|
||||
|
||||
"github.com/minio/cli"
|
||||
)
|
||||
|
||||
@ -110,249 +103,6 @@ func enableLoggers() {
|
||||
log.SetConsoleTarget(consoleLogTarget)
|
||||
}
|
||||
|
||||
type serverCmdConfig struct {
|
||||
serverAddr string
|
||||
endpoints []*url.URL
|
||||
}
|
||||
|
||||
// Parse an array of end-points (from the command line)
|
||||
func parseStorageEndpoints(eps []string) (endpoints []*url.URL, err error) {
|
||||
for _, ep := range eps {
|
||||
if ep == "" {
|
||||
return nil, errInvalidArgument
|
||||
}
|
||||
var u *url.URL
|
||||
u, err = url.Parse(ep)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if u.Host != "" {
|
||||
_, port, err := net.SplitHostPort(u.Host)
|
||||
// Ignore the missing port error as the default port can be globalMinioPort.
|
||||
if err != nil && !strings.Contains(err.Error(), "missing port in address") {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if globalMinioHost == "" {
|
||||
// For ex.: minio server host1:port1 host2:port2...
|
||||
// we return error as port is configurable only
|
||||
// using "--address :port"
|
||||
if port != "" {
|
||||
return nil, fmt.Errorf("Invalid Argument %s, port configurable using --address :<port>", u.Host)
|
||||
}
|
||||
u.Host = net.JoinHostPort(u.Host, globalMinioPort)
|
||||
} else {
|
||||
// For ex.: minio server --address host:port host1:port1 host2:port2...
|
||||
// i.e if "--address host:port" is specified
|
||||
// port info in u.Host is mandatory else return error.
|
||||
if port == "" {
|
||||
return nil, fmt.Errorf("Invalid Argument %s, port mandatory when --address <host>:<port> is used", u.Host)
|
||||
}
|
||||
}
|
||||
}
|
||||
endpoints = append(endpoints, u)
|
||||
}
|
||||
return endpoints, nil
|
||||
}
|
||||
|
||||
// Validate if input disks are sufficient for initializing XL.
|
||||
func checkSufficientDisks(eps []*url.URL) error {
|
||||
// Verify total number of disks.
|
||||
total := len(eps)
|
||||
if total > maxErasureBlocks {
|
||||
return errXLMaxDisks
|
||||
}
|
||||
if total < minErasureBlocks {
|
||||
return errXLMinDisks
|
||||
}
|
||||
|
||||
// isEven function to verify if a given number if even.
|
||||
isEven := func(number int) bool {
|
||||
return number%2 == 0
|
||||
}
|
||||
|
||||
// Verify if we have even number of disks.
|
||||
// only combination of 4, 6, 8, 10, 12, 14, 16 are supported.
|
||||
if !isEven(total) {
|
||||
return errXLNumDisks
|
||||
}
|
||||
|
||||
// Success.
|
||||
return nil
|
||||
}
|
||||
|
||||
// Returns if slice of disks is a distributed setup.
|
||||
func isDistributedSetup(eps []*url.URL) bool {
|
||||
// Validate if one the disks is not local.
|
||||
for _, ep := range eps {
|
||||
if !isLocalStorage(ep) {
|
||||
// One or more disks supplied as arguments are
|
||||
// not attached to the local node.
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Returns true if path is empty, or equals to '.', '/', '\' characters.
|
||||
func isPathSentinel(path string) bool {
|
||||
return path == "" || path == "." || path == "/" || path == `\`
|
||||
}
|
||||
|
||||
// Returned when path is empty or root path.
|
||||
var errEmptyRootPath = errors.New("Empty or root path is not allowed")
|
||||
|
||||
// Invalid scheme passed.
|
||||
var errInvalidScheme = errors.New("Invalid scheme")
|
||||
|
||||
// Check if endpoint is in expected syntax by valid scheme/path across all platforms.
|
||||
func checkEndpointURL(endpointURL *url.URL) (err error) {
|
||||
// Applicable to all OS.
|
||||
if endpointURL.Scheme == "" || endpointURL.Scheme == httpScheme || endpointURL.Scheme == httpsScheme {
|
||||
if isPathSentinel(path.Clean(endpointURL.Path)) {
|
||||
err = errEmptyRootPath
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Applicable to Windows only.
|
||||
if runtime.GOOS == globalWindowsOSName {
|
||||
// On Windows, endpoint can be a path with drive eg. C:\Export and its URL.Scheme is 'C'.
|
||||
// Check if URL.Scheme is a single letter alphabet to represent a drive.
|
||||
// Note: URL.Parse() converts scheme into lower case always.
|
||||
if len(endpointURL.Scheme) == 1 && endpointURL.Scheme[0] >= 'a' && endpointURL.Scheme[0] <= 'z' {
|
||||
// If endpoint is C:\ or C:\export, URL.Path does not have path information like \ or \export
|
||||
// hence we directly work with endpoint.
|
||||
if isPathSentinel(strings.SplitN(path.Clean(endpointURL.String()), ":", 2)[1]) {
|
||||
err = errEmptyRootPath
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return errInvalidScheme
|
||||
}
|
||||
|
||||
// Check if endpoints are in expected syntax by valid scheme/path across all platforms.
|
||||
func checkEndpointsSyntax(eps []*url.URL, disks []string) error {
|
||||
for i, u := range eps {
|
||||
if err := checkEndpointURL(u); err != nil {
|
||||
return fmt.Errorf("%s: %s (%s)", err.Error(), u.Path, disks[i])
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Make sure all the command line parameters are OK and exit in case of invalid parameters.
|
||||
func checkServerSyntax(endpoints []*url.URL, disks []string) {
|
||||
// Validate if endpoints follow the expected syntax.
|
||||
err := checkEndpointsSyntax(endpoints, disks)
|
||||
fatalIf(err, "Invalid endpoints found %s", strings.Join(disks, " "))
|
||||
|
||||
// Validate for duplicate endpoints are supplied.
|
||||
err = checkDuplicateEndpoints(endpoints)
|
||||
fatalIf(err, "Duplicate entries in %s", strings.Join(disks, " "))
|
||||
|
||||
if len(endpoints) > 1 {
|
||||
// Validate if we have sufficient disks for XL setup.
|
||||
err = checkSufficientDisks(endpoints)
|
||||
fatalIf(err, "Insufficient number of disks.")
|
||||
} else {
|
||||
// Validate if we have invalid disk for FS setup.
|
||||
if endpoints[0].Host != "" && endpoints[0].Scheme != "" {
|
||||
fatalIf(errInvalidArgument, "%s, FS setup expects a filesystem path", endpoints[0])
|
||||
}
|
||||
}
|
||||
|
||||
if !isDistributedSetup(endpoints) {
|
||||
// for FS and singlenode-XL validation is done, return.
|
||||
return
|
||||
}
|
||||
|
||||
// Rest of the checks applies only to distributed XL setup.
|
||||
if globalMinioHost != "" {
|
||||
// We are here implies --address host:port is passed, hence the user is trying
|
||||
// to run one minio process per export disk.
|
||||
if globalMinioPort == "" {
|
||||
fatalIf(errInvalidArgument, "Port missing, Host:Port should be specified for --address")
|
||||
}
|
||||
foundCnt := 0
|
||||
for _, ep := range endpoints {
|
||||
if ep.Host == globalMinioAddr {
|
||||
foundCnt++
|
||||
}
|
||||
}
|
||||
if foundCnt == 0 {
|
||||
// --address host:port should be available in the XL disk list.
|
||||
fatalIf(errInvalidArgument, "%s is not available in %s", globalMinioAddr, strings.Join(disks, " "))
|
||||
}
|
||||
if foundCnt > 1 {
|
||||
// --address host:port should match exactly one entry in the XL disk list.
|
||||
fatalIf(errInvalidArgument, "%s matches % entries in %s", globalMinioAddr, foundCnt, strings.Join(disks, " "))
|
||||
}
|
||||
}
|
||||
|
||||
for _, ep := range endpoints {
|
||||
if ep.Scheme == httpsScheme && !globalIsSSL {
|
||||
// Certificates should be provided for https configuration.
|
||||
fatalIf(errInvalidArgument, "Certificates not provided for secure configuration")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Checks if any of the endpoints supplied is local to this server.
|
||||
func isAnyEndpointLocal(eps []*url.URL) bool {
|
||||
anyLocalEp := false
|
||||
for _, ep := range eps {
|
||||
if isLocalStorage(ep) {
|
||||
anyLocalEp = true
|
||||
break
|
||||
}
|
||||
}
|
||||
return anyLocalEp
|
||||
}
|
||||
|
||||
// Returned when there are no ports.
|
||||
var errEmptyPort = errors.New("Port cannot be empty or '0', please use `--address` to pick a specific port")
|
||||
|
||||
// Convert an input address of form host:port into, host and port, returns if any.
|
||||
func getHostPort(address string) (host, port string, err error) {
|
||||
// Check if requested port is available.
|
||||
host, port, err = net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
// Empty ports.
|
||||
if port == "0" || port == "" {
|
||||
// Port zero or empty means use requested to choose any freely available
|
||||
// port. Avoid this since it won't work with any configured clients,
|
||||
// can lead to serious loss of availability.
|
||||
return "", "", errEmptyPort
|
||||
}
|
||||
|
||||
// Parse port.
|
||||
if _, err = strconv.Atoi(port); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
if runtime.GOOS == "darwin" {
|
||||
// On macOS, if a process already listens on 127.0.0.1:PORT, net.Listen() falls back
|
||||
// to IPv6 address ie minio will start listening on IPv6 address whereas another
|
||||
// (non-)minio process is listening on IPv4 of given port.
|
||||
// To avoid this error sutiation we check for port availability only for macOS.
|
||||
if err = checkPortAvailability(port); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
}
|
||||
|
||||
// Success.
|
||||
return host, port, nil
|
||||
}
|
||||
|
||||
func initConfig() {
|
||||
// Config file does not exist, we create it fresh and return upon success.
|
||||
if isFile(getConfigFile()) {
|
||||
@ -365,62 +115,46 @@ func initConfig() {
|
||||
}
|
||||
|
||||
func serverHandleCmdArgs(ctx *cli.Context) {
|
||||
// Get configuration directory from command line argument.
|
||||
configDir := ctx.String("config-dir")
|
||||
if !ctx.IsSet("config-dir") && ctx.GlobalIsSet("config-dir") {
|
||||
configDir = ctx.GlobalString("config-dir")
|
||||
}
|
||||
if configDir == "" {
|
||||
fatalIf(errors.New("empty directory"), "Configuration directory cannot be empty.")
|
||||
}
|
||||
|
||||
// Disallow relative paths, figure out absolute paths.
|
||||
// Set configuration directory.
|
||||
{
|
||||
// Get configuration directory from command line argument.
|
||||
configDir := ctx.String("config-dir")
|
||||
if !ctx.IsSet("config-dir") && ctx.GlobalIsSet("config-dir") {
|
||||
configDir = ctx.GlobalString("config-dir")
|
||||
}
|
||||
if configDir == "" {
|
||||
fatalIf(errors.New("empty directory"), "Configuration directory cannot be empty.")
|
||||
}
|
||||
|
||||
// Disallow relative paths, figure out absolute paths.
|
||||
configDirAbs, err := filepath.Abs(configDir)
|
||||
fatalIf(err, "Unable to fetch absolute path for config directory %s", configDir)
|
||||
|
||||
configDir = configDirAbs
|
||||
setConfigDir(configDirAbs)
|
||||
}
|
||||
|
||||
// Set configuration directory.
|
||||
setConfigDir(configDir)
|
||||
|
||||
// Server address.
|
||||
globalMinioAddr = ctx.String("address")
|
||||
serverAddr := ctx.String("address")
|
||||
fatalIf(CheckLocalServerAddr(serverAddr), "Invalid address ‘%s’ in command line argument.", serverAddr)
|
||||
|
||||
var setupType SetupType
|
||||
var err error
|
||||
globalMinioHost, globalMinioPort, err = getHostPort(globalMinioAddr)
|
||||
fatalIf(err, "Unable to extract host and port %s", globalMinioAddr)
|
||||
|
||||
// Disks to be used in server init.
|
||||
endpoints, err := parseStorageEndpoints(ctx.Args())
|
||||
fatalIf(err, "Unable to parse storage endpoints %s", ctx.Args())
|
||||
|
||||
// Sort endpoints for consistent ordering across multiple
|
||||
// nodes in a distributed setup. This is to avoid format.json
|
||||
// corruption if the disks aren't supplied in the same order
|
||||
// on all nodes.
|
||||
sort.Sort(byHostPath(endpoints))
|
||||
|
||||
checkServerSyntax(endpoints, ctx.Args())
|
||||
|
||||
// Should exit gracefully if none of the endpoints passed
|
||||
// as command line args are local to this server.
|
||||
if !isAnyEndpointLocal(endpoints) {
|
||||
fatalIf(errInvalidArgument, "None of the disks passed as command line args are local to this server.")
|
||||
globalMinioAddr, globalEndpoints, setupType, err = CreateEndpoints(serverAddr, ctx.Args()...)
|
||||
fatalIf(err, "Invalid command line arguments server=‘%s’, args=%s", serverAddr, ctx.Args())
|
||||
globalMinioHost, globalMinioPort = mustSplitHostPort(globalMinioAddr)
|
||||
if runtime.GOOS == "darwin" {
|
||||
// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
|
||||
// to IPv6 address ie minio will start listening on IPv6 address whereas another
|
||||
// (non-)minio process is listening on IPv4 of given port.
|
||||
// To avoid this error sutiation we check for port availability only for macOS.
|
||||
fatalIf(checkPortAvailability(globalMinioPort), "Port %d already in use", globalMinioPort)
|
||||
}
|
||||
|
||||
// Check if endpoints are part of distributed setup.
|
||||
globalIsDistXL = isDistributedSetup(endpoints)
|
||||
|
||||
// Set globalIsXL if erasure code backend is about to be
|
||||
// initialized for the given endpoints.
|
||||
if len(endpoints) > 1 {
|
||||
globalIsXL = (setupType == XLSetupType)
|
||||
globalIsDistXL = (setupType == DistXLSetupType)
|
||||
if globalIsDistXL {
|
||||
globalIsXL = true
|
||||
}
|
||||
|
||||
// Set endpoints of []*url.URL type to globalEndpoints.
|
||||
globalEndpoints = endpoints
|
||||
}
|
||||
|
||||
func serverHandleEnvVars() {
|
||||
@ -497,11 +231,10 @@ func serverMain(ctx *cli.Context) {
|
||||
if !quietFlag {
|
||||
// Check for new updates from dl.minio.io.
|
||||
mode := globalMinioModeFS
|
||||
if globalIsXL {
|
||||
mode = globalMinioModeXL
|
||||
}
|
||||
if globalIsDistXL {
|
||||
mode = globalMinioModeDistXL
|
||||
} else if globalIsXL {
|
||||
mode = globalMinioModeXL
|
||||
}
|
||||
checkUpdate(mode)
|
||||
}
|
||||
@ -518,31 +251,18 @@ func serverMain(ctx *cli.Context) {
|
||||
initNSLock(globalIsDistXL)
|
||||
|
||||
// Configure server.
|
||||
srvConfig := serverCmdConfig{
|
||||
serverAddr: globalMinioAddr,
|
||||
endpoints: globalEndpoints,
|
||||
}
|
||||
|
||||
// Configure server.
|
||||
handler, err := configureServerHandler(srvConfig)
|
||||
handler, err := configureServerHandler(globalEndpoints)
|
||||
fatalIf(err, "Unable to configure one of server's RPC services.")
|
||||
|
||||
// Initialize a new HTTP server.
|
||||
apiServer := NewServerMux(globalMinioAddr, handler)
|
||||
|
||||
// Set the global minio addr for this server.
|
||||
globalMinioAddr = getLocalAddress(srvConfig)
|
||||
|
||||
// Initialize S3 Peers inter-node communication only in distributed setup.
|
||||
initGlobalS3Peers(globalEndpoints)
|
||||
|
||||
// Initialize Admin Peers inter-node communication only in distributed setup.
|
||||
initGlobalAdminPeers(globalEndpoints)
|
||||
|
||||
// Determine API endpoints where we are going to serve the S3 API from.
|
||||
globalAPIEndpoints, err = finalizeAPIEndpoints(apiServer.Addr)
|
||||
fatalIf(err, "Unable to finalize API endpoints for %s", apiServer.Addr)
|
||||
|
||||
// Start server, automatically configures TLS if certs are available.
|
||||
go func() {
|
||||
cert, key := "", ""
|
||||
@ -552,7 +272,7 @@ func serverMain(ctx *cli.Context) {
|
||||
fatalIf(apiServer.ListenAndServe(cert, key), "Failed to start minio server.")
|
||||
}()
|
||||
|
||||
newObject, err := newObjectLayer(srvConfig)
|
||||
newObject, err := newObjectLayer(globalEndpoints)
|
||||
fatalIf(err, "Initializing object layer failed")
|
||||
|
||||
globalObjLayerMutex.Lock()
|
||||
@ -560,7 +280,8 @@ func serverMain(ctx *cli.Context) {
|
||||
globalObjLayerMutex.Unlock()
|
||||
|
||||
// Prints the formatted startup message once object layer is initialized.
|
||||
printStartupMessage(globalAPIEndpoints)
|
||||
apiEndpoints := getAPIEndpoints(apiServer.Addr)
|
||||
printStartupMessage(apiEndpoints)
|
||||
|
||||
// Set uptime time after object layer has initialized.
|
||||
globalBootTime = UTCNow()
|
||||
@ -570,40 +291,26 @@ func serverMain(ctx *cli.Context) {
|
||||
}
|
||||
|
||||
// Initialize object layer with the supplied disks, objectLayer is nil upon any error.
|
||||
func newObjectLayer(srvCmdCfg serverCmdConfig) (newObject ObjectLayer, err error) {
|
||||
func newObjectLayer(endpoints EndpointList) (newObject ObjectLayer, err error) {
|
||||
// For FS only, directly use the disk.
|
||||
isFS := len(srvCmdCfg.endpoints) == 1
|
||||
isFS := len(endpoints) == 1
|
||||
if isFS {
|
||||
// Unescape is needed for some UNC paths on windows
|
||||
// which are of this form \\127.0.0.1\\export\test.
|
||||
var fsPath string
|
||||
fsPath, err = url.QueryUnescape(srvCmdCfg.endpoints[0].String())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Initialize new FS object layer.
|
||||
newObject, err = newFSObjectLayer(fsPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// FS initialized, return.
|
||||
return newObject, nil
|
||||
return newFSObjectLayer(endpoints[0].Path)
|
||||
}
|
||||
|
||||
// First disk argument check if it is local.
|
||||
firstDisk := isLocalStorage(srvCmdCfg.endpoints[0])
|
||||
|
||||
// Initialize storage disks.
|
||||
storageDisks, err := initStorageDisks(srvCmdCfg.endpoints)
|
||||
storageDisks, err := initStorageDisks(endpoints)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Wait for formatting disks for XL backend.
|
||||
var formattedDisks []StorageAPI
|
||||
formattedDisks, err = waitForFormatXLDisks(firstDisk, srvCmdCfg.endpoints, storageDisks)
|
||||
|
||||
// First disk argument check if it is local.
|
||||
firstDisk := endpoints[0].IsLocal
|
||||
formattedDisks, err = waitForFormatXLDisks(firstDisk, endpoints, storageDisks)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -17,205 +17,10 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGetListenIPs(t *testing.T) {
|
||||
testCases := []struct {
|
||||
addr string
|
||||
port string
|
||||
shouldPass bool
|
||||
}{
|
||||
{"127.0.0.1", "9000", true},
|
||||
{"", "9000", true},
|
||||
{"", "", false},
|
||||
}
|
||||
for _, test := range testCases {
|
||||
var addr string
|
||||
// Please keep this we need to do this because
|
||||
// of odd https://play.golang.org/p/4dMPtM6Wdd
|
||||
// implementation issue.
|
||||
if test.port != "" {
|
||||
addr = test.addr + ":" + test.port
|
||||
}
|
||||
hosts, port, err := getListenIPs(addr)
|
||||
if !test.shouldPass && err == nil {
|
||||
t.Fatalf("Test should fail but succeeded %s", err)
|
||||
}
|
||||
if test.shouldPass && err != nil {
|
||||
t.Fatalf("Test should succeed but failed %s", err)
|
||||
}
|
||||
if test.shouldPass {
|
||||
if port != test.port {
|
||||
t.Errorf("Test expected %s, got %s", test.port, port)
|
||||
}
|
||||
if len(hosts) == 0 {
|
||||
t.Errorf("Test unexpected value hosts cannot be empty %#v", test)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Tests get host port.
|
||||
func TestGetHostPort(t *testing.T) {
|
||||
testCases := []struct {
|
||||
addr string
|
||||
err error
|
||||
}{
|
||||
// Test 1 - successful.
|
||||
{
|
||||
addr: ":" + getFreePort(),
|
||||
err: nil,
|
||||
},
|
||||
// Test 2 port empty.
|
||||
{
|
||||
addr: ":0",
|
||||
err: errEmptyPort,
|
||||
},
|
||||
// Test 3 port empty.
|
||||
{
|
||||
addr: ":",
|
||||
err: errEmptyPort,
|
||||
},
|
||||
// Test 4 invalid port.
|
||||
{
|
||||
addr: "linux:linux",
|
||||
err: errors.New("strconv.ParseInt: parsing \"linux\": invalid syntax"),
|
||||
},
|
||||
// Test 5 port not present.
|
||||
{
|
||||
addr: "hostname",
|
||||
err: errors.New("missing port in address hostname"),
|
||||
},
|
||||
}
|
||||
|
||||
// Validate all tests.
|
||||
for i, testCase := range testCases {
|
||||
_, _, err := getHostPort(testCase.addr)
|
||||
if err != nil {
|
||||
if err.Error() != testCase.err.Error() {
|
||||
t.Fatalf("Test %d: Error: %s", i+1, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Tests finalize api endpoints.
|
||||
func TestFinalizeAPIEndpoints(t *testing.T) {
|
||||
testCases := []struct {
|
||||
addr string
|
||||
}{
|
||||
{":80"},
|
||||
{":80"},
|
||||
{"127.0.0.1:80"},
|
||||
{"127.0.0.1:80"},
|
||||
}
|
||||
|
||||
for i, test := range testCases {
|
||||
endPoints, err := finalizeAPIEndpoints(test.addr)
|
||||
if err != nil && len(endPoints) <= 0 {
|
||||
t.Errorf("Test case %d returned with no API end points for %s",
|
||||
i+1, test.addr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Tests all the expected input disks for function checkSufficientDisks.
|
||||
func TestCheckSufficientDisks(t *testing.T) {
|
||||
var xlDisks []string
|
||||
if runtime.GOOS == globalWindowsOSName {
|
||||
xlDisks = []string{
|
||||
"C:\\mnt\\backend1",
|
||||
"C:\\mnt\\backend2",
|
||||
"C:\\mnt\\backend3",
|
||||
"C:\\mnt\\backend4",
|
||||
"C:\\mnt\\backend5",
|
||||
"C:\\mnt\\backend6",
|
||||
"C:\\mnt\\backend7",
|
||||
"C:\\mnt\\backend8",
|
||||
"C:\\mnt\\backend9",
|
||||
"C:\\mnt\\backend10",
|
||||
"C:\\mnt\\backend11",
|
||||
"C:\\mnt\\backend12",
|
||||
"C:\\mnt\\backend13",
|
||||
"C:\\mnt\\backend14",
|
||||
"C:\\mnt\\backend15",
|
||||
"C:\\mnt\\backend16",
|
||||
"C:\\mnt\\backend17",
|
||||
}
|
||||
} else {
|
||||
xlDisks = []string{
|
||||
"/mnt/backend1",
|
||||
"/mnt/backend2",
|
||||
"/mnt/backend3",
|
||||
"/mnt/backend4",
|
||||
"/mnt/backend5",
|
||||
"/mnt/backend6",
|
||||
"/mnt/backend7",
|
||||
"/mnt/backend8",
|
||||
"/mnt/backend9",
|
||||
"/mnt/backend10",
|
||||
"/mnt/backend11",
|
||||
"/mnt/backend12",
|
||||
"/mnt/backend13",
|
||||
"/mnt/backend14",
|
||||
"/mnt/backend15",
|
||||
"/mnt/backend16",
|
||||
"/mnt/backend17",
|
||||
}
|
||||
}
|
||||
// List of test cases fo sufficient disk verification.
|
||||
testCases := []struct {
|
||||
disks []string
|
||||
expectedErr error
|
||||
}{
|
||||
// Even number of disks '6'.
|
||||
{
|
||||
xlDisks[0:6],
|
||||
nil,
|
||||
},
|
||||
// Even number of disks '12'.
|
||||
{
|
||||
xlDisks[0:12],
|
||||
nil,
|
||||
},
|
||||
// Even number of disks '16'.
|
||||
{
|
||||
xlDisks[0:16],
|
||||
nil,
|
||||
},
|
||||
// Larger than maximum number of disks > 16.
|
||||
{
|
||||
xlDisks,
|
||||
errXLMaxDisks,
|
||||
},
|
||||
// Lesser than minimum number of disks < 6.
|
||||
{
|
||||
xlDisks[0:3],
|
||||
errXLMinDisks,
|
||||
},
|
||||
// Odd number of disks, not divisible by '2'.
|
||||
{
|
||||
append(xlDisks[0:10], xlDisks[11]),
|
||||
errXLNumDisks,
|
||||
},
|
||||
}
|
||||
|
||||
// Validates different variations of input disks.
|
||||
for i, testCase := range testCases {
|
||||
endpoints, err := parseStorageEndpoints(testCase.disks)
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error %s", err)
|
||||
}
|
||||
if checkSufficientDisks(endpoints) != testCase.expectedErr {
|
||||
t.Errorf("Test %d expected to pass for disks %s", i+1, testCase.disks)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Tests initializing new object layer.
|
||||
func TestNewObjectLayer(t *testing.T) {
|
||||
// Tests for FS object layer.
|
||||
@ -226,15 +31,8 @@ func TestNewObjectLayer(t *testing.T) {
|
||||
}
|
||||
defer removeRoots(disks)
|
||||
|
||||
endpoints, err := parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected parse error", err)
|
||||
}
|
||||
|
||||
obj, err := newObjectLayer(serverCmdConfig{
|
||||
serverAddr: ":9000",
|
||||
endpoints: endpoints,
|
||||
})
|
||||
endpoints := mustGetNewEndpointList(disks...)
|
||||
obj, err := newObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected object layer initialization error", err)
|
||||
}
|
||||
@ -253,15 +51,8 @@ func TestNewObjectLayer(t *testing.T) {
|
||||
}
|
||||
defer removeRoots(disks)
|
||||
|
||||
endpoints, err = parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected parse error", err)
|
||||
}
|
||||
|
||||
obj, err = newObjectLayer(serverCmdConfig{
|
||||
serverAddr: ":9000",
|
||||
endpoints: endpoints,
|
||||
})
|
||||
endpoints = mustGetNewEndpointList(disks...)
|
||||
obj, err = newObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected object layer initialization error", err)
|
||||
}
|
||||
@ -271,176 +62,3 @@ func TestNewObjectLayer(t *testing.T) {
|
||||
t.Fatal("Unexpected object layer detected", reflect.TypeOf(obj))
|
||||
}
|
||||
}
|
||||
|
||||
// Tests parsing various types of input endpoints and paths.
|
||||
func TestParseStorageEndpoints(t *testing.T) {
|
||||
testCases := []struct {
|
||||
globalMinioHost string
|
||||
host string
|
||||
expectedErr error
|
||||
}{
|
||||
{"", "http://127.0.0.1/export", nil},
|
||||
{
|
||||
"testhost",
|
||||
"http://127.0.0.1/export",
|
||||
errors.New("Invalid Argument 127.0.0.1, port mandatory when --address <host>:<port> is used"),
|
||||
},
|
||||
{
|
||||
"",
|
||||
"http://127.0.0.1:9000/export",
|
||||
errors.New("Invalid Argument 127.0.0.1:9000, port configurable using --address :<port>"),
|
||||
},
|
||||
{"testhost", "http://127.0.0.1:9000/export", nil},
|
||||
}
|
||||
for i, test := range testCases {
|
||||
globalMinioHost = test.globalMinioHost
|
||||
_, err := parseStorageEndpoints([]string{test.host})
|
||||
if err != nil {
|
||||
if err.Error() != test.expectedErr.Error() {
|
||||
t.Errorf("Test %d : got %v, expected %v", i+1, err, test.expectedErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Should be reset back to "" so that we don't affect other tests.
|
||||
globalMinioHost = ""
|
||||
}
|
||||
|
||||
// Test check endpoints syntax function for syntax verification
|
||||
// across various scenarios of inputs.
|
||||
func TestCheckEndpointsSyntax(t *testing.T) {
|
||||
successCases := []string{
|
||||
"export",
|
||||
"/export",
|
||||
"http://127.0.0.1/export",
|
||||
"https://127.0.0.1/export",
|
||||
}
|
||||
|
||||
failureCases := []string{
|
||||
"/",
|
||||
"http://127.0.0.1",
|
||||
"http://127.0.0.1/",
|
||||
"ftp://127.0.0.1/export",
|
||||
"server:/export",
|
||||
}
|
||||
|
||||
if runtime.GOOS == globalWindowsOSName {
|
||||
successCases = append(successCases,
|
||||
`\export`,
|
||||
`D:\export`,
|
||||
)
|
||||
|
||||
failureCases = append(failureCases,
|
||||
"D:",
|
||||
`D:\`,
|
||||
`\`,
|
||||
)
|
||||
}
|
||||
|
||||
for _, disk := range successCases {
|
||||
eps, err := parseStorageEndpoints([]string{disk})
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to parse %s, error %s", disk, err)
|
||||
}
|
||||
if err = checkEndpointsSyntax(eps, []string{disk}); err != nil {
|
||||
t.Errorf("expected: <nil>, got: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, disk := range failureCases {
|
||||
eps, err := parseStorageEndpoints([]string{disk})
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to parse %s, error %s", disk, err)
|
||||
}
|
||||
if err = checkEndpointsSyntax(eps, []string{disk}); err == nil {
|
||||
t.Errorf("expected: <error>, got: <nil>")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsDistributedSetup(t *testing.T) {
|
||||
var testCases []struct {
|
||||
disks []string
|
||||
result bool
|
||||
}
|
||||
if runtime.GOOS == globalWindowsOSName {
|
||||
testCases = []struct {
|
||||
disks []string
|
||||
result bool
|
||||
}{
|
||||
{[]string{`http://4.4.4.4/c:\mnt\disk1`, `http://4.4.4.4/c:\mnt\disk2`}, true},
|
||||
{[]string{`http://4.4.4.4/c:\mnt\disk1`, `http://127.0.0.1/c:\mnt\disk2`}, true},
|
||||
{[]string{`c:\mnt\disk1`, `c:\mnt\disk2`}, false},
|
||||
}
|
||||
} else {
|
||||
testCases = []struct {
|
||||
disks []string
|
||||
result bool
|
||||
}{
|
||||
{[]string{"http://4.4.4.4/mnt/disk1", "http://4.4.4.4/mnt/disk2"}, true},
|
||||
{[]string{"http://4.4.4.4/mnt/disk1", "http://127.0.0.1/mnt/disk2"}, true},
|
||||
{[]string{"/mnt/disk1", "/mnt/disk2"}, false},
|
||||
}
|
||||
}
|
||||
for i, test := range testCases {
|
||||
endpoints, err := parseStorageEndpoints(test.disks)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: Unexpected error: %s", i+1, err)
|
||||
}
|
||||
res := isDistributedSetup(endpoints)
|
||||
if res != test.result {
|
||||
t.Errorf("Test %d: expected result %t but received %t", i+1, test.result, res)
|
||||
}
|
||||
}
|
||||
|
||||
// Test cases when globalMinioHost is set
|
||||
globalMinioHost = "testhost"
|
||||
testCases = []struct {
|
||||
disks []string
|
||||
result bool
|
||||
}{
|
||||
{[]string{"http://127.0.0.1:9001/mnt/disk1", "http://127.0.0.1:9002/mnt/disk2", "http://127.0.0.1:9003/mnt/disk3", "http://127.0.0.1:9004/mnt/disk4"}, true},
|
||||
{[]string{"/mnt/disk1", "/mnt/disk2"}, false},
|
||||
}
|
||||
|
||||
for i, test := range testCases {
|
||||
endpoints, err := parseStorageEndpoints(test.disks)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: Unexpected error: %s", i+1, err)
|
||||
}
|
||||
res := isDistributedSetup(endpoints)
|
||||
if res != test.result {
|
||||
t.Errorf("Test %d: expected result %t but received %t", i+1, test.result, res)
|
||||
}
|
||||
}
|
||||
// Reset so that we don't affect other tests.
|
||||
globalMinioHost = ""
|
||||
}
|
||||
|
||||
// Tests isAnyEndpointLocal function with inputs such that it returns true and false respectively.
|
||||
func TestIsAnyEndpointLocal(t *testing.T) {
|
||||
testCases := []struct {
|
||||
disks []string
|
||||
result bool
|
||||
}{
|
||||
{
|
||||
disks: []string{"http://4.4.4.4/mnt/disk1",
|
||||
"http://4.4.4.4/mnt/disk1"},
|
||||
result: false,
|
||||
},
|
||||
{
|
||||
disks: []string{"http://127.0.0.1/mnt/disk1",
|
||||
"http://127.0.0.1/mnt/disk1"},
|
||||
result: true,
|
||||
},
|
||||
}
|
||||
for i, test := range testCases {
|
||||
endpoints, err := parseStorageEndpoints(test.disks)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d - Failed to parse storage endpoints %v", i+1, err)
|
||||
}
|
||||
actual := isAnyEndpointLocal(endpoints)
|
||||
if actual != test.result {
|
||||
t.Errorf("Test %d - Expected %v but received %v", i+1, test.result, actual)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -1,71 +0,0 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
)
|
||||
|
||||
// getListenIPs - gets all the ips to listen on.
|
||||
func getListenIPs(serverAddr string) (hosts []string, port string, err error) {
|
||||
var host string
|
||||
host, port, err = net.SplitHostPort(serverAddr)
|
||||
if err != nil {
|
||||
return nil, port, fmt.Errorf("Unable to parse host address %s", err)
|
||||
}
|
||||
if host == "" {
|
||||
var ipv4s []net.IP
|
||||
ipv4s, err = getInterfaceIPv4s()
|
||||
if err != nil {
|
||||
return nil, port, fmt.Errorf("Unable reverse sort ips from hosts %s", err)
|
||||
}
|
||||
for _, ip := range ipv4s {
|
||||
hosts = append(hosts, ip.String())
|
||||
}
|
||||
return hosts, port, nil
|
||||
} // if host != "" {
|
||||
|
||||
// Proceed to append itself, since user requested a specific endpoint.
|
||||
hosts = append(hosts, host)
|
||||
|
||||
// Success.
|
||||
return hosts, port, nil
|
||||
}
|
||||
|
||||
// Finalizes the API endpoints based on the host list and port.
|
||||
func finalizeAPIEndpoints(addr string) (endPoints []string, err error) {
|
||||
// Verify current scheme.
|
||||
scheme := httpScheme
|
||||
if globalIsSSL {
|
||||
scheme = httpsScheme
|
||||
}
|
||||
|
||||
// Get list of listen ips and port.
|
||||
hosts, port, err1 := getListenIPs(addr)
|
||||
if err1 != nil {
|
||||
return nil, err1
|
||||
}
|
||||
|
||||
// Construct proper endpoints.
|
||||
for _, host := range hosts {
|
||||
endPoints = append(endPoints, fmt.Sprintf("%s://%s:%s", scheme, host, port))
|
||||
}
|
||||
|
||||
// Success.
|
||||
return endPoints, nil
|
||||
}
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -16,19 +16,29 @@
|
||||
|
||||
package cmd
|
||||
|
||||
import "net/url"
|
||||
// SetupType - enum for setup type.
|
||||
type SetupType int
|
||||
|
||||
type byHostPath []*url.URL
|
||||
const (
|
||||
// FSSetupType - FS setup type enum.
|
||||
FSSetupType SetupType = iota + 1
|
||||
|
||||
func (s byHostPath) Swap(i, j int) {
|
||||
s[i], s[j] = s[j], s[i]
|
||||
}
|
||||
|
||||
func (s byHostPath) Len() int {
|
||||
return len(s)
|
||||
}
|
||||
|
||||
// Note: Host in url.URL includes the port too.
|
||||
func (s byHostPath) Less(i, j int) bool {
|
||||
return (s[i].Host + s[i].Path) < (s[j].Host + s[j].Path)
|
||||
// XLSetupType - XL setup type enum.
|
||||
XLSetupType
|
||||
|
||||
// DistXLSetupType - Distributed XL setup type enum.
|
||||
DistXLSetupType
|
||||
)
|
||||
|
||||
func (setupType SetupType) String() string {
|
||||
switch setupType {
|
||||
case FSSetupType:
|
||||
return globalMinioModeFS
|
||||
case XLSetupType:
|
||||
return globalMinioModeXL
|
||||
case DistXLSetupType:
|
||||
return globalMinioModeDistXL
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
@ -85,8 +85,12 @@ func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode {
|
||||
cred := serverConfig.GetCredential()
|
||||
|
||||
// r.RequestURI will have raw encoded URI as sent by the client.
|
||||
splits := splitStr(r.RequestURI, "?", 2)
|
||||
encodedResource, encodedQuery := splits[0], splits[1]
|
||||
tokens := strings.SplitN(r.RequestURI, "?", 2)
|
||||
encodedResource := tokens[0]
|
||||
encodedQuery := ""
|
||||
if len(tokens) == 2 {
|
||||
encodedQuery = tokens[1]
|
||||
}
|
||||
|
||||
queries := strings.Split(encodedQuery, "&")
|
||||
var filteredQueries []string
|
||||
@ -206,8 +210,12 @@ func doesSignV2Match(r *http.Request) APIErrorCode {
|
||||
}
|
||||
|
||||
// r.RequestURI will have raw encoded URI as sent by the client.
|
||||
splits := splitStr(r.RequestURI, "?", 2)
|
||||
encodedResource, encodedQuery := splits[0], splits[1]
|
||||
tokens := strings.SplitN(r.RequestURI, "?", 2)
|
||||
encodedResource := tokens[0]
|
||||
encodedQuery := ""
|
||||
if len(tokens) == 2 {
|
||||
encodedQuery = tokens[1]
|
||||
}
|
||||
|
||||
expectedAuth := signatureV2(r.Method, encodedResource, encodedQuery, r.Header)
|
||||
if v2Auth != expectedAuth {
|
||||
|
@ -21,7 +21,6 @@ import (
|
||||
"io"
|
||||
"net"
|
||||
"net/rpc"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
@ -96,39 +95,22 @@ func toStorageErr(err error) error {
|
||||
}
|
||||
|
||||
// Initialize new storage rpc client.
|
||||
func newStorageRPC(ep *url.URL) (StorageAPI, error) {
|
||||
if ep == nil {
|
||||
return nil, errInvalidArgument
|
||||
}
|
||||
|
||||
func newStorageRPC(endpoint Endpoint) StorageAPI {
|
||||
// Dial minio rpc storage http path.
|
||||
rpcPath := path.Join(minioReservedBucketPath, storageRPCPath, getPath(ep))
|
||||
rpcAddr := ep.Host
|
||||
|
||||
rpcPath := path.Join(minioReservedBucketPath, storageRPCPath, endpoint.Path)
|
||||
serverCred := serverConfig.GetCredential()
|
||||
accessKey := serverCred.AccessKey
|
||||
secretKey := serverCred.SecretKey
|
||||
if ep.User != nil {
|
||||
accessKey = ep.User.Username()
|
||||
if password, ok := ep.User.Password(); ok {
|
||||
secretKey = password
|
||||
}
|
||||
}
|
||||
|
||||
storageAPI := &networkStorage{
|
||||
return &networkStorage{
|
||||
rpcClient: newAuthRPCClient(authConfig{
|
||||
accessKey: accessKey,
|
||||
secretKey: secretKey,
|
||||
serverAddr: rpcAddr,
|
||||
accessKey: serverCred.AccessKey,
|
||||
secretKey: serverCred.SecretKey,
|
||||
serverAddr: endpoint.Host,
|
||||
serviceEndpoint: rpcPath,
|
||||
secureConn: globalIsSSL,
|
||||
serviceName: "Storage",
|
||||
disableReconnect: true,
|
||||
}),
|
||||
}
|
||||
|
||||
// Returns successfully here.
|
||||
return storageAPI, nil
|
||||
}
|
||||
|
||||
// Stringer interface compatible representation of network device.
|
||||
|
@ -23,7 +23,6 @@ import (
|
||||
"io"
|
||||
"net"
|
||||
"net/rpc"
|
||||
"net/url"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
@ -146,25 +145,15 @@ func (s *TestRPCStorageSuite) SetUpSuite(c *testing.T) {
|
||||
listenAddress := s.testServer.Server.Listener.Addr().String()
|
||||
|
||||
for _, ep := range s.testServer.Disks {
|
||||
ep.Host = listenAddress
|
||||
storageDisk, err := newStorageRPC(ep)
|
||||
if err != nil {
|
||||
c.Fatal("Unable to initialize RPC client", err)
|
||||
// Eventhough s.testServer.Disks is EndpointList, we would need a URLEndpointType here.
|
||||
endpoint := ep
|
||||
if endpoint.Type() == PathEndpointType {
|
||||
endpoint.Scheme = "http"
|
||||
}
|
||||
endpoint.Host = listenAddress
|
||||
storageDisk := newStorageRPC(endpoint)
|
||||
s.remoteDisks = append(s.remoteDisks, storageDisk)
|
||||
}
|
||||
_, err := newStorageRPC(nil)
|
||||
if err != errInvalidArgument {
|
||||
c.Fatalf("Unexpected error %s, expecting %s", err, errInvalidArgument)
|
||||
}
|
||||
u, err := url.Parse("http://abcd:abcd123@localhost/mnt/disk")
|
||||
if err != nil {
|
||||
c.Fatal("Unexpected error", err)
|
||||
}
|
||||
_, err = newStorageRPC(u)
|
||||
if err != nil {
|
||||
c.Fatal("Unexpected error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// No longer used with gocheck, but used in explicit teardown code in
|
||||
|
@ -197,30 +197,28 @@ func (s *storageServer) RenameFileHandler(args *RenameFileArgs, reply *AuthRPCRe
|
||||
}
|
||||
|
||||
// Initialize new storage rpc.
|
||||
func newRPCServer(srvConfig serverCmdConfig) (servers []*storageServer, err error) {
|
||||
for _, ep := range srvConfig.endpoints {
|
||||
// e.g server:/mnt/disk1
|
||||
if isLocalStorage(ep) {
|
||||
// Get the posix path.
|
||||
path := getPath(ep)
|
||||
var storage StorageAPI
|
||||
storage, err = newPosix(path)
|
||||
func newRPCServer(endpoints EndpointList) (servers []*storageServer, err error) {
|
||||
for _, endpoint := range endpoints {
|
||||
if endpoint.IsLocal {
|
||||
storage, err := newPosix(endpoint.Path)
|
||||
if err != nil && err != errDiskNotFound {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
servers = append(servers, &storageServer{
|
||||
storage: storage,
|
||||
path: path,
|
||||
path: endpoint.Path,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return servers, nil
|
||||
}
|
||||
|
||||
// registerStorageRPCRouter - register storage rpc router.
|
||||
func registerStorageRPCRouters(mux *router.Router, srvCmdConfig serverCmdConfig) error {
|
||||
func registerStorageRPCRouters(mux *router.Router, endpoints EndpointList) error {
|
||||
// Initialize storage rpc servers for every disk that is hosted on this node.
|
||||
storageRPCs, err := newRPCServer(srvCmdConfig)
|
||||
storageRPCs, err := newRPCServer(endpoints)
|
||||
if err != nil {
|
||||
return traceError(err)
|
||||
}
|
||||
|
@ -17,7 +17,6 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/pkg/disk"
|
||||
@ -30,7 +29,7 @@ type testStorageRPCServer struct {
|
||||
token string
|
||||
diskDirs []string
|
||||
stServer *storageServer
|
||||
endpoints []*url.URL
|
||||
endpoints EndpointList
|
||||
}
|
||||
|
||||
func createTestStorageServer(t *testing.T) *testStorageRPCServer {
|
||||
@ -50,11 +49,7 @@ func createTestStorageServer(t *testing.T) *testStorageRPCServer {
|
||||
t.Fatalf("unable to create FS backend, %s", err)
|
||||
}
|
||||
|
||||
endpoints, err := parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to parse storage endpoints, %s", err)
|
||||
}
|
||||
|
||||
endpoints := mustGetNewEndpointList(fsDirs...)
|
||||
storageDisks, err := initStorageDisks(endpoints)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to initialize storage disks, %s", err)
|
||||
|
@ -91,11 +91,7 @@ func prepareXL() (ObjectLayer, []string, error) {
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
removeRoots(fsDirs)
|
||||
return nil, nil, err
|
||||
@ -180,12 +176,12 @@ func isSameType(obj1, obj2 interface{}) bool {
|
||||
// defer s.Stop()
|
||||
type TestServer struct {
|
||||
Root string
|
||||
Disks []*url.URL
|
||||
Disks EndpointList
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
Server *httptest.Server
|
||||
Obj ObjectLayer
|
||||
SrvCmdCfg serverCmdConfig
|
||||
endpoints EndpointList
|
||||
}
|
||||
|
||||
// UnstartedTestServer - Configures a temp FS/XL backend,
|
||||
@ -210,50 +206,31 @@ func UnstartedTestServer(t TestErrHandler, instanceType string) TestServer {
|
||||
credentials := serverConfig.GetCredential()
|
||||
|
||||
testServer.Obj = objLayer
|
||||
testServer.Disks, err = parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error %v", err)
|
||||
}
|
||||
testServer.Disks = mustGetNewEndpointList(disks...)
|
||||
testServer.Root = root
|
||||
testServer.AccessKey = credentials.AccessKey
|
||||
testServer.SecretKey = credentials.SecretKey
|
||||
|
||||
srvCmdCfg := serverCmdConfig{
|
||||
endpoints: testServer.Disks,
|
||||
}
|
||||
|
||||
httpHandler, err := configureServerHandler(
|
||||
srvCmdCfg,
|
||||
)
|
||||
httpHandler, err := configureServerHandler(testServer.Disks)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to configure one of the RPC services <ERROR> %s", err)
|
||||
}
|
||||
|
||||
// Run TestServer.
|
||||
testServer.Server = httptest.NewUnstartedServer(httpHandler)
|
||||
// obtain server address.
|
||||
srvCmdCfg.serverAddr = testServer.Server.Listener.Addr().String()
|
||||
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = objLayer
|
||||
globalObjLayerMutex.Unlock()
|
||||
|
||||
// initialize peer rpc
|
||||
host, port, err := net.SplitHostPort(srvCmdCfg.serverAddr)
|
||||
if err != nil {
|
||||
t.Fatal("Early setup error:", err)
|
||||
}
|
||||
host, port := mustSplitHostPort(testServer.Server.Listener.Addr().String())
|
||||
globalMinioHost = host
|
||||
globalMinioPort = port
|
||||
globalMinioAddr = getLocalAddress(srvCmdCfg)
|
||||
endpoints, err := parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatal("Early setup error:", err)
|
||||
}
|
||||
initGlobalS3Peers(endpoints)
|
||||
globalMinioAddr = getEndpointsLocalAddr(testServer.Disks)
|
||||
initGlobalS3Peers(testServer.Disks)
|
||||
|
||||
return testServer
|
||||
|
||||
}
|
||||
|
||||
// testServerCertPEM and testServerKeyPEM are generated by
|
||||
@ -339,10 +316,10 @@ func StartTestServer(t TestErrHandler, instanceType string) TestServer {
|
||||
|
||||
// Initializes storage RPC endpoints.
|
||||
// The object Layer will be a temp back used for testing purpose.
|
||||
func initTestStorageRPCEndPoint(srvCmdConfig serverCmdConfig) http.Handler {
|
||||
func initTestStorageRPCEndPoint(endpoints EndpointList) http.Handler {
|
||||
// Initialize router.
|
||||
muxRouter := router.NewRouter()
|
||||
registerStorageRPCRouters(muxRouter, srvCmdConfig)
|
||||
registerStorageRPCRouters(muxRouter, endpoints)
|
||||
return muxRouter
|
||||
}
|
||||
|
||||
@ -354,10 +331,6 @@ func StartTestStorageRPCServer(t TestErrHandler, instanceType string, diskN int)
|
||||
if err != nil {
|
||||
t.Fatal("Failed to create disks for the backend")
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatalf("%s", err)
|
||||
}
|
||||
|
||||
root, err := newTestConfig(globalMinioDefaultRegion)
|
||||
if err != nil {
|
||||
@ -369,15 +342,14 @@ func StartTestStorageRPCServer(t TestErrHandler, instanceType string, diskN int)
|
||||
// Get credential.
|
||||
credentials := serverConfig.GetCredential()
|
||||
|
||||
endpoints := mustGetNewEndpointList(disks...)
|
||||
testRPCServer.Root = root
|
||||
testRPCServer.Disks = endpoints
|
||||
testRPCServer.AccessKey = credentials.AccessKey
|
||||
testRPCServer.SecretKey = credentials.SecretKey
|
||||
|
||||
// Run TestServer.
|
||||
testRPCServer.Server = httptest.NewServer(initTestStorageRPCEndPoint(serverCmdConfig{
|
||||
endpoints: endpoints,
|
||||
}))
|
||||
testRPCServer.Server = httptest.NewServer(initTestStorageRPCEndPoint(endpoints))
|
||||
return testRPCServer
|
||||
}
|
||||
|
||||
@ -389,10 +361,6 @@ func StartTestPeersRPCServer(t TestErrHandler, instanceType string) TestServer {
|
||||
if err != nil {
|
||||
t.Fatal("Failed to create disks for the backend")
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatalf("%s", err)
|
||||
}
|
||||
|
||||
root, err := newTestConfig(globalMinioDefaultRegion)
|
||||
if err != nil {
|
||||
@ -404,6 +372,7 @@ func StartTestPeersRPCServer(t TestErrHandler, instanceType string) TestServer {
|
||||
// Get credential.
|
||||
credentials := serverConfig.GetCredential()
|
||||
|
||||
endpoints := mustGetNewEndpointList(disks...)
|
||||
testRPCServer.Root = root
|
||||
testRPCServer.Disks = endpoints
|
||||
testRPCServer.AccessKey = credentials.AccessKey
|
||||
@ -420,13 +389,9 @@ func StartTestPeersRPCServer(t TestErrHandler, instanceType string) TestServer {
|
||||
testRPCServer.Obj = objLayer
|
||||
globalObjLayerMutex.Unlock()
|
||||
|
||||
srvCfg := serverCmdConfig{
|
||||
endpoints: endpoints,
|
||||
}
|
||||
|
||||
mux := router.NewRouter()
|
||||
// need storage layer for bucket config storage.
|
||||
registerStorageRPCRouters(mux, srvCfg)
|
||||
registerStorageRPCRouters(mux, endpoints)
|
||||
// need API layer to send requests, etc.
|
||||
registerAPIRouter(mux)
|
||||
// module being tested is Peer RPCs router.
|
||||
@ -436,7 +401,7 @@ func StartTestPeersRPCServer(t TestErrHandler, instanceType string) TestServer {
|
||||
testRPCServer.Server = httptest.NewServer(mux)
|
||||
|
||||
// initialize remainder of serverCmdConfig
|
||||
testRPCServer.SrvCmdCfg = srvCfg
|
||||
testRPCServer.endpoints = endpoints
|
||||
|
||||
return testRPCServer
|
||||
}
|
||||
@ -481,7 +446,7 @@ func resetGlobalEventnotify() {
|
||||
}
|
||||
|
||||
func resetGlobalEndpoints() {
|
||||
globalEndpoints = []*url.URL{}
|
||||
globalEndpoints = EndpointList{}
|
||||
}
|
||||
|
||||
func resetGlobalIsXL() {
|
||||
@ -1659,7 +1624,7 @@ func getRandomDisks(N int) ([]string, error) {
|
||||
}
|
||||
|
||||
// initObjectLayer - Instantiates object layer and returns it.
|
||||
func initObjectLayer(endpoints []*url.URL) (ObjectLayer, []StorageAPI, error) {
|
||||
func initObjectLayer(endpoints EndpointList) (ObjectLayer, []StorageAPI, error) {
|
||||
storageDisks, err := initStorageDisks(endpoints)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
@ -1738,12 +1703,8 @@ func prepareXLStorageDisks(t *testing.T) ([]StorageAPI, []string) {
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
_, storageDisks, err := initObjectLayer(endpoints)
|
||||
_, storageDisks, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
removeRoots(fsDirs)
|
||||
t.Fatal("Unable to initialize storage disks", err)
|
||||
@ -2077,11 +2038,7 @@ func ExecObjectLayerStaleFilesTest(t *testing.T, objTest objTestStaleFilesType)
|
||||
if err != nil {
|
||||
t.Fatalf("Initialization of disks for XL setup: %s", err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(erasureDisks)
|
||||
if err != nil {
|
||||
t.Fatalf("Initialization of disks for XL setup: %s", err)
|
||||
}
|
||||
objLayer, _, err := initObjectLayer(endpoints)
|
||||
objLayer, _, err := initObjectLayer(mustGetNewEndpointList(erasureDisks...))
|
||||
if err != nil {
|
||||
t.Fatalf("Initialization of object layer failed for XL setup: %s", err)
|
||||
}
|
||||
@ -2380,3 +2337,27 @@ func generateTLSCertKey(host string) ([]byte, []byte, error) {
|
||||
|
||||
return certOut.Bytes(), keyOut.Bytes(), nil
|
||||
}
|
||||
|
||||
func mustGetNewEndpointList(args ...string) (endpoints EndpointList) {
|
||||
if len(args) == 1 {
|
||||
endpoint, err := NewEndpoint(args[0])
|
||||
fatalIf(err, "unable to create new endpoint")
|
||||
endpoints = append(endpoints, endpoint)
|
||||
} else {
|
||||
var err error
|
||||
endpoints, err = NewEndpointList(args...)
|
||||
fatalIf(err, "unable to create new endpoint list")
|
||||
}
|
||||
|
||||
return endpoints
|
||||
}
|
||||
|
||||
func getEndpointsLocalAddr(endpoints EndpointList) string {
|
||||
for _, endpoint := range endpoints {
|
||||
if endpoint.IsLocal && endpoint.Type() == URLEndpointType {
|
||||
return endpoint.Host
|
||||
}
|
||||
}
|
||||
|
||||
return globalMinioHost + ":" + globalMinioPort
|
||||
}
|
||||
|
@ -164,10 +164,7 @@ func TestTreeWalk(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to create tmp directory: %s", err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints([]string{fsDir})
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error %s", err)
|
||||
}
|
||||
endpoints := mustGetNewEndpointList(fsDir)
|
||||
disk, err := newStorageAPI(endpoints[0])
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to create StorageAPI: %s", err)
|
||||
@ -205,10 +202,7 @@ func TestTreeWalkTimeout(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to create tmp directory: %s", err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints([]string{fsDir})
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error %s", err)
|
||||
}
|
||||
endpoints := mustGetNewEndpointList(fsDir)
|
||||
disk, err := newStorageAPI(endpoints[0])
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to create StorageAPI: %s", err)
|
||||
@ -285,18 +279,15 @@ func TestListDir(t *testing.T) {
|
||||
t.Errorf("Unable to create tmp directory: %s", err)
|
||||
}
|
||||
|
||||
endpoints, err := parseStorageEndpoints([]string{fsDir1, fsDir2})
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error %s", err)
|
||||
}
|
||||
|
||||
// Create two StorageAPIs disk1 and disk2.
|
||||
endpoints := mustGetNewEndpointList(fsDir1)
|
||||
disk1, err := newStorageAPI(endpoints[0])
|
||||
if err != nil {
|
||||
t.Errorf("Unable to create StorageAPI: %s", err)
|
||||
}
|
||||
|
||||
disk2, err := newStorageAPI(endpoints[1])
|
||||
endpoints = mustGetNewEndpointList(fsDir2)
|
||||
disk2, err := newStorageAPI(endpoints[0])
|
||||
if err != nil {
|
||||
t.Errorf("Unable to create StorageAPI: %s", err)
|
||||
}
|
||||
@ -366,10 +357,7 @@ func TestRecursiveTreeWalk(t *testing.T) {
|
||||
t.Fatalf("Unable to create tmp directory: %s", err)
|
||||
}
|
||||
|
||||
endpoints, err := parseStorageEndpoints([]string{fsDir1})
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error %s", err)
|
||||
}
|
||||
endpoints := mustGetNewEndpointList(fsDir1)
|
||||
disk1, err := newStorageAPI(endpoints[0])
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to create StorageAPI: %s", err)
|
||||
@ -476,10 +464,7 @@ func TestSortedness(t *testing.T) {
|
||||
t.Errorf("Unable to create tmp directory: %s", err)
|
||||
}
|
||||
|
||||
endpoints, err := parseStorageEndpoints([]string{fsDir1})
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error %s", err)
|
||||
}
|
||||
endpoints := mustGetNewEndpointList(fsDir1)
|
||||
disk1, err := newStorageAPI(endpoints[0])
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to create StorageAPI: %s", err)
|
||||
@ -554,10 +539,7 @@ func TestTreeWalkIsEnd(t *testing.T) {
|
||||
t.Errorf("Unable to create tmp directory: %s", err)
|
||||
}
|
||||
|
||||
endpoints, err := parseStorageEndpoints([]string{fsDir1})
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error %s", err)
|
||||
}
|
||||
endpoints := mustGetNewEndpointList(fsDir1)
|
||||
disk1, err := newStorageAPI(endpoints[0])
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to create StorageAPI: %s", err)
|
||||
|
@ -1,99 +0,0 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestSortByHostPath - tests if ordering of urls are based on
|
||||
// host+path concatenated.
|
||||
func TestSortByHostPath(t *testing.T) {
|
||||
testCases := []struct {
|
||||
given []string
|
||||
expected []*url.URL
|
||||
}{
|
||||
{
|
||||
given: []string{
|
||||
"http://abcd.com/a/b/d",
|
||||
"http://abcd.com/a/b/c",
|
||||
"http://abcd.com/a/b/e",
|
||||
},
|
||||
expected: []*url.URL{
|
||||
{
|
||||
Scheme: httpScheme,
|
||||
Host: "abcd.com:9000",
|
||||
Path: "/a/b/c",
|
||||
},
|
||||
{
|
||||
Scheme: httpScheme,
|
||||
Host: "abcd.com:9000",
|
||||
Path: "/a/b/d",
|
||||
},
|
||||
{
|
||||
Scheme: httpScheme,
|
||||
Host: "abcd.com:9000",
|
||||
Path: "/a/b/e",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
given: []string{
|
||||
"http://defg.com/a/b/c",
|
||||
"http://abcd.com/a/b/c",
|
||||
"http://hijk.com/a/b/c",
|
||||
},
|
||||
expected: []*url.URL{
|
||||
{
|
||||
Scheme: httpScheme,
|
||||
Host: "abcd.com:9000",
|
||||
Path: "/a/b/c",
|
||||
},
|
||||
{
|
||||
Scheme: httpScheme,
|
||||
Host: "defg.com:9000",
|
||||
Path: "/a/b/c",
|
||||
},
|
||||
{
|
||||
Scheme: httpScheme,
|
||||
Host: "hijk.com:9000",
|
||||
Path: "/a/b/c",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
saveGlobalPort := globalMinioPort
|
||||
globalMinioPort = "9000"
|
||||
for i, test := range testCases {
|
||||
eps, err := parseStorageEndpoints(test.given)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d - Failed to parse storage endpoint %v", i+1, err)
|
||||
}
|
||||
sort.Sort(byHostPath(eps))
|
||||
if !sort.IsSorted(byHostPath(eps)) {
|
||||
t.Errorf("Test %d - Expected order %v but got %v", i+1, test.expected, eps)
|
||||
}
|
||||
if !reflect.DeepEqual(eps, test.expected) {
|
||||
t.Errorf("Test %d - Expected order %v but got %v", i+1, test.expected, eps)
|
||||
}
|
||||
}
|
||||
globalMinioPort = saveGlobalPort
|
||||
}
|
70
cmd/utils.go
70
cmd/utils.go
@ -45,44 +45,6 @@ func cloneHeader(h http.Header) http.Header {
|
||||
return h2
|
||||
}
|
||||
|
||||
// checkDuplicates - function to validate if there are duplicates in a slice of strings.
|
||||
func checkDuplicateStrings(list []string) error {
|
||||
// Empty lists are not allowed.
|
||||
if len(list) == 0 {
|
||||
return errInvalidArgument
|
||||
}
|
||||
// Empty keys are not allowed.
|
||||
for _, key := range list {
|
||||
if key == "" {
|
||||
return errInvalidArgument
|
||||
}
|
||||
}
|
||||
listMaps := make(map[string]int)
|
||||
// Navigate through each configs and count the entries.
|
||||
for _, key := range list {
|
||||
listMaps[key]++
|
||||
}
|
||||
// Validate if there are any duplicate counts.
|
||||
for key, count := range listMaps {
|
||||
if count != 1 {
|
||||
return fmt.Errorf("Duplicate key: \"%s\" found of count: \"%d\"", key, count)
|
||||
}
|
||||
}
|
||||
// No duplicates.
|
||||
return nil
|
||||
}
|
||||
|
||||
// splitStr splits a string into n parts, empty strings are added
|
||||
// if we are not able to reach n elements
|
||||
func splitStr(path, sep string, n int) []string {
|
||||
splits := strings.SplitN(path, sep, n)
|
||||
// Add empty strings if we found elements less than nr
|
||||
for i := n - len(splits); i > 0; i-- {
|
||||
splits = append(splits, "")
|
||||
}
|
||||
return splits
|
||||
}
|
||||
|
||||
// Convert url path into bucket and object name.
|
||||
func urlPath2BucketObjectName(u *url.URL) (bucketName, objectName string) {
|
||||
if u == nil {
|
||||
@ -95,10 +57,11 @@ func urlPath2BucketObjectName(u *url.URL) (bucketName, objectName string) {
|
||||
|
||||
// Split urlpath using slash separator into a given number of
|
||||
// expected tokens.
|
||||
tokens := splitStr(urlPath, slashSeparator, 2)
|
||||
|
||||
// Extract bucket and objects.
|
||||
bucketName, objectName = tokens[0], tokens[1]
|
||||
tokens := strings.SplitN(urlPath, slashSeparator, 2)
|
||||
bucketName = tokens[0]
|
||||
if len(tokens) == 2 {
|
||||
objectName = tokens[1]
|
||||
}
|
||||
|
||||
// Success.
|
||||
return bucketName, objectName
|
||||
@ -110,29 +73,6 @@ const (
|
||||
httpsScheme = "https"
|
||||
)
|
||||
|
||||
// checkDuplicates - function to validate if there are duplicates in a slice of endPoints.
|
||||
func checkDuplicateEndpoints(endpoints []*url.URL) error {
|
||||
var strs []string
|
||||
for _, ep := range endpoints {
|
||||
strs = append(strs, ep.String())
|
||||
}
|
||||
return checkDuplicateStrings(strs)
|
||||
}
|
||||
|
||||
// Find local node through the command line arguments. Returns in `host:port` format.
|
||||
func getLocalAddress(srvCmdConfig serverCmdConfig) string {
|
||||
if !globalIsDistXL {
|
||||
return srvCmdConfig.serverAddr
|
||||
}
|
||||
for _, ep := range srvCmdConfig.endpoints {
|
||||
// Validates if remote endpoint is local.
|
||||
if isLocalStorage(ep) {
|
||||
return ep.Host
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// xmlDecoder provide decoded value in xml.
|
||||
func xmlDecoder(body io.Reader, v interface{}, size int64) error {
|
||||
var lbody io.Reader
|
||||
|
@ -18,12 +18,9 @@ package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@ -52,57 +49,6 @@ func TestCloneHeader(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Tests check duplicates function.
|
||||
func TestCheckDuplicates(t *testing.T) {
|
||||
tests := []struct {
|
||||
list []string
|
||||
err error
|
||||
shouldPass bool
|
||||
}{
|
||||
// Test 1 - for '/tmp/1' repeated twice.
|
||||
{
|
||||
list: []string{"/tmp/1", "/tmp/1", "/tmp/2", "/tmp/3"},
|
||||
err: fmt.Errorf("Duplicate key: \"/tmp/1\" found of count: \"2\""),
|
||||
shouldPass: false,
|
||||
},
|
||||
// Test 2 - for '/tmp/1' repeated thrice.
|
||||
{
|
||||
list: []string{"/tmp/1", "/tmp/1", "/tmp/1", "/tmp/3"},
|
||||
err: fmt.Errorf("Duplicate key: \"/tmp/1\" found of count: \"3\""),
|
||||
shouldPass: false,
|
||||
},
|
||||
// Test 3 - empty string.
|
||||
{
|
||||
list: []string{""},
|
||||
err: errInvalidArgument,
|
||||
shouldPass: false,
|
||||
},
|
||||
// Test 4 - empty string.
|
||||
{
|
||||
list: nil,
|
||||
err: errInvalidArgument,
|
||||
shouldPass: false,
|
||||
},
|
||||
// Test 5 - non repeated strings.
|
||||
{
|
||||
list: []string{"/tmp/1", "/tmp/2", "/tmp/3"},
|
||||
err: nil,
|
||||
shouldPass: true,
|
||||
},
|
||||
}
|
||||
|
||||
// Validate if function runs as expected.
|
||||
for i, test := range tests {
|
||||
err := checkDuplicateStrings(test.list)
|
||||
if test.shouldPass && err != test.err {
|
||||
t.Errorf("Test: %d, Expected %s got %s", i+1, test.err, err)
|
||||
}
|
||||
if !test.shouldPass && err.Error() != test.err.Error() {
|
||||
t.Errorf("Test: %d, Expected %s got %s", i+1, test.err, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Tests maximum object size.
|
||||
func TestMaxObjectSize(t *testing.T) {
|
||||
sizes := []struct {
|
||||
@ -275,122 +221,6 @@ func TestStartProfiler(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Tests fetch local address.
|
||||
func TestLocalAddress(t *testing.T) {
|
||||
if runtime.GOOS == globalWindowsOSName {
|
||||
return
|
||||
}
|
||||
|
||||
currentIsDistXL := globalIsDistXL
|
||||
defer func() {
|
||||
globalIsDistXL = currentIsDistXL
|
||||
}()
|
||||
|
||||
// need to set this to avoid stale values from other tests.
|
||||
globalMinioPort = "9000"
|
||||
globalMinioHost = ""
|
||||
testCases := []struct {
|
||||
isDistXL bool
|
||||
srvCmdConfig serverCmdConfig
|
||||
localAddr string
|
||||
}{
|
||||
// Test 1 - local address is found.
|
||||
{
|
||||
isDistXL: true,
|
||||
srvCmdConfig: serverCmdConfig{
|
||||
endpoints: []*url.URL{{
|
||||
Scheme: httpScheme,
|
||||
Host: "localhost:9000",
|
||||
Path: "/mnt/disk1",
|
||||
}, {
|
||||
Scheme: httpScheme,
|
||||
Host: "1.1.1.2:9000",
|
||||
Path: "/mnt/disk2",
|
||||
}, {
|
||||
Scheme: httpScheme,
|
||||
Host: "1.1.2.1:9000",
|
||||
Path: "/mnt/disk3",
|
||||
}, {
|
||||
Scheme: httpScheme,
|
||||
Host: "1.1.2.2:9000",
|
||||
Path: "/mnt/disk4",
|
||||
}},
|
||||
},
|
||||
localAddr: net.JoinHostPort("localhost", globalMinioPort),
|
||||
},
|
||||
// Test 2 - local address is everything.
|
||||
{
|
||||
isDistXL: false,
|
||||
srvCmdConfig: serverCmdConfig{
|
||||
serverAddr: net.JoinHostPort("", globalMinioPort),
|
||||
endpoints: []*url.URL{{
|
||||
Path: "/mnt/disk1",
|
||||
}, {
|
||||
Path: "/mnt/disk2",
|
||||
}, {
|
||||
Path: "/mnt/disk3",
|
||||
}, {
|
||||
Path: "/mnt/disk4",
|
||||
}},
|
||||
},
|
||||
localAddr: net.JoinHostPort("", globalMinioPort),
|
||||
},
|
||||
// Test 3 - local address is not found.
|
||||
{
|
||||
isDistXL: true,
|
||||
srvCmdConfig: serverCmdConfig{
|
||||
endpoints: []*url.URL{{
|
||||
Scheme: httpScheme,
|
||||
Host: "1.1.1.1:9000",
|
||||
Path: "/mnt/disk2",
|
||||
}, {
|
||||
Scheme: httpScheme,
|
||||
Host: "1.1.1.2:9000",
|
||||
Path: "/mnt/disk2",
|
||||
}, {
|
||||
Scheme: httpScheme,
|
||||
Host: "1.1.2.1:9000",
|
||||
Path: "/mnt/disk3",
|
||||
}, {
|
||||
Scheme: httpScheme,
|
||||
Host: "1.1.2.2:9000",
|
||||
Path: "/mnt/disk4",
|
||||
}},
|
||||
},
|
||||
localAddr: "",
|
||||
},
|
||||
// Test 4 - in case of FS mode, with SSL, the host
|
||||
// name is specified in the --address option on the
|
||||
// server command line.
|
||||
{
|
||||
isDistXL: false,
|
||||
srvCmdConfig: serverCmdConfig{
|
||||
serverAddr: "play.minio.io:9000",
|
||||
endpoints: []*url.URL{{
|
||||
Path: "/mnt/disk1",
|
||||
}, {
|
||||
Path: "/mnt/disk2",
|
||||
}, {
|
||||
Path: "/mnt/disk3",
|
||||
}, {
|
||||
Path: "/mnt/disk4",
|
||||
}},
|
||||
},
|
||||
localAddr: "play.minio.io:9000",
|
||||
},
|
||||
}
|
||||
|
||||
// Validates fetching local address.
|
||||
for i, testCase := range testCases {
|
||||
globalIsDistXL = testCase.isDistXL
|
||||
localAddr := getLocalAddress(testCase.srvCmdConfig)
|
||||
if localAddr != testCase.localAddr {
|
||||
t.Fatalf("Test %d: Expected %s, got %s", i+1, testCase.localAddr, localAddr)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// TestCheckURL tests valid url.
|
||||
func TestCheckURL(t *testing.T) {
|
||||
testCases := []struct {
|
||||
|
@ -37,13 +37,8 @@ func TestHealFormatXL(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err := parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Everything is fine, should return nil
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -59,13 +54,8 @@ func TestHealFormatXL(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Disks 0..15 are nil
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -84,13 +74,8 @@ func TestHealFormatXL(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// One disk returns Faulty Disk
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -112,13 +97,8 @@ func TestHealFormatXL(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// One disk is not found, heal corrupted disks should return nil
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -134,13 +114,8 @@ func TestHealFormatXL(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Remove format.json of all disks
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -160,13 +135,8 @@ func TestHealFormatXL(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Corrupted format json in one disk
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -186,13 +156,8 @@ func TestHealFormatXL(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Remove format.json on 3 disks.
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -212,13 +177,8 @@ func TestHealFormatXL(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// One disk is not found, heal corrupted disks should return nil
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -246,13 +206,8 @@ func TestHealFormatXL(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// One disk is not found, heal corrupted disks should return nil
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -286,13 +241,8 @@ func TestUndoMakeBucket(t *testing.T) {
|
||||
}
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
endpoints, err := parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Remove format.json on 16 disks.
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -331,13 +281,8 @@ func TestQuickHeal(t *testing.T) {
|
||||
}
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
endpoints, err := parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Remove format.json on 16 disks.
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -382,13 +327,8 @@ func TestQuickHeal(t *testing.T) {
|
||||
}
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// One disk is not found, heal corrupted disks should return nil
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -404,13 +344,8 @@ func TestQuickHeal(t *testing.T) {
|
||||
}
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// One disk is not found, heal corrupted disks should return nil
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -441,12 +376,7 @@ func TestListBucketsHeal(t *testing.T) {
|
||||
}
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
endpoints, err := parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -505,13 +435,8 @@ func TestHealObjectXL(t *testing.T) {
|
||||
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
endpoints, err := parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Everything is fine, should return nil
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
@ -392,11 +392,7 @@ func TestShuffleDisks(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
objLayer, _, err := initObjectLayer(endpoints)
|
||||
objLayer, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
|
||||
if err != nil {
|
||||
removeRoots(disks)
|
||||
t.Fatal(err)
|
||||
|
@ -50,12 +50,7 @@ func TestStorageInfo(t *testing.T) {
|
||||
t.Fatalf("Diskinfo total values should be greater 0")
|
||||
}
|
||||
|
||||
endpoints, err := parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error %s", err)
|
||||
}
|
||||
|
||||
storageDisks, err := initStorageDisks(endpoints)
|
||||
storageDisks, err := initStorageDisks(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
@ -145,11 +140,7 @@ func TestNewXL(t *testing.T) {
|
||||
t.Fatalf("Unable to initialize erasure, %s", err)
|
||||
}
|
||||
|
||||
endpoints, err := parseStorageEndpoints(erasureDisks)
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to initialize erasure, %s", err)
|
||||
}
|
||||
|
||||
endpoints := mustGetNewEndpointList(erasureDisks...)
|
||||
storageDisks, err := initStorageDisks(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
|
8
vendor/github.com/minio/minio-go/pkg/set/stringset.go
generated
vendored
8
vendor/github.com/minio/minio-go/pkg/set/stringset.go
generated
vendored
@ -25,8 +25,8 @@ import (
|
||||
// StringSet - uses map as set of strings.
|
||||
type StringSet map[string]struct{}
|
||||
|
||||
// keys - returns StringSet keys.
|
||||
func (set StringSet) keys() []string {
|
||||
// ToSlice - returns StringSet as string slice.
|
||||
func (set StringSet) ToSlice() []string {
|
||||
keys := make([]string, 0, len(set))
|
||||
for k := range set {
|
||||
keys = append(keys, k)
|
||||
@ -141,7 +141,7 @@ func (set StringSet) Union(sset StringSet) StringSet {
|
||||
|
||||
// MarshalJSON - converts to JSON data.
|
||||
func (set StringSet) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal(set.keys())
|
||||
return json.Marshal(set.ToSlice())
|
||||
}
|
||||
|
||||
// UnmarshalJSON - parses JSON data and creates new set with it.
|
||||
@ -169,7 +169,7 @@ func (set *StringSet) UnmarshalJSON(data []byte) error {
|
||||
|
||||
// String - returns printable string of the set.
|
||||
func (set StringSet) String() string {
|
||||
return fmt.Sprintf("%s", set.keys())
|
||||
return fmt.Sprintf("%s", set.ToSlice())
|
||||
}
|
||||
|
||||
// NewStringSet - creates new string set.
|
||||
|
6
vendor/vendor.json
vendored
6
vendor/vendor.json
vendored
@ -227,10 +227,10 @@
|
||||
"revisionTime": "2016-12-20T20:43:13Z"
|
||||
},
|
||||
{
|
||||
"checksumSHA1": "A8QOw1aWwc+RtjGozY0XeS5varo=",
|
||||
"checksumSHA1": "maUy+dbN6VfTTnfErrAW2lLit1w=",
|
||||
"path": "github.com/minio/minio-go/pkg/set",
|
||||
"revision": "9e734013294ab153b0bdbe182738bcddd46f1947",
|
||||
"revisionTime": "2016-08-18T00:31:20Z"
|
||||
"revision": "7a3619e41885dcbcfafee193c10eb80530c2be53",
|
||||
"revisionTime": "2017-02-17T20:03:45Z"
|
||||
},
|
||||
{
|
||||
"checksumSHA1": "URVle4qtadmW9w9BulDRHY3kxnA=",
|
||||
|
Loading…
Reference in New Issue
Block a user