MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-24 06:05:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add extensive endpoints validation (#4019)

Browse Source
This commit is contained in:
Bala FA 2017-04-12 04:14:27 +05:30 committed by Harshavardhana
parent 1b1b9e4801
commit de204a0a52
48 changed files with 1432 additions and 2269 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
cmd/admin-handlers_test.go
View File

@ -158,12 +158,7 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
// Initialize boot time
globalBootTime = UTCNow()
// Set globalEndpoints for a single node XL setup.
for _, xlDir := range xlDirs {
globalEndpoints = append(globalEndpoints, &url.URL{
Path: xlDir,
})
}
globalEndpoints = mustGetNewEndpointList(xlDirs...)
// Set globalIsXL to indicate that the setup uses an erasure code backend.
globalIsXL = true
@ -301,14 +296,8 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
// Initialize admin peers to make admin RPC calls. Note: In a
// single node setup, this degenerates to a simple function
// call under the hood.
eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
if err != nil {
t.Fatalf("Failed to parse storage end point - %v", err)
}
// Set globalMinioAddr to be able to distinguish local endpoints from remote.
globalMinioAddr = eps[0].Host
initGlobalAdminPeers(eps)
globalMinioAddr = "127.0.0.1:9000"
initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
// Setting up a go routine to simulate ServerMux's
// handleServiceSignals for stop and restart commands.
@ -367,14 +356,8 @@ func TestServiceSetCreds(t *testing.T) {
// Initialize admin peers to make admin RPC calls. Note: In a
// single node setup, this degenerates to a simple function
// call under the hood.
eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
if err != nil {
t.Fatalf("Failed to parse storage end point - %v", err)
}
// Set globalMinioAddr to be able to distinguish local endpoints from remote.
globalMinioAddr = eps[0].Host
initGlobalAdminPeers(eps)
globalMinioAddr = "127.0.0.1:9000"
initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
credentials := serverConfig.GetCredential()
var body []byte
@ -455,14 +438,8 @@ func TestListLocksHandler(t *testing.T) {
defer adminTestBed.TearDown()
// Initialize admin peers to make admin RPC calls.
eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
if err != nil {
t.Fatalf("Failed to parse storage end point - %v", err)
}
// Set globalMinioAddr to be able to distinguish local endpoints from remote.
globalMinioAddr = eps[0].Host
initGlobalAdminPeers(eps)
globalMinioAddr = "127.0.0.1:9000"
initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
testCases := []struct {
bucket string
@ -530,11 +507,7 @@ func TestClearLocksHandler(t *testing.T) {
defer adminTestBed.TearDown()
// Initialize admin peers to make admin RPC calls.
eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
if err != nil {
t.Fatalf("Failed to parse storage end point - %v", err)
}
initGlobalAdminPeers(eps)
initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
testCases := []struct {
bucket string
@ -1238,14 +1211,8 @@ func TestGetConfigHandler(t *testing.T) {
defer adminTestBed.TearDown()
// Initialize admin peers to make admin RPC calls.
eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
if err != nil {
t.Fatalf("Failed to parse storage end point - %v", err)
}
// Set globalMinioAddr to be able to distinguish local endpoints from remote.
globalMinioAddr = eps[0].Host
initGlobalAdminPeers(eps)
globalMinioAddr = "127.0.0.1:9000"
initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
// Prepare query params for get-config mgmt REST API.
queryVal := url.Values{}
@ -1273,14 +1240,8 @@ func TestSetConfigHandler(t *testing.T) {
defer adminTestBed.TearDown()
// Initialize admin peers to make admin RPC calls.
eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
if err != nil {
t.Fatalf("Failed to parse storage end point - %v", err)
}
// Set globalMinioAddr to be able to distinguish local endpoints from remote.
globalMinioAddr = eps[0].Host
initGlobalAdminPeers(eps)
globalMinioAddr = "127.0.0.1:9000"
initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
// SetConfigHandler restarts minio setup - need to start a
// signal receiver to receive on globalServiceSignalCh.
@ -1321,14 +1282,8 @@ func TestAdminServerInfo(t *testing.T) {
defer adminTestBed.TearDown()
// Initialize admin peers to make admin RPC calls.
eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
if err != nil {
t.Fatalf("Failed to parse storage end point - %v", err)
}
// Set globalMinioAddr to be able to distinguish local endpoints from remote.
globalMinioAddr = eps[0].Host
initGlobalAdminPeers(eps)
globalMinioAddr = "127.0.0.1:9000"
initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
// Prepare query params for set-config mgmt REST API.
queryVal := url.Values{}

61
cmd/admin-rpc-client.go
View File

@ -20,7 +20,7 @@ import (
"encoding/json"
"errors"
"fmt"
"net/url"
"net"
"os"
"path"
"path/filepath"
@ -28,6 +28,8 @@ import (
"sort"
"sync"
"time"
"github.com/minio/minio-go/pkg/set"
)
const (
@ -211,52 +213,43 @@ type adminPeer struct {
type adminPeers []adminPeer
// makeAdminPeers - helper function to construct a collection of adminPeer.
func makeAdminPeers(eps []*url.URL) adminPeers {
var servicePeers []adminPeer
// map to store peers that are already added to ret
seenAddr := make(map[string]bool)
// add local (self) as peer in the array
servicePeers = append(servicePeers, adminPeer{
globalMinioAddr,
func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
thisPeer := globalMinioAddr
if globalMinioHost == "" {
thisPeer = net.JoinHostPort("localhost", globalMinioPort)
}
adminPeerList = append(adminPeerList, adminPeer{
thisPeer,
localAdminClient{},
})
seenAddr[globalMinioAddr] = true
serverCred := serverConfig.GetCredential()
// iterate over endpoints to find new remote peers and add
// them to ret.
for _, ep := range eps {
if ep.Host == "" {
hostSet := set.CreateStringSet(globalMinioAddr)
cred := serverConfig.GetCredential()
serviceEndpoint := path.Join(minioReservedBucketPath, adminPath)
for _, host := range GetRemotePeers(endpoints) {
if hostSet.Contains(host) {
continue
}
// Check if the remote host has been added already
if !seenAddr[ep.Host] {
cfg := authConfig{
accessKey: serverCred.AccessKey,
secretKey: serverCred.SecretKey,
serverAddr: ep.Host,
hostSet.Add(host)
adminPeerList = append(adminPeerList, adminPeer{
addr: host,
cmdRunner: &remoteAdminClient{newAuthRPCClient(authConfig{
accessKey: cred.AccessKey,
secretKey: cred.SecretKey,
serverAddr: host,
serviceEndpoint: serviceEndpoint,
secureConn: globalIsSSL,
serviceEndpoint: path.Join(minioReservedBucketPath, adminPath),
serviceName: "Admin",
}
servicePeers = append(servicePeers, adminPeer{
addr: ep.Host,
cmdRunner: &remoteAdminClient{newAuthRPCClient(cfg)},
})},
})
seenAddr[ep.Host] = true
}
}
return servicePeers
return adminPeerList
}
// Initialize global adminPeer collection.
func initGlobalAdminPeers(eps []*url.URL) {
globalAdminPeers = makeAdminPeers(eps)
func initGlobalAdminPeers(endpoints EndpointList) {
globalAdminPeers = makeAdminPeers(endpoints)
}
// invokeServiceCmd - Invoke Restart command.

5
cmd/admin-rpc-server_test.go
View File

@ -18,7 +18,6 @@ package cmd
import (
"encoding/json"
"net/url"
"testing"
)
@ -86,9 +85,7 @@ func TestReInitDisks(t *testing.T) {
defer removeRoots(xlDirs)
// Set globalEndpoints for a single node XL setup.
for _, xlDir := range xlDirs {
globalEndpoints = append(globalEndpoints, &url.URL{Path: xlDir})
}
globalEndpoints = mustGetNewEndpointList(xlDirs...)
// Setup admin rpc server for an XL backend.
globalIsXL = true

14
cmd/bucket-notification-utils.go
View File

@ -16,7 +16,11 @@
package cmd
import "strings"
import (
"strings"
"github.com/minio/minio-go/pkg/set"
)
// List of valid event types.
var suppportedEventTypes = map[string]struct{}{
@ -207,16 +211,14 @@ func validateQueueConfigs(queueConfigs []queueConfig) APIErrorCode {
// Check all the queue configs for any duplicates.
func checkDuplicateQueueConfigs(configs []queueConfig) APIErrorCode {
var queueConfigARNS []string
queueConfigARNS := set.NewStringSet()
// Navigate through each configs and count the entries.
for _, config := range configs {
queueConfigARNS = append(queueConfigARNS, config.QueueARN)
queueConfigARNS.Add(config.QueueARN)
}
// Check if there are any duplicate counts.
if err := checkDuplicateStrings(queueConfigARNS); err != nil {
errorIf(err, "Invalid queue configs found.")
if len(queueConfigARNS) != len(configs) {
return ErrOverlappingConfigs
}

64
cmd/checkport.go
View File

@ -1,64 +0,0 @@
/*
* Minio Cloud Storage, (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import (
"net"
"os"
"syscall"
)
// checkPortAvailability - check if given port is already in use.
// Note: The check method tries to listen on given port and closes it.
// It is possible to have a disconnected client in this tiny window of time.
func checkPortAvailability(port string) error {
network := [3]string{"tcp", "tcp4", "tcp6"}
for _, n := range network {
l, err := net.Listen(n, net.JoinHostPort("", port))
if err != nil {
if isAddrInUse(err) {
// Return error if another process is listening on the
// same port.
return err
}
// Ignore any other error (ex. EAFNOSUPPORT)
continue
}
// look for error so we don't have dangling connection
if err = l.Close(); err != nil {
return err
}
}
return nil
}
// Return true if err is "address already in use" error.
// syscall.EADDRINUSE is available on all OSes.
func isAddrInUse(err error) bool {
if opErr, ok := err.(*net.OpError); ok {
if sysErr, ok := opErr.Err.(*os.SyscallError); ok {
if errno, ok := sysErr.Err.(syscall.Errno); ok {
if errno == syscall.EADDRINUSE {
return true
}
}
}
}
return false
}

54
cmd/checkport_test.go
View File

@ -1,54 +0,0 @@
/*
* Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import (
"net"
"runtime"
"testing"
)
// Tests for port availability logic written for server startup sequence.
func TestCheckPortAvailability(t *testing.T) {
tests := []struct {
port string
}{
{getFreePort()},
{getFreePort()},
}
for _, test := range tests {
// This test should pass if the ports are available
err := checkPortAvailability(test.port)
if err != nil {
t.Fatalf("checkPortAvailability test failed for port: %s. Error: %v", test.port, err)
}
// Now use the ports and check again
ln, err := net.Listen("tcp", net.JoinHostPort("", test.port))
if err != nil {
t.Fail()
}
defer ln.Close()
err = checkPortAvailability(test.port)
// Skip if the os is windows due to https://github.com/golang/go/issues/7598
if err == nil && runtime.GOOS != globalWindowsOSName {
t.Fatalf("checkPortAvailability should fail for port: %s. Error: %v", test.port, err)
}
}
}

382
cmd/endpoint.go Normal file
View File

@ -0,0 +1,382 @@
/*
* Minio Cloud Storage, (C) 2017 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import (
"fmt"
"net"
"net/url"
"path"
"sort"
"strconv"
"strings"
"github.com/minio/minio-go/pkg/set"
)
// EndpointType - enum for endpoint type.
type EndpointType int
const (
// PathEndpointType - path style endpoint type enum.
PathEndpointType EndpointType = iota + 1
// URLEndpointType - URL style endpoint type enum.
URLEndpointType
)
// Endpoint - any type of endpoint.
type Endpoint struct {
*url.URL
IsLocal bool
}
func (endpoint Endpoint) String() string {
if endpoint.Host == "" {
return endpoint.Path
}
return endpoint.URL.String()
}
// Type - returns type of endpoint.
func (endpoint Endpoint) Type() EndpointType {
if endpoint.Host == "" {
return PathEndpointType
}
return URLEndpointType
}
// SetHTTPS - sets secure http for URLEndpointType.
func (endpoint Endpoint) SetHTTPS() {
if endpoint.Host != "" {
endpoint.Scheme = "https"
}
}
// SetHTTP - sets insecure http for URLEndpointType.
func (endpoint Endpoint) SetHTTP() {
if endpoint.Host != "" {
endpoint.Scheme = "http"
}
}
// NewEndpoint - returns new endpoint based on given arguments.
func NewEndpoint(arg string) (Endpoint, error) {
// isEmptyPath - check whether given path is not empty.
isEmptyPath := func(path string) bool {
return path == "" || path == "." || path == "/" || path == `\`
}
if isEmptyPath(arg) {
return Endpoint{}, fmt.Errorf("empty or root endpoint is not supported")
}
var isLocal bool
u, err := url.Parse(arg)
if err == nil && u.Host != "" {
// URL style of endpoint.
// Valid URL style endpoint is
// - Scheme field must contain "http" or "https"
// - All field should be empty except Host and Path.
if !((u.Scheme == "http" || u.Scheme == "https") &&
u.User == nil && u.Opaque == "" && u.ForceQuery == false && u.RawQuery == "" && u.Fragment == "") {
return Endpoint{}, fmt.Errorf("invalid URL endpoint format")
}
host, port, err := net.SplitHostPort(u.Host)
if err != nil {
if !strings.Contains(err.Error(), "missing port in address") {
return Endpoint{}, fmt.Errorf("invalid URL endpoint format: %s", err)
}
host = u.Host
} else {
var p int
p, err = strconv.Atoi(port)
if err != nil {
return Endpoint{}, fmt.Errorf("invalid URL endpoint format: invalid port number")
} else if p < 1 || p > 65535 {
return Endpoint{}, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535")
}
}
if host == "" {
return Endpoint{}, fmt.Errorf("invalid URL endpoint format: empty host name")
}
// As this is path in the URL, we should use path package, not filepath package.
// On MS Windows, filepath.Clean() converts into Windows path style ie `/foo` becomes `\foo`
u.Path = path.Clean(u.Path)
if isEmptyPath(u.Path) {
return Endpoint{}, fmt.Errorf("empty or root path is not supported in URL endpoint")
}
// Get IPv4 address of the host.
hostIPs, err := getHostIP4(host)
if err != nil {
return Endpoint{}, err
}
// If intersection of two IP sets is not empty, then the host is local host.
isLocal = !localIP4.Intersection(hostIPs).IsEmpty()
} else {
u = &url.URL{Path: path.Clean(arg)}
isLocal = true
}
return Endpoint{
URL: u,
IsLocal: isLocal,
}, nil
}
// EndpointList - list of same type of endpoint.
type EndpointList []Endpoint
// Swap - helper method for sorting.
func (endpoints EndpointList) Swap(i, j int) {
endpoints[i], endpoints[j] = endpoints[j], endpoints[i]
}
// Len - helper method for sorting.
func (endpoints EndpointList) Len() int {
return len(endpoints)
}
// Less - helper method for sorting.
func (endpoints EndpointList) Less(i, j int) bool {
return endpoints[i].String() < endpoints[j].String()
}
// SetHTTPS - sets secure http for URLEndpointType.
func (endpoints EndpointList) SetHTTPS() {
for i := range endpoints {
endpoints[i].SetHTTPS()
}
}
// SetHTTP - sets insecure http for URLEndpointType.
func (endpoints EndpointList) SetHTTP() {
for i := range endpoints {
endpoints[i].SetHTTP()
}
}
// NewEndpointList - returns new endpoint list based on input args.
func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
// isValidDistribution - checks whether given count is a valid distribution for erasure coding.
isValidDistribution := func(count int) bool {
return (count >= 4 && count <= 16 && count%2 == 0)
}
// Check whether no. of args are valid for XL distribution.
if !isValidDistribution(len(args)) {
return nil, fmt.Errorf("total endpoints %d found. For XL/Distribute, it should be 4, 6, 8, 10, 12, 14 or 16", len(args))
}
var endpointType EndpointType
var scheme string
uniqueArgs := set.NewStringSet()
// Loop through args and adds to endpoint list.
for i, arg := range args {
endpoint, err := NewEndpoint(arg)
if err != nil {
return nil, fmt.Errorf("'%s': %s", arg, err.Error())
}
// All endpoints have to be same type and scheme if applicable.
if i == 0 {
endpointType = endpoint.Type()
scheme = endpoint.Scheme
} else if endpoint.Type() != endpointType {
return nil, fmt.Errorf("mixed style endpoints are not supported")
} else if endpoint.Scheme != scheme {
return nil, fmt.Errorf("mixed scheme is not supported")
}
arg = endpoint.String()
if uniqueArgs.Contains(arg) {
return nil, fmt.Errorf("duplicate endpoints found")
}
uniqueArgs.Add(arg)
endpoints = append(endpoints, endpoint)
}
sort.Sort(endpoints)
return endpoints, nil
}
// CreateEndpoints - validates and creates new endpoints for given args.
func CreateEndpoints(serverAddr string, args ...string) (string, EndpointList, SetupType, error) {
var endpoints EndpointList
var setupType SetupType
var err error
// Check whether serverAddr is valid for this host.
if err = CheckLocalServerAddr(serverAddr); err != nil {
return serverAddr, endpoints, setupType, err
}
_, serverAddrPort := mustSplitHostPort(serverAddr)
// For single arg, return FS setup.
if len(args) == 1 {
var endpoint Endpoint
endpoint, err = NewEndpoint(args[0])
if err != nil {
return serverAddr, endpoints, setupType, err
}
if endpoint.Type() != PathEndpointType {
return serverAddr, endpoints, setupType, fmt.Errorf("use path style endpoint for FS setup")
}
endpoints = append(endpoints, endpoint)
setupType = FSSetupType
return serverAddr, endpoints, setupType, nil
}
// Convert args to endpoints
if endpoints, err = NewEndpointList(args...); err != nil {
return serverAddr, endpoints, setupType, err
}
// Return XL setup when all endpoints are path style.
if endpoints[0].Type() == PathEndpointType {
setupType = XLSetupType
return serverAddr, endpoints, setupType, nil
}
// Here all endpoints are URL style.
endpointPathSet := set.NewStringSet()
localEndpointCount := 0
localServerAddrSet := set.NewStringSet()
localPortSet := set.NewStringSet()
for _, endpoint := range endpoints {
endpointPathSet.Add(endpoint.Path)
if endpoint.IsLocal {
localServerAddrSet.Add(endpoint.Host)
var port string
_, port, err = net.SplitHostPort(endpoint.Host)
if err != nil {
port = serverAddrPort
}
localPortSet.Add(port)
localEndpointCount++
}
}
// No local endpoint found.
if localEndpointCount == 0 {
return serverAddr, endpoints, setupType, fmt.Errorf("no endpoint found for this host")
}
// Check whether same path is not used in endpoints of a host.
{
pathIPMap := make(map[string]set.StringSet)
for _, endpoint := range endpoints {
var host string
host, _, err = net.SplitHostPort(endpoint.Host)
if err != nil {
host = endpoint.Host
}
hostIPSet, _ := getHostIP4(host)
if IPSet, ok := pathIPMap[endpoint.Path]; ok {
if !IPSet.Intersection(hostIPSet).IsEmpty() {
err = fmt.Errorf("path '%s' can not be served from different address/port", endpoint.Path)
return serverAddr, endpoints, setupType, err
}
} else {
pathIPMap[endpoint.Path] = hostIPSet
}
}
}
// Check whether serverAddrPort matches at least in one of port used in local endpoints.
{
if !localPortSet.Contains(serverAddrPort) {
if len(localPortSet) > 1 {
err = fmt.Errorf("port number in server address must match with one of the port in local endpoints")
} else {
err = fmt.Errorf("server address and local endpoint have different ports")
}
return serverAddr, endpoints, setupType, err
}
}
// If all endpoints are pointing to local host and having same port number, then this is XL setup using URL style endpoints.
if len(endpoints) == localEndpointCount && len(localPortSet) == 1 {
if len(localServerAddrSet) > 1 {
// TODO: Eventhough all endpoints are local, the local host is referred by different IP/name.
// eg '172.0.0.1', 'localhost' and 'mylocalhostname' point to same local host.
//
// In this case, we bind to 0.0.0.0 ie to all interfaces.
// The actual way to do is bind to only IPs in uniqueLocalHosts.
serverAddr = net.JoinHostPort("", serverAddrPort)
}
endpointPaths := endpointPathSet.ToSlice()
endpoints, _ = NewEndpointList(endpointPaths...)
setupType = XLSetupType
return serverAddr, endpoints, setupType, nil
}
// Add missing port in all endpoints.
for i := range endpoints {
_, port, err := net.SplitHostPort(endpoints[i].Host)
if err != nil {
endpoints[i].Host = net.JoinHostPort(endpoints[i].Host, serverAddrPort)
} else if endpoints[i].IsLocal && serverAddrPort != port {
// If endpoint is local, but port is different than serverAddrPort, then make it as remote.
endpoints[i].IsLocal = false
}
}
// This is DistXL setup.
setupType = DistXLSetupType
return serverAddr, endpoints, setupType, nil
}
// GetRemotePeers - get hosts information other than this minio service.
func GetRemotePeers(endpoints EndpointList) []string {
peerSet := set.NewStringSet()
for _, endpoint := range endpoints {
if endpoint.Type() != URLEndpointType {
continue
}
peer := endpoint.Host
if endpoint.IsLocal {
if _, port := mustSplitHostPort(peer); port == globalMinioPort {
continue
}
}
peerSet.Add(peer)
}
return peerSet.ToSlice()
}

312
cmd/endpoint_test.go Normal file
View File

@ -0,0 +1,312 @@
/*
* Minio Cloud Storage, (C) 2017 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import (
"fmt"
"net/url"
"reflect"
"runtime"
"strings"
"testing"
)
func TestNewEndpoint(t *testing.T) {
u1, _ := url.Parse("http://localhost/path")
u2, _ := url.Parse("https://example.org/path")
u3, _ := url.Parse("http://127.0.0.1:8080/path")
u4, _ := url.Parse("http://192.168.253.200/path")
errMsg := ": no such host"
if runtime.GOOS == "windows" {
errMsg = ": No such host is known."
}
testCases := []struct {
arg string
expectedEndpoint Endpoint
expectedType EndpointType
expectedErr error
}{
{"foo", Endpoint{URL: &url.URL{Path: "foo"}, IsLocal: true}, PathEndpointType, nil},
{"/foo", Endpoint{URL: &url.URL{Path: "/foo"}, IsLocal: true}, PathEndpointType, nil},
{`\foo`, Endpoint{URL: &url.URL{Path: `\foo`}, IsLocal: true}, PathEndpointType, nil},
{"C", Endpoint{URL: &url.URL{Path: `C`}, IsLocal: true}, PathEndpointType, nil},
{"C:", Endpoint{URL: &url.URL{Path: `C:`}, IsLocal: true}, PathEndpointType, nil},
{"C:/", Endpoint{URL: &url.URL{Path: "C:"}, IsLocal: true}, PathEndpointType, nil},
{`C:\`, Endpoint{URL: &url.URL{Path: `C:\`}, IsLocal: true}, PathEndpointType, nil},
{`C:\foo`, Endpoint{URL: &url.URL{Path: `C:\foo`}, IsLocal: true}, PathEndpointType, nil},
{"C:/foo", Endpoint{URL: &url.URL{Path: "C:/foo"}, IsLocal: true}, PathEndpointType, nil},
{`C:\\foo`, Endpoint{URL: &url.URL{Path: `C:\\foo`}, IsLocal: true}, PathEndpointType, nil},
{"http:path", Endpoint{URL: &url.URL{Path: "http:path"}, IsLocal: true}, PathEndpointType, nil},
{"http:/path", Endpoint{URL: &url.URL{Path: "http:/path"}, IsLocal: true}, PathEndpointType, nil},
{"http:///path", Endpoint{URL: &url.URL{Path: "http:/path"}, IsLocal: true}, PathEndpointType, nil},
{"http://localhost/path", Endpoint{URL: u1, IsLocal: true}, URLEndpointType, nil},
{"http://localhost/path//", Endpoint{URL: u1, IsLocal: true}, URLEndpointType, nil},
{"https://example.org/path", Endpoint{URL: u2}, URLEndpointType, nil},
{"http://127.0.0.1:8080/path", Endpoint{URL: u3, IsLocal: true}, URLEndpointType, nil},
{"http://192.168.253.200/path", Endpoint{URL: u4}, URLEndpointType, nil},
{"", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
{".", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
{"/", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
{`\`, Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
{"c://foo", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format")},
{"ftp://foo", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format")},
{"http://server/path?location", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format")},
{"http://:/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: invalid port number")},
{"http://:8080/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: empty host name")},
{"http://server:/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: invalid port number")},
{"https://93.184.216.34:808080/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535")},
{"http://server:8080//", Endpoint{}, -1, fmt.Errorf("empty or root path is not supported in URL endpoint")},
{"http://server:8080/", Endpoint{}, -1, fmt.Errorf("empty or root path is not supported in URL endpoint")},
{"http://server/path", Endpoint{}, -1, fmt.Errorf("lookup server" + errMsg)},
}
for _, testCase := range testCases {
endpoint, err := NewEndpoint(testCase.arg)
if testCase.expectedErr == nil {
if err != nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
}
} else if err == nil {
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
} else {
match := false
if strings.HasSuffix(testCase.expectedErr.Error(), errMsg) {
match = strings.HasSuffix(err.Error(), errMsg)
} else {
match = (testCase.expectedErr.Error() == err.Error())
}
if !match {
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
}
}
if err == nil && !reflect.DeepEqual(testCase.expectedEndpoint, endpoint) {
t.Fatalf("endpoint: expected = %+v, got = %+v", testCase.expectedEndpoint, endpoint)
}
if err == nil && testCase.expectedType != endpoint.Type() {
t.Fatalf("type: expected = %+v, got = %+v", testCase.expectedType, endpoint.Type())
}
}
}
func TestNewEndpointList(t *testing.T) {
testCases := []struct {
args []string
expectedErr error
}{
{[]string{"d1", "d2", "d3", "d4"}, nil},
{[]string{"/d1", "/d2", "/d3", "/d4"}, nil},
{[]string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}, nil},
{[]string{"http://example.org/d1", "http://example.com/d1", "http://example.net/d1", "http://example.edu/d1"}, nil},
{[]string{"http://localhost/d1", "http://localhost/d2", "http://example.org/d1", "http://example.org/d2"}, nil},
{[]string{"https://localhost:9000/d1", "https://localhost:9001/d2", "https://localhost:9002/d3", "https://localhost:9003/d4"}, nil},
// // It is valid WRT endpoint list that same path is expected with different port on same server.
{[]string{"https://127.0.0.1:9000/d1", "https://127.0.0.1:9001/d1", "https://127.0.0.1:9002/d1", "https://127.0.0.1:9003/d1"}, nil},
{[]string{"d1", "d2", "d3", "d1"}, fmt.Errorf("duplicate endpoints found")},
{[]string{"d1", "d2", "d3", "./d1"}, fmt.Errorf("duplicate endpoints found")},
{[]string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d1", "http://localhost/d4"}, fmt.Errorf("duplicate endpoints found")},
{[]string{"d1", "d2", "d3", "d4", "d5"}, fmt.Errorf("total endpoints 5 found. For XL/Distribute, it should be 4, 6, 8, 10, 12, 14 or 16")},
{[]string{"ftp://server/d1", "http://server/d2", "http://server/d3", "http://server/d4"}, fmt.Errorf("'ftp://server/d1': invalid URL endpoint format")},
{[]string{"d1", "http://localhost/d2", "d3", "d4"}, fmt.Errorf("mixed style endpoints are not supported")},
{[]string{"http://example.org/d1", "https://example.com/d1", "http://example.net/d1", "https://example.edut/d1"}, fmt.Errorf("mixed scheme is not supported")},
}
for _, testCase := range testCases {
_, err := NewEndpointList(testCase.args...)
if testCase.expectedErr == nil {
if err != nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
}
} else if err == nil {
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
} else if testCase.expectedErr.Error() != err.Error() {
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
}
}
}
func TestCreateEndpoints(t *testing.T) {
case1u1, _ := url.Parse("http://example.com:10000/d4")
case1u2, _ := url.Parse("http://example.org:10000/d3")
case1u3, _ := url.Parse("http://localhost:10000/d1")
case1u4, _ := url.Parse("http://localhost:10000/d2")
case2u1, _ := url.Parse("http://example.com:10000/d4")
case2u2, _ := url.Parse("http://example.org:10000/d3")
case2u3, _ := url.Parse("http://localhost:10000/d1")
case2u4, _ := url.Parse("http://localhost:9000/d2")
case3u1, _ := url.Parse("http://example.com:80/d3")
case3u2, _ := url.Parse("http://example.net:80/d4")
case3u3, _ := url.Parse("http://example.org:9000/d2")
case3u4, _ := url.Parse("http://localhost:80/d1")
case4u1, _ := url.Parse("http://example.com:9000/d3")
case4u2, _ := url.Parse("http://example.net:9000/d4")
case4u3, _ := url.Parse("http://example.org:9000/d2")
case4u4, _ := url.Parse("http://localhost:9000/d1")
case5u1, _ := url.Parse("http://localhost:9000/d1")
case5u2, _ := url.Parse("http://localhost:9001/d2")
case5u3, _ := url.Parse("http://localhost:9002/d3")
case5u4, _ := url.Parse("http://localhost:9003/d4")
case6u1, _ := url.Parse("http://10.0.0.1:9000/export")
case6u2, _ := url.Parse("http://10.0.0.2:9000/export")
case6u3, _ := url.Parse("http://10.0.0.3:9000/export")
case6u4, _ := url.Parse("http://localhost:9001/export")
testCases := []struct {
serverAddr string
args []string
expectedServerAddr string
expectedEndpoints EndpointList
expectedSetupType SetupType
expectedErr error
}{
{"localhost", []string{}, "", EndpointList{}, -1, fmt.Errorf("missing port in address localhost")},
// FS Setup
{"localhost:9000", []string{"http://localhost/d1"}, "", EndpointList{}, -1, fmt.Errorf("use path style endpoint for FS setup")},
{":443", []string{"d1"}, ":443", EndpointList{Endpoint{URL: &url.URL{Path: "d1"}, IsLocal: true}}, FSSetupType, nil},
{"localhost:10000", []string{"/d1"}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true}}, FSSetupType, nil},
{"localhost:10000", []string{"./d1"}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: "d1"}, IsLocal: true}}, FSSetupType, nil},
{"localhost:10000", []string{`\d1`}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: `\d1`}, IsLocal: true}}, FSSetupType, nil},
{"localhost:10000", []string{`.\d1`}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: `.\d1`}, IsLocal: true}}, FSSetupType, nil},
{":8080", []string{"https://example.org/d1", "https://example.org/d2", "https://example.org/d3", "https://example.org/d4"}, "", EndpointList{}, -1, fmt.Errorf("no endpoint found for this host")},
{":8080", []string{"https://example.org/d1", "https://example.com/d2", "https://example.net:8000/d3", "https://example.edu/d1"}, "", EndpointList{}, -1, fmt.Errorf("no endpoint found for this host")},
{"localhost:9000", []string{"https://127.0.0.1:9000/d1", "https://localhost:9001/d1", "https://example.com/d1", "https://example.com/d2"}, "", EndpointList{}, -1, fmt.Errorf("path '/d1' can not be served from different address/port")},
{"localhost:9000", []string{"https://127.0.0.1:8000/d1", "https://localhost:9001/d2", "https://example.com/d1", "https://example.com/d2"}, "", EndpointList{}, -1, fmt.Errorf("port number in server address must match with one of the port in local endpoints")},
{"localhost:10000", []string{"https://127.0.0.1:8000/d1", "https://localhost:8000/d2", "https://example.com/d1", "https://example.com/d2"}, "", EndpointList{}, -1, fmt.Errorf("server address and local endpoint have different ports")},
// XL Setup with PathEndpointType
{":1234", []string{"/d1", "/d2", "d3", "d4"}, ":1234",
EndpointList{
Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true},
Endpoint{URL: &url.URL{Path: "/d2"}, IsLocal: true},
Endpoint{URL: &url.URL{Path: "d3"}, IsLocal: true},
Endpoint{URL: &url.URL{Path: "d4"}, IsLocal: true},
}, XLSetupType, nil},
// XL Setup with URLEndpointType
{":9000", []string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}, ":9000", EndpointList{
Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true},
Endpoint{URL: &url.URL{Path: "/d2"}, IsLocal: true},
Endpoint{URL: &url.URL{Path: "/d3"}, IsLocal: true},
Endpoint{URL: &url.URL{Path: "/d4"}, IsLocal: true},
}, XLSetupType, nil},
// XL Setup with URLEndpointType having mixed naming to local host.
{"127.0.0.1:10000", []string{"http://localhost/d1", "http://localhost/d2", "http://127.0.0.1/d3", "http://127.0.0.1/d4"}, ":10000", EndpointList{
Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true},
Endpoint{URL: &url.URL{Path: "/d2"}, IsLocal: true},
Endpoint{URL: &url.URL{Path: "/d3"}, IsLocal: true},
Endpoint{URL: &url.URL{Path: "/d4"}, IsLocal: true},
}, XLSetupType, nil},
// DistXL type
{"127.0.0.1:10000", []string{"http://localhost/d1", "http://localhost/d2", "http://example.org/d3", "http://example.com/d4"}, "127.0.0.1:10000", EndpointList{
Endpoint{URL: case1u1, IsLocal: false},
Endpoint{URL: case1u2, IsLocal: false},
Endpoint{URL: case1u3, IsLocal: true},
Endpoint{URL: case1u4, IsLocal: true},
}, DistXLSetupType, nil},
{"127.0.0.1:10000", []string{"http://localhost/d1", "http://localhost:9000/d2", "http://example.org/d3", "http://example.com/d4"}, "127.0.0.1:10000", EndpointList{
Endpoint{URL: case2u1, IsLocal: false},
Endpoint{URL: case2u2, IsLocal: false},
Endpoint{URL: case2u3, IsLocal: true},
Endpoint{URL: case2u4, IsLocal: false},
}, DistXLSetupType, nil},
{":80", []string{"http://localhost/d1", "http://example.org:9000/d2", "http://example.com/d3", "http://example.net/d4"}, ":80", EndpointList{
Endpoint{URL: case3u1, IsLocal: false},
Endpoint{URL: case3u2, IsLocal: false},
Endpoint{URL: case3u3, IsLocal: false},
Endpoint{URL: case3u4, IsLocal: true},
}, DistXLSetupType, nil},
{":9000", []string{"http://localhost/d1", "http://example.org/d2", "http://example.com/d3", "http://example.net/d4"}, ":9000", EndpointList{
Endpoint{URL: case4u1, IsLocal: false},
Endpoint{URL: case4u2, IsLocal: false},
Endpoint{URL: case4u3, IsLocal: false},
Endpoint{URL: case4u4, IsLocal: true},
}, DistXLSetupType, nil},
{":9000", []string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://localhost:9002/d3", "http://localhost:9003/d4"}, ":9000", EndpointList{
Endpoint{URL: case5u1, IsLocal: true},
Endpoint{URL: case5u2, IsLocal: false},
Endpoint{URL: case5u3, IsLocal: false},
Endpoint{URL: case5u4, IsLocal: false},
}, DistXLSetupType, nil},
{":9001", []string{"http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export", "http://localhost:9001/export", "http://10.0.0.3:9000/export"}, ":9001", EndpointList{
Endpoint{URL: case6u1, IsLocal: false},
Endpoint{URL: case6u2, IsLocal: false},
Endpoint{URL: case6u3, IsLocal: false},
Endpoint{URL: case6u4, IsLocal: true},
}, DistXLSetupType, nil},
}
for _, testCase := range testCases {
serverAddr, endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, testCase.args...)
if err == nil {
if testCase.expectedErr != nil {
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
} else {
if serverAddr != testCase.expectedServerAddr {
t.Fatalf("serverAddr: expected = %v, got = %v", testCase.expectedServerAddr, serverAddr)
}
if !reflect.DeepEqual(endpoints, testCase.expectedEndpoints) {
t.Fatalf("endpoints: expected = %v, got = %v", testCase.expectedEndpoints, endpoints)
}
if setupType != testCase.expectedSetupType {
t.Fatalf("setupType: expected = %v, got = %v", testCase.expectedSetupType, setupType)
}
}
} else if testCase.expectedErr == nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
} else if err.Error() != testCase.expectedErr.Error() {
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
}
}
}
func TestGetRemotePeers(t *testing.T) {
tempGlobalMinioPort := globalMinioPort
defer func() {
globalMinioPort = tempGlobalMinioPort
}()
globalMinioPort = "9000"
testCases := []struct {
endpointArgs []string
expectedResult []string
}{
{[]string{"/d1", "/d2", "d3", "d4"}, []string{}},
{[]string{"http://localhost:9000/d1", "http://localhost:9000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000"}},
{[]string{"http://localhost:9000/d1", "http://localhost:10000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000", "localhost:10000"}},
{[]string{"http://localhost:9000/d1", "http://example.org:9000/d2", "http://example.com:9000/d3", "http://example.net:9000/d4"}, []string{"example.com:9000", "example.net:9000", "example.org:9000"}},
{[]string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://localhost:9002/d3", "http://localhost:9003/d4"}, []string{"localhost:9001", "localhost:9002", "localhost:9003"}},
}
for _, testCase := range testCases {
endpoints, _ := NewEndpointList(testCase.endpointArgs...)
remotePeers := GetRemotePeers(endpoints)
if !reflect.DeepEqual(remotePeers, testCase.expectedResult) {
t.Fatalf("expected: %v, got: %v", testCase.expectedResult, remotePeers)
}
}
}

6
cmd/erasure-readfile_test.go
View File

@ -194,11 +194,7 @@ func TestErasureReadUtils(t *testing.T) {
if err != nil {
t.Fatal(err)
}
endpoints, err := parseStorageEndpoints(disks)
if err != nil {
t.Fatal(err)
}
objLayer, _, err := initObjectLayer(endpoints)
objLayer, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
if err != nil {
removeRoots(disks)
t.Fatal(err)

25
cmd/event-notifier.go
View File

@ -94,6 +94,21 @@ type eventData struct {
// New notification event constructs a new notification event message from
// input request metadata which completed successfully.
func newNotificationEvent(event eventData) NotificationEvent {
getResponseOriginEndpointKey := func() string {
host := globalMinioHost
if host == "" {
// FIXME: Send FQDN or hostname of this machine than sending IP address.
host = localIP4.ToSlice()[0]
}
scheme := httpScheme
if globalIsSSL {
scheme = httpsScheme
}
return fmt.Sprintf("%s://%s:%s", scheme, host, globalMinioPort)
}
// Fetch the region.
region := serverConfig.GetRegion()
@ -103,14 +118,6 @@ func newNotificationEvent(event eventData) NotificationEvent {
// Time when Minio finished processing the request.
eventTime := UTCNow()
// API endpoint is captured here to be returned back
// to the client for it to differentiate from which
// server the request came from.
var apiEndpoint string
if len(globalAPIEndpoints) >= 1 {
apiEndpoint = globalAPIEndpoints[0]
}
// Fetch a hexadecimal representation of event time in nano seconds.
uniqueID := mustGetRequestID(eventTime)
@ -131,7 +138,7 @@ func newNotificationEvent(event eventData) NotificationEvent {
responseRequestIDKey: uniqueID,
// Following is a custom response element to indicate
// event origin server endpoint.
responseOriginEndpointKey: apiEndpoint,
responseOriginEndpointKey: getResponseOriginEndpointKey(),
},
S3: eventMeta{
SchemaVersion: eventSchemaVersion,

52
cmd/event-notifier_test.go
View File

@ -19,7 +19,6 @@ package cmd
import (
"bytes"
"fmt"
"net"
"reflect"
"testing"
"time"
@ -40,11 +39,7 @@ func TestInitEventNotifierFaultyDisks(t *testing.T) {
t.Fatal("Unable to create directories for FS backend. ", err)
}
defer removeRoots(disks)
endpoints, err := parseStorageEndpoints(disks)
if err != nil {
t.Fatal(err)
}
obj, _, err := initObjectLayer(endpoints)
obj, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
if err != nil {
t.Fatal("Unable to initialize FS backend.", err)
}
@ -97,11 +92,7 @@ func TestInitEventNotifierWithPostgreSQL(t *testing.T) {
if err != nil {
t.Fatal("Unable to create directories for FS backend. ", err)
}
endpoints, err := parseStorageEndpoints(disks)
if err != nil {
t.Fatal(err)
}
fs, _, err := initObjectLayer(endpoints)
fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
if err != nil {
t.Fatal("Unable to initialize FS backend.", err)
}
@ -128,11 +119,7 @@ func TestInitEventNotifierWithNATS(t *testing.T) {
if err != nil {
t.Fatal("Unable to create directories for FS backend. ", err)
}
endpoints, err := parseStorageEndpoints(disks)
if err != nil {
t.Fatal(err)
}
fs, _, err := initObjectLayer(endpoints)
fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
if err != nil {
t.Fatal("Unable to initialize FS backend.", err)
}
@ -159,11 +146,7 @@ func TestInitEventNotifierWithWebHook(t *testing.T) {
if err != nil {
t.Fatal("Unable to create directories for FS backend. ", err)
}
endpoints, err := parseStorageEndpoints(disks)
if err != nil {
t.Fatal(err)
}
fs, _, err := initObjectLayer(endpoints)
fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
if err != nil {
t.Fatal("Unable to initialize FS backend.", err)
}
@ -190,11 +173,7 @@ func TestInitEventNotifierWithAMQP(t *testing.T) {
if err != nil {
t.Fatal("Unable to create directories for FS backend. ", err)
}
endpoints, err := parseStorageEndpoints(disks)
if err != nil {
t.Fatal(err)
}
fs, _, err := initObjectLayer(endpoints)
fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
if err != nil {
t.Fatal("Unable to initialize FS backend.", err)
}
@ -221,11 +200,7 @@ func TestInitEventNotifierWithElasticSearch(t *testing.T) {
if err != nil {
t.Fatal("Unable to create directories for FS backend. ", err)
}
endpoints, err := parseStorageEndpoints(disks)
if err != nil {
t.Fatal(err)
}
fs, _, err := initObjectLayer(endpoints)
fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
if err != nil {
t.Fatal("Unable to initialize FS backend.", err)
}
@ -252,11 +227,7 @@ func TestInitEventNotifierWithRedis(t *testing.T) {
if err != nil {
t.Fatal("Unable to create directories for FS backend. ", err)
}
endpoints, err := parseStorageEndpoints(disks)
if err != nil {
t.Fatal(err)
}
fs, _, err := initObjectLayer(endpoints)
fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
if err != nil {
t.Fatal("Unable to initialize FS backend.", err)
}
@ -276,15 +247,10 @@ func (s *TestPeerRPCServerData) Setup(t *testing.T) {
s.testServer = StartTestPeersRPCServer(t, s.serverType)
// setup port and minio addr
host, port, err := net.SplitHostPort(s.testServer.Server.Listener.Addr().String())
if err != nil {
t.Fatalf("Initialisation error: %v", err)
}
host, port := mustSplitHostPort(s.testServer.Server.Listener.Addr().String())
globalMinioHost = host
globalMinioPort = port
globalMinioAddr = getLocalAddress(
s.testServer.SrvCmdCfg,
)
globalMinioAddr = getEndpointsLocalAddr(s.testServer.endpoints)
// initialize the peer client(s)
initGlobalS3Peers(s.testServer.Disks)

119
cmd/format-config-v1_test.go
View File

@ -273,12 +273,8 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
if err != nil {
t.Fatal(err)
}
endpoints, err := parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// Create an instance of xl backend.
obj, _, err := initObjectLayer(endpoints)
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Error(err)
}
@ -309,12 +305,8 @@ func TestFormatXLHealFreshDisksErrorExpected(t *testing.T) {
if err != nil {
t.Fatal(err)
}
endpoints, err := parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// Create an instance of xl backend.
obj, _, err := initObjectLayer(endpoints)
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Error(err)
}
@ -596,12 +588,8 @@ func TestInitFormatXLErrors(t *testing.T) {
t.Fatal(err)
}
defer removeRoots(fsDirs)
endpoints, err := parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// Create an instance of xl backend.
obj, _, err := initObjectLayer(endpoints)
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -702,12 +690,8 @@ func TestLoadFormatXLErrs(t *testing.T) {
}
defer removeRoots(fsDirs)
endpoints, err := parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// Create an instance of xl backend.
obj, _, err := initObjectLayer(endpoints)
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -733,11 +717,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
}
defer removeRoots(fsDirs)
endpoints, err = parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
obj, _, err = initObjectLayer(endpoints)
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -761,11 +741,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
}
defer removeRoots(fsDirs)
endpoints, err = parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
obj, _, err = initObjectLayer(endpoints)
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -787,11 +763,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
}
defer removeRoots(fsDirs)
endpoints, err = parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
obj, _, err = initObjectLayer(endpoints)
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -820,13 +792,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
t.Fatal(err)
}
endpoints, err := parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// Everything is fine, should return nil
obj, _, err := initObjectLayer(endpoints)
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -842,13 +809,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
t.Fatal(err)
}
endpoints, err = parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// Disks 0..15 are nil
obj, _, err = initObjectLayer(endpoints)
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -866,13 +828,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
t.Fatal(err)
}
endpoints, err = parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// One disk returns Faulty Disk
obj, _, err = initObjectLayer(endpoints)
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -892,13 +849,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
t.Fatal(err)
}
endpoints, err = parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// One disk is not found, heal corrupted disks should return nil
obj, _, err = initObjectLayer(endpoints)
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -914,13 +866,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
t.Fatal(err)
}
endpoints, err = parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// Remove format.json of all disks
obj, _, err = initObjectLayer(endpoints)
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -940,13 +887,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
t.Fatal(err)
}
endpoints, err = parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// Corrupted format json in one disk
obj, _, err = initObjectLayer(endpoints)
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -976,13 +918,8 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
t.Fatal(err)
}
endpoints, err := parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// Everything is fine, should return nil
obj, _, err := initObjectLayer(endpoints)
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -997,13 +934,8 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
t.Fatal(err)
}
endpoints, err = parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// Disks 0..15 are nil
obj, _, err = initObjectLayer(endpoints)
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -1021,13 +953,8 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
t.Fatal(err)
}
endpoints, err = parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// One disk returns Faulty Disk
obj, _, err = initObjectLayer(endpoints)
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -1047,13 +974,8 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
t.Fatal(err)
}
endpoints, err = parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// One disk is not found, heal corrupted disks should return nil
obj, _, err = initObjectLayer(endpoints)
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -1069,13 +991,8 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
t.Fatal(err)
}
endpoints, err = parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// Remove format.json of all disks
obj, _, err = initObjectLayer(endpoints)
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}

6
cmd/gateway-main.go
View File

@ -187,9 +187,6 @@ func gatewayMain(ctx *cli.Context) {
fatalIf(aerr, "Failed to start minio server")
}()
apiEndPoints, err := finalizeAPIEndpoints(apiServer.Addr)
fatalIf(err, "Unable to finalize API endpoints for %s", apiServer.Addr)
// Once endpoints are finalized, initialize the new object api.
globalObjLayerMutex.Lock()
globalObjectAPI = newObject
@ -202,7 +199,8 @@ func gatewayMain(ctx *cli.Context) {
mode = globalMinioModeGatewayAzure
}
checkUpdate(mode)
printGatewayStartupMessage(apiEndPoints, accessKey, secretKey, backendType)
apiEndpoints := getAPIEndpoints(apiServer.Addr)
printGatewayStartupMessage(apiEndpoints, accessKey, secretKey, backendType)
}
<-globalServiceDoneCh

7
cmd/globals.go
View File

@ -18,7 +18,6 @@ package cmd
import (
"crypto/x509"
"net/url"
"runtime"
"time"
@ -85,9 +84,6 @@ var (
// Holds the host that was passed using --address
globalMinioHost = ""
// Holds the list of API endpoints for a given server.
globalAPIEndpoints = []string{}
// Peer communication struct
globalS3Peers = s3Peers{}
@ -103,8 +99,7 @@ var (
// Minio server user agent string.
globalServerUserAgent = "Minio/" + ReleaseTag + " (" + runtime.GOOS + "; " + runtime.GOARCH + ")"
// url.URL endpoints of disks that belong to the object storage.
globalEndpoints = []*url.URL{}
globalEndpoints EndpointList
// Global server's network statistics
globalConnStats = newConnStats()

69
cmd/interface-ips.go
View File

@ -1,69 +0,0 @@
/*
* Minio Cloud Storage, (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import (
"fmt"
"net"
"sort"
)
// byLastOctetValue implements sort.Interface used in sorting a list
// of ip address by their last octet value.
type byLastOctetValue []net.IP
func (n byLastOctetValue) Len() int { return len(n) }
func (n byLastOctetValue) Swap(i, j int) { n[i], n[j] = n[j], n[i] }
func (n byLastOctetValue) Less(i, j int) bool {
return []byte(n[i].To4())[3] < []byte(n[j].To4())[3]
}
// getInterfaceIPv4s is synonymous to net.InterfaceAddrs()
// returns net.IP IPv4 only representation of the net.Addr.
// Additionally the returned list is sorted by their last
// octet value.
//
// [The logic to sort by last octet is implemented to
// prefer CIDRs with higher octects, this in-turn skips the
// localhost/loopback address to be not preferred as the
// first ip on the list. Subsequently this list helps us print
// a user friendly message with appropriate values].
func getInterfaceIPv4s() ([]net.IP, error) {
addrs, err := net.InterfaceAddrs()
if err != nil {
return nil, fmt.Errorf("Unable to determine network interface address. %s", err)
}
// Go through each return network address and collate IPv4 addresses.
var nips []net.IP
for _, addr := range addrs {
if addr.Network() == "ip+net" {
var nip net.IP
// Attempt to parse the addr through CIDR.
nip, _, err = net.ParseCIDR(addr.String())
if err != nil {
return nil, fmt.Errorf("Unable to parse addrss %s, error %s", addr, err)
}
// Collect only IPv4 addrs.
if nip.To4() != nil {
nips = append(nips, nip)
}
}
}
// Sort the list of IPs by their last octet value.
sort.Sort(sort.Reverse(byLastOctetValue(nips)))
return nips, nil
}

31
cmd/interface-ips_test.go
View File

@ -1,31 +0,0 @@
/*
* Minio Cloud Storage, (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import "testing"
func TestGetInterfaceIPv4s(t *testing.T) {
ipv4s, err := getInterfaceIPv4s()
if err != nil {
t.Fatalf("Unexpected error %s", err)
}
for _, ip := range ipv4s {
if ip.To4() == nil {
t.Fatalf("Unexpected expecting only IPv4 addresses only %s", ip)
}
}
}

19
cmd/lock-rpc-server.go
View File

@ -90,9 +90,9 @@ func startLockMaintainence(lockServers []*lockServer) {
}
// Register distributed NS lock handlers.
func registerDistNSLockRouter(mux *router.Router, serverConfig serverCmdConfig) error {
func registerDistNSLockRouter(mux *router.Router, endpoints EndpointList) error {
// Initialize a new set of lock servers.
lockServers := newLockServers(serverConfig)
lockServers := newLockServers(endpoints)
// Start lock maintenance from all lock servers.
startLockMaintainence(lockServers)
@ -102,19 +102,18 @@ func registerDistNSLockRouter(mux *router.Router, serverConfig serverCmdConfig)
}
// Create one lock server for every local storage rpc server.
func newLockServers(srvConfig serverCmdConfig) (lockServers []*lockServer) {
for _, ep := range srvConfig.endpoints {
func newLockServers(endpoints EndpointList) (lockServers []*lockServer) {
for _, endpoint := range endpoints {
// Initialize new lock server for each local node.
if isLocalStorage(ep) {
// Create handler for lock RPCs
locker := &lockServer{
serviceEndpoint: getPath(ep),
if endpoint.IsLocal {
lockServers = append(lockServers, &lockServer{
serviceEndpoint: endpoint.Path,
mutex: sync.Mutex{},
lockMap: make(map[string][]lockRequesterInfo),
}
lockServers = append(lockServers, locker)
})
}
}
return lockServers
}

77
cmd/lock-rpc-server_test.go
View File

@ -17,7 +17,6 @@
package cmd
import (
"net/url"
"runtime"
"sync"
"testing"
@ -433,66 +432,46 @@ func TestLockServers(t *testing.T) {
globalIsDistXL = currentIsDistXL
}()
case1Endpoints := mustGetNewEndpointList(
"http://localhost:9000/mnt/disk1",
"http://1.1.1.2:9000/mnt/disk2",
"http://1.1.2.1:9000/mnt/disk3",
"http://1.1.2.2:9000/mnt/disk4",
)
for i := range case1Endpoints {
if case1Endpoints[i].Host == "localhost:9000" {
case1Endpoints[i].IsLocal = true
}
}
case2Endpoints := mustGetNewEndpointList(
"http://localhost:9000/mnt/disk1",
"http://localhost:9000/mnt/disk2",
"http://1.1.2.1:9000/mnt/disk3",
"http://1.1.2.2:9000/mnt/disk4",
)
for i := range case2Endpoints {
if case2Endpoints[i].Host == "localhost:9000" {
case2Endpoints[i].IsLocal = true
}
}
globalMinioHost = ""
testCases := []struct {
isDistXL bool
srvCmdConfig serverCmdConfig
endpoints EndpointList
totalLockServers int
}{
// Test - 1 one lock server initialized.
{
isDistXL: true,
srvCmdConfig: serverCmdConfig{
endpoints: []*url.URL{{
Scheme: httpScheme,
Host: "localhost:9000",
Path: "/mnt/disk1",
}, {
Scheme: httpScheme,
Host: "1.1.1.2:9000",
Path: "/mnt/disk2",
}, {
Scheme: httpScheme,
Host: "1.1.2.1:9000",
Path: "/mnt/disk3",
}, {
Scheme: httpScheme,
Host: "1.1.2.2:9000",
Path: "/mnt/disk4",
}},
},
totalLockServers: 1,
},
{true, case1Endpoints, 1},
// Test - 2 two servers possible.
{
isDistXL: true,
srvCmdConfig: serverCmdConfig{
endpoints: []*url.URL{{
Scheme: httpScheme,
Host: "localhost:9000",
Path: "/mnt/disk1",
}, {
Scheme: httpScheme,
Host: "localhost:9000",
Path: "/mnt/disk2",
}, {
Scheme: httpScheme,
Host: "1.1.2.1:9000",
Path: "/mnt/disk3",
}, {
Scheme: httpScheme,
Host: "1.1.2.2:9000",
Path: "/mnt/disk4",
}},
},
totalLockServers: 2,
},
{true, case2Endpoints, 2},
}
// Validates lock server initialization.
for i, testCase := range testCases {
globalIsDistXL = testCase.isDistXL
lockServers := newLockServers(testCase.srvCmdConfig)
lockServers := newLockServers(testCase.endpoints)
if len(lockServers) != testCase.totalLockServers {
t.Fatalf("Test %d: Expected total %d, got %d", i+1, testCase.totalLockServers, len(lockServers))
}

11
cmd/namespace-lock.go
View File

@ -42,19 +42,16 @@ func initDsyncNodes() error {
// Initialize rpc lock client information only if this instance is a distributed setup.
clnts := make([]dsync.NetLocker, len(globalEndpoints))
myNode := -1
for index, ep := range globalEndpoints {
if ep == nil {
return errInvalidArgument
}
for index, endpoint := range globalEndpoints {
clnts[index] = newLockRPCClient(authConfig{
accessKey: cred.AccessKey,
secretKey: cred.SecretKey,
serverAddr: ep.Host,
serverAddr: endpoint.Host,
secureConn: globalIsSSL,
serviceEndpoint: pathutil.Join(minioReservedBucketPath, lockServicePath, getPath(ep)),
serviceEndpoint: pathutil.Join(minioReservedBucketPath, lockServicePath, endpoint.Path),
serviceName: lockServiceName,
})
if isLocalStorage(ep) && myNode == -1 {
if endpoint.IsLocal && myNode == -1 {
myNode = index
}
}

167
cmd/net.go Normal file
View File

@ -0,0 +1,167 @@
/*
* Minio Cloud Storage, (C) 2017 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import (
"fmt"
"net"
"os"
"sort"
"strconv"
"syscall"
"github.com/minio/minio-go/pkg/set"
)
// IPv4 addresses of local host.
var localIP4 = mustGetLocalIP4()
// mustSplitHostPort is a wrapper to net.SplitHostPort() where error is assumed to be a fatal.
func mustSplitHostPort(hostPort string) (host, port string) {
host, port, err := net.SplitHostPort(hostPort)
fatalIf(err, "Unable to split host port %s", hostPort)
return host, port
}
// mustGetLocalIP4 returns IPv4 addresses of local host. It panics on error.
func mustGetLocalIP4() (ipList set.StringSet) {
ipList = set.NewStringSet()
addrs, err := net.InterfaceAddrs()
fatalIf(err, "Unable to get IP addresses of this host.")
for _, addr := range addrs {
var ip net.IP
switch v := addr.(type) {
case *net.IPNet:
ip = v.IP
case *net.IPAddr:
ip = v.IP
}
if ip.To4() != nil {
ipList.Add(ip.String())
}
}
return ipList
}
// getHostIP4 returns IPv4 address of given host.
func getHostIP4(host string) (ipList set.StringSet, err error) {
ipList = set.NewStringSet()
ips, err := net.LookupIP(host)
if err != nil {
return ipList, err
}
for _, ip := range ips {
if ip.To4() != nil {
ipList.Add(ip.String())
}
}
return ipList, err
}
func getAPIEndpoints(serverAddr string) (apiEndpoints []string) {
host, port := mustSplitHostPort(serverAddr)
var ipList []string
if host == "" {
ipList = localIP4.ToSlice()
} else {
ipList = []string{host}
}
sort.Strings(ipList)
scheme := httpScheme
if globalIsSSL {
scheme = httpsScheme
}
for _, ip := range ipList {
apiEndpoints = append(apiEndpoints, fmt.Sprintf("%s://%s:%s", scheme, ip, port))
}
return apiEndpoints
}
// checkPortAvailability - check if given port is already in use.
// Note: The check method tries to listen on given port and closes it.
// It is possible to have a disconnected client in this tiny window of time.
func checkPortAvailability(port string) (err error) {
// Return true if err is "address already in use" error.
isAddrInUseErr := func(err error) (b bool) {
if opErr, ok := err.(*net.OpError); ok {
if sysErr, ok := opErr.Err.(*os.SyscallError); ok {
if errno, ok := sysErr.Err.(syscall.Errno); ok {
b = (errno == syscall.EADDRINUSE)
}
}
}
return b
}
network := []string{"tcp", "tcp4", "tcp6"}
for _, n := range network {
l, err := net.Listen(n, net.JoinHostPort("", port))
if err == nil {
// As we are able to listen on this network, the port is not in use.
// Close the listener and continue check other networks.
if err = l.Close(); err != nil {
return err
}
} else if isAddrInUseErr(err) {
// As we got EADDRINUSE error, the port is in use by other process.
// Return the error.
return err
}
}
return nil
}
// CheckLocalServerAddr - checks if serverAddr is valid and local host.
func CheckLocalServerAddr(serverAddr string) error {
host, port, err := net.SplitHostPort(serverAddr)
if err != nil {
return err
}
// Check whether port is a valid port number.
p, err := strconv.Atoi(port)
if err != nil {
return fmt.Errorf("invalid port number")
} else if p < 1 || p > 65535 {
return fmt.Errorf("port number must be between 1 to 65535")
}
if host != "" {
hostIPs, err := getHostIP4(host)
if err != nil {
return err
}
if localIP4.Intersection(hostIPs).IsEmpty() {
return fmt.Errorf("host in server address should be this server")
}
}
return nil
}

183
cmd/net_test.go Normal file
View File

@ -0,0 +1,183 @@
/*
* Minio Cloud Storage, (C) 2017 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import (
"fmt"
"net"
"runtime"
"testing"
"github.com/minio/minio-go/pkg/set"
)
func TestMustSplitHostPort(t *testing.T) {
testCases := []struct {
hostPort string
expectedHost string
expectedPort string
}{
{":54321", "", "54321"},
{"server:54321", "server", "54321"},
{":", "", ""},
{":0", "", "0"},
{":-10", "", "-10"},
{"server:100000000", "server", "100000000"},
{"server:https", "server", "https"},
}
for _, testCase := range testCases {
host, port := mustSplitHostPort(testCase.hostPort)
if testCase.expectedHost != host {
t.Fatalf("host: expected = %v, got = %v", testCase.expectedHost, host)
}
if testCase.expectedPort != port {
t.Fatalf("port: expected = %v, got = %v", testCase.expectedPort, port)
}
}
}
func TestMustGetLocalIP4(t *testing.T) {
testCases := []struct {
expectedIPList set.StringSet
}{
{set.CreateStringSet("127.0.0.1")},
}
for _, testCase := range testCases {
ipList := mustGetLocalIP4()
if testCase.expectedIPList != nil && testCase.expectedIPList.Intersection(ipList).IsEmpty() {
t.Fatalf("host: expected = %v, got = %v", testCase.expectedIPList, ipList)
}
}
}
func TestGetHostIP(t *testing.T) {
_, err := getHostIP4("myserver")
testCases := []struct {
host string
expectedIPList set.StringSet
expectedErr error
}{
{"localhost", set.CreateStringSet("127.0.0.1"), nil},
{"example.org", set.CreateStringSet("93.184.216.34"), nil},
{"myserver", nil, err},
}
for _, testCase := range testCases {
ipList, err := getHostIP4(testCase.host)
if testCase.expectedErr == nil {
if err != nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
}
} else if err == nil {
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
} else if testCase.expectedErr.Error() != err.Error() {
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
}
if testCase.expectedIPList != nil && testCase.expectedIPList.Intersection(ipList).IsEmpty() {
t.Fatalf("host: expected = %v, got = %v", testCase.expectedIPList, ipList)
}
}
}
// Tests finalize api endpoints.
func TestGetAPIEndpoints(t *testing.T) {
testCases := []struct {
serverAddr string
expectedResult string
}{
{":80", "http://127.0.0.1:80"},
{"127.0.0.1:80", "http://127.0.0.1:80"},
{"localhost:80", "http://localhost:80"},
}
for i, testCase := range testCases {
apiEndpoints := getAPIEndpoints(testCase.serverAddr)
apiEndpointSet := set.CreateStringSet(apiEndpoints...)
if !apiEndpointSet.Contains(testCase.expectedResult) {
t.Fatalf("test %d: expected: Found, got: Not Found", i+1)
}
}
}
// Tests for port availability logic written for server startup sequence.
func TestCheckPortAvailability(t *testing.T) {
// Make a port is not available.
port := getFreePort()
listener, err := net.Listen("tcp", net.JoinHostPort("", port))
if err != nil {
t.Fatalf("Unable to listen on port %v", port)
}
defer listener.Close()
testCases := []struct {
port string
expectedErr error
}{
{port, fmt.Errorf("listen tcp :%v: bind: address already in use", port)},
{getFreePort(), nil},
}
for _, testCase := range testCases {
// On MS Windows, skip checking error case due to https://github.com/golang/go/issues/7598
if runtime.GOOS == globalWindowsOSName && testCase.expectedErr != nil {
continue
}
err := checkPortAvailability(testCase.port)
if testCase.expectedErr == nil {
if err != nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
}
} else if err == nil {
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
} else if testCase.expectedErr.Error() != err.Error() {
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
}
}
}
func TestCheckLocalServerAddr(t *testing.T) {
testCases := []struct {
serverAddr string
expectedErr error
}{
{":54321", nil},
{"localhost:54321", nil},
{"", fmt.Errorf("missing port in address")},
{"localhost", fmt.Errorf("missing port in address localhost")},
{"example.org:54321", fmt.Errorf("host in server address should be this server")},
{":0", fmt.Errorf("port number must be between 1 to 65535")},
{":-10", fmt.Errorf("port number must be between 1 to 65535")},
}
for _, testCase := range testCases {
err := CheckLocalServerAddr(testCase.serverAddr)
if testCase.expectedErr == nil {
if err != nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
}
} else if err == nil {
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
} else if testCase.expectedErr.Error() != err.Error() {
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
}
}
}

90
cmd/object-api-common.go
View File

@ -17,9 +17,6 @@
package cmd
import (
"net"
"net/url"
"runtime"
"sync"
humanize "github.com/dustin/go-humanize"
@ -132,90 +129,13 @@ func houseKeeping(storageDisks []StorageAPI) error {
return nil
}
// Check if a network path is local to this node.
func isLocalStorage(ep *url.URL) bool {
if ep.Host == "" {
return true
}
if globalMinioHost != "" && globalMinioPort != "" {
// if --address host:port was specified for distXL we short
// circuit only the endPoint that matches host:port
return net.JoinHostPort(globalMinioHost, globalMinioPort) == ep.Host
}
// Split host to extract host information.
host, _, err := net.SplitHostPort(ep.Host)
if err != nil {
errorIf(err, "Cannot split host port")
return false
}
// Resolve host to address to check if the IP is loopback.
// If address resolution fails, assume it's a non-local host.
addrs, err := net.LookupHost(host)
if err != nil {
errorIf(err, "Failed to lookup host")
return false
}
for _, addr := range addrs {
if ip := net.ParseIP(addr); ip.IsLoopback() {
return true
}
}
iaddrs, err := net.InterfaceAddrs()
if err != nil {
errorIf(err, "Unable to list interface addresses")
return false
}
for _, addr := range addrs {
for _, iaddr := range iaddrs {
ip, _, err := net.ParseCIDR(iaddr.String())
if err != nil {
errorIf(err, "Unable to parse CIDR")
return false
}
if ip.String() == addr {
return true
}
}
}
return false
}
// Fetch the path component from *url.URL*.
func getPath(ep *url.URL) string {
if ep == nil {
return ""
}
var diskPath string
// For windows ep.Path is usually empty
if runtime.GOOS == globalWindowsOSName {
switch ep.Scheme {
case "":
// Eg. "minio server .\export"
diskPath = ep.Path
case httpScheme, httpsScheme:
// For full URLs windows drive is part of URL path.
// Eg: http://ip:port/C:\mydrive
// For windows trim off the preceding "/".
diskPath = ep.Path[1:]
default:
// For the rest url splits drive letter into
// Scheme contruct the disk path back.
diskPath = ep.Scheme + ":" + ep.Opaque
}
} else {
// For other operating systems ep.Path is non empty.
diskPath = ep.Path
}
return diskPath
}
// Depending on the disk type network or local, initialize storage API.
func newStorageAPI(ep *url.URL) (storage StorageAPI, err error) {
if isLocalStorage(ep) {
return newPosix(getPath(ep))
func newStorageAPI(endpoint Endpoint) (storage StorageAPI, err error) {
if endpoint.IsLocal {
return newPosix(endpoint.Path)
}
return newStorageRPC(ep)
return newStorageRPC(endpoint), nil
}
var initMetaVolIgnoredErrs = append(baseIgnoredErrs, errVolumeExists)

51
cmd/object-api-common_test.go
View File

@ -17,7 +17,6 @@
package cmd
import (
"runtime"
"sync"
"testing"
)
@ -98,53 +97,3 @@ func TestHouseKeeping(t *testing.T) {
}
}
}
// Test getPath() - the path that needs to be passed to newPosix()
func TestGetPath(t *testing.T) {
globalMinioHost = ""
var testCases []struct {
epStr string
path string
}
if runtime.GOOS == globalWindowsOSName {
testCases = []struct {
epStr string
path string
}{
{"\\export", "\\export"},
{"D:\\export", "d:\\export"},
{"D:\\", "d:\\"},
{"D:", "d:"},
{"\\", "\\"},
{"http://127.0.0.1/d:/export", "d:/export"},
{"https://127.0.0.1/d:/export", "d:/export"},
}
} else {
testCases = []struct {
epStr string
path string
}{
{"/export", "/export"},
{"http://127.0.0.1/export", "/export"},
{"https://127.0.0.1/export", "/export"},
}
}
testCasesCommon := []struct {
epStr string
path string
}{
{"export", "export"},
}
testCases = append(testCases, testCasesCommon...)
for i, test := range testCases {
eps, err := parseStorageEndpoints([]string{test.epStr})
if err != nil {
t.Errorf("Test %d: %s - %s", i+1, test.epStr, err)
continue
}
path := getPath(eps[0])
if path != test.path {
t.Errorf("Test %d: For endpoing %s, getPath() failed, got: %s, expected: %s,", i+1, test.epStr, path, test.path)
}
}
}

46
cmd/prepare-storage-msg.go
View File

@ -18,8 +18,6 @@ package cmd
import (
"fmt"
"net"
"net/url"
"sync"
humanize "github.com/dustin/go-humanize"
@ -52,45 +50,11 @@ func printOnceFn() printOnceFunc {
}
// Prints custom message when healing is required for XL and Distributed XL backend.
func printHealMsg(endpoints []*url.URL, storageDisks []StorageAPI, fn printOnceFunc) {
func printHealMsg(endpoints EndpointList, storageDisks []StorageAPI, fn printOnceFunc) {
msg := getHealMsg(endpoints, storageDisks)
fn(msg)
}
// Heal endpoint constructs the final endpoint URL for control heal command.
// Disk heal endpoint needs to be just a URL and no special paths.
// This function constructs the right endpoint under various conditions
// for single node XL, distributed XL and when minio server is bound
// to a specific ip:port.
func getHealEndpoint(tls bool, firstEndpoint *url.URL) (cEndpoint *url.URL) {
scheme := httpScheme
if tls {
scheme = httpsScheme
}
cEndpoint = &url.URL{
Scheme: scheme,
}
// Bind to `--address host:port` was specified.
if globalMinioHost != "" {
cEndpoint.Host = net.JoinHostPort(globalMinioHost, globalMinioPort)
return cEndpoint
}
// For distributed XL setup.
if firstEndpoint.Host != "" {
cEndpoint.Host = firstEndpoint.Host
return cEndpoint
}
// For single node XL setup, we need to find the endpoint.
cEndpoint.Host = globalMinioAddr
// Fetch all the listening ips. For single node XL we
// just use the first host.
hosts, _, err := getListenIPs(cEndpoint.Host)
if err == nil {
cEndpoint.Host = net.JoinHostPort(hosts[0], globalMinioPort)
}
return cEndpoint
}
// Disks offline and online strings..
const (
diskOffline = "offline"
@ -100,7 +64,7 @@ const (
// Constructs a formatted heal message, when cluster is found to be in state where it requires healing.
// healing is optional, server continues to initialize object layer after printing this message.
// it is upto the end user to perform a heal if needed.
func getHealMsg(endpoints []*url.URL, storageDisks []StorageAPI) string {
func getHealMsg(endpoints EndpointList, storageDisks []StorageAPI) string {
healFmtCmd := `"mc admin heal myminio"`
msg := fmt.Sprintf("New disk(s) were found, format them by running - %s\n",
healFmtCmd)
@ -126,13 +90,13 @@ func getHealMsg(endpoints []*url.URL, storageDisks []StorageAPI) string {
}
// Prints regular message when we have sufficient disks to start the cluster.
func printRegularMsg(endpoints []*url.URL, storageDisks []StorageAPI, fn printOnceFunc) {
func printRegularMsg(endpoints EndpointList, storageDisks []StorageAPI, fn printOnceFunc) {
msg := getStorageInitMsg("\nInitializing data volume.", endpoints, storageDisks)
fn(msg)
}
// Constructs a formatted regular message when we have sufficient disks to start the cluster.
func getStorageInitMsg(titleMsg string, endpoints []*url.URL, storageDisks []StorageAPI) string {
func getStorageInitMsg(titleMsg string, endpoints EndpointList, storageDisks []StorageAPI) string {
msg := colorBlue(titleMsg)
disksInfo, _, _ := getDisksInfo(storageDisks)
for i, info := range disksInfo {
@ -156,7 +120,7 @@ func getStorageInitMsg(titleMsg string, endpoints []*url.URL, storageDisks []Sto
}
// Prints initialization message when cluster is being initialized for the first time.
func printFormatMsg(endpoints []*url.URL, storageDisks []StorageAPI, fn printOnceFunc) {
func printFormatMsg(endpoints EndpointList, storageDisks []StorageAPI, fn printOnceFunc) {
msg := getStorageInitMsg("\nInitializing data volume for the first time.", endpoints, storageDisks)
fn(msg)
}

85
cmd/prepare-storage-msg_test.go
View File

@ -17,60 +17,10 @@
package cmd
import (
"net/url"
"reflect"
"fmt"
"testing"
)
// Tests and validates the output for heal endpoint.
func TestGetHealEndpoint(t *testing.T) {
// Test for a SSL scheme.
tls := true
hURL := getHealEndpoint(tls, &url.URL{
Scheme: httpScheme,
Host: "localhost:9000",
})
sHURL := &url.URL{
Scheme: httpsScheme,
Host: "localhost:9000",
}
if !reflect.DeepEqual(hURL, sHURL) {
t.Fatalf("Expected %#v, but got %#v", sHURL, hURL)
}
// Test a non-TLS scheme.
tls = false
hURL = getHealEndpoint(tls, &url.URL{
Scheme: httpsScheme,
Host: "localhost:9000",
})
sHURL = &url.URL{
Scheme: httpScheme,
Host: "localhost:9000",
}
if !reflect.DeepEqual(hURL, sHURL) {
t.Fatalf("Expected %#v, but got %#v", sHURL, hURL)
}
// FIXME(GLOBAL): purposefully Host is left empty because
// we need to bring in safe handling on global values
// add a proper test case here once that happens.
/*
tls = false
hURL = getHealEndpoint(tls, &url.URL{
Path: "/export",
})
sHURL = &url.URL{
Scheme: httpScheme,
Host: "",
}
globalMinioAddr = ""
if !reflect.DeepEqual(hURL, sHURL) {
t.Fatalf("Expected %#v, but got %#v", sHURL, hURL)
}
*/
}
// Tests heal message to be correct and properly formatted.
func TestHealMsg(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
@ -85,39 +35,26 @@ func TestHealMsg(t *testing.T) {
nilDisks[5] = nil
authErrs := make([]error, len(storageDisks))
authErrs[5] = errAuthentication
endpointURL, err := url.Parse("http://10.1.10.1:9000")
if err != nil {
t.Fatal("Unexpected error:", err)
}
endpointURLs := make([]*url.URL, len(storageDisks))
for idx := 0; idx < len(endpointURLs); idx++ {
endpointURLs[idx] = endpointURL
args := []string{}
for i := range storageDisks {
args = append(args, fmt.Sprintf("http://10.1.10.%d:9000/d1", i+1))
}
endpoints := mustGetNewEndpointList(args...)
testCases := []struct {
endPoints []*url.URL
endPoints EndpointList
storageDisks []StorageAPI
serrs []error
}{
// Test - 1 for valid disks and errors.
{
endPoints: endpointURLs,
storageDisks: storageDisks,
serrs: errs,
},
{endpoints, storageDisks, errs},
// Test - 2 for one of the disks is nil.
{
endPoints: endpointURLs,
storageDisks: nilDisks,
serrs: errs,
},
{endpoints, nilDisks, errs},
// Test - 3 for one of the errs is authentication.
{
endPoints: endpointURLs,
storageDisks: nilDisks,
serrs: authErrs,
},
{endpoints, nilDisks, authErrs},
}
for i, testCase := range testCases {
msg := getHealMsg(testCase.endPoints, testCase.storageDisks)
if msg == "" {

18
cmd/prepare-storage.go
View File

@ -18,7 +18,6 @@ package cmd
import (
"errors"
"net/url"
"time"
"github.com/minio/mc/pkg/console"
@ -191,7 +190,7 @@ func printRetryMsg(sErrs []error, storageDisks []StorageAPI) {
// Implements a jitter backoff loop for formatting all disks during
// initialization of the server.
func retryFormattingXLDisks(firstDisk bool, endpoints []*url.URL, storageDisks []StorageAPI) error {
func retryFormattingXLDisks(firstDisk bool, endpoints EndpointList, storageDisks []StorageAPI) error {
if len(endpoints) == 0 {
return errInvalidArgument
}
@ -276,16 +275,13 @@ func retryFormattingXLDisks(firstDisk bool, endpoints []*url.URL, storageDisks [
}
// Initialize storage disks based on input arguments.
func initStorageDisks(endpoints []*url.URL) ([]StorageAPI, error) {
func initStorageDisks(endpoints EndpointList) ([]StorageAPI, error) {
// Bootstrap disks.
storageDisks := make([]StorageAPI, len(endpoints))
for index, ep := range endpoints {
if ep == nil {
return nil, errInvalidArgument
}
for index, endpoint := range endpoints {
// Intentionally ignore disk not found errors. XL is designed
// to handle these errors internally.
storage, err := newStorageAPI(ep)
storage, err := newStorageAPI(endpoint)
if err != nil && err != errDiskNotFound {
return nil, err
}
@ -295,14 +291,10 @@ func initStorageDisks(endpoints []*url.URL) ([]StorageAPI, error) {
}
// Format disks before initialization of object layer.
func waitForFormatXLDisks(firstDisk bool, endpoints []*url.URL, storageDisks []StorageAPI) (formattedDisks []StorageAPI, err error) {
func waitForFormatXLDisks(firstDisk bool, endpoints EndpointList, storageDisks []StorageAPI) (formattedDisks []StorageAPI, err error) {
if len(endpoints) == 0 {
return nil, errInvalidArgument
}
firstEndpoint := endpoints[0]
if firstEndpoint == nil {
return nil, errInvalidArgument
}
if storageDisks == nil {
return nil, errInvalidArgument
}

10
cmd/routers.go
View File

@ -30,15 +30,15 @@ func newObjectLayerFn() (layer ObjectLayer) {
}
// Composed function registering routers for only distributed XL setup.
func registerDistXLRouters(mux *router.Router, srvCmdConfig serverCmdConfig) error {
func registerDistXLRouters(mux *router.Router, endpoints EndpointList) error {
// Register storage rpc router only if its a distributed setup.
err := registerStorageRPCRouters(mux, srvCmdConfig)
err := registerStorageRPCRouters(mux, endpoints)
if err != nil {
return err
}
// Register distributed namespace lock.
err = registerDistNSLockRouter(mux, srvCmdConfig)
err = registerDistNSLockRouter(mux, endpoints)
if err != nil {
return err
}
@ -54,14 +54,14 @@ func registerDistXLRouters(mux *router.Router, srvCmdConfig serverCmdConfig) err
}
// configureServer handler returns final handler for the http server.
func configureServerHandler(srvCmdConfig serverCmdConfig) (http.Handler, error) {
func configureServerHandler(endpoints EndpointList) (http.Handler, error) {
// Initialize router. `SkipClean(true)` stops gorilla/mux from
// normalizing URL path minio/minio#3256
mux := router.NewRouter().SkipClean(true)
// Initialize distributed NS lock.
if globalIsDistXL {
registerDistXLRouters(mux, srvCmdConfig)
registerDistXLRouters(mux, endpoints)
}
// Add Admin RPC router

61
cmd/s3-peer-client.go
View File

@ -19,9 +19,11 @@ package cmd
import (
"encoding/json"
"fmt"
"net/url"
"net"
"path"
"sync"
"github.com/minio/minio-go/pkg/set"
)
// s3Peer structs contains the address of a peer in the cluster, and
@ -39,53 +41,44 @@ type s3Peers []s3Peer
// makeS3Peers makes an s3Peers struct value from the given urls
// slice. The urls slice is assumed to be non-empty and free of nil
// values.
func makeS3Peers(eps []*url.URL) s3Peers {
var ret []s3Peer
// map to store peers that are already added to ret
seenAddr := make(map[string]bool)
// add local (self) as peer in the array
ret = append(ret, s3Peer{
globalMinioAddr,
func makeS3Peers(endpoints EndpointList) (s3PeerList s3Peers) {
thisPeer := globalMinioAddr
if globalMinioHost == "" {
thisPeer = net.JoinHostPort("localhost", globalMinioPort)
}
s3PeerList = append(s3PeerList, s3Peer{
thisPeer,
&localBucketMetaState{ObjectAPI: newObjectLayerFn},
})
seenAddr[globalMinioAddr] = true
serverCred := serverConfig.GetCredential()
// iterate over endpoints to find new remote peers and add
// them to ret.
for _, ep := range eps {
if ep.Host == "" {
hostSet := set.CreateStringSet(globalMinioAddr)
cred := serverConfig.GetCredential()
serviceEndpoint := path.Join(minioReservedBucketPath, s3Path)
for _, host := range GetRemotePeers(endpoints) {
if hostSet.Contains(host) {
continue
}
// Check if the remote host has been added already
if !seenAddr[ep.Host] {
cfg := authConfig{
accessKey: serverCred.AccessKey,
secretKey: serverCred.SecretKey,
serverAddr: ep.Host,
serviceEndpoint: path.Join(minioReservedBucketPath, s3Path),
hostSet.Add(host)
s3PeerList = append(s3PeerList, s3Peer{
addr: host,
bmsClient: &remoteBucketMetaState{newAuthRPCClient(authConfig{
accessKey: cred.AccessKey,
secretKey: cred.SecretKey,
serverAddr: host,
serviceEndpoint: serviceEndpoint,
secureConn: globalIsSSL,
serviceName: "S3",
}
ret = append(ret, s3Peer{
addr: ep.Host,
bmsClient: &remoteBucketMetaState{newAuthRPCClient(cfg)},
})},
})
seenAddr[ep.Host] = true
}
}
return ret
return s3PeerList
}
// initGlobalS3Peers - initialize globalS3Peers by passing in
// endpoints - intended to be called early in program start-up.
func initGlobalS3Peers(eps []*url.URL) {
globalS3Peers = makeS3Peers(eps)
func initGlobalS3Peers(endpoints EndpointList) {
globalS3Peers = makeS3Peers(endpoints)
}
// GetPeerClient - fetch BucketMetaState interface by peer address

9
cmd/s3-peer-client_test.go
View File

@ -17,7 +17,6 @@
package cmd
import (
"net/url"
"reflect"
"testing"
)
@ -35,12 +34,12 @@ func TestMakeS3Peers(t *testing.T) {
// test cases
testCases := []struct {
gMinioAddr string
eps []*url.URL
eps EndpointList
peers []string
}{
{":9000", []*url.URL{{Path: "/mnt/disk1"}}, []string{":9000"}},
{":9000", []*url.URL{{Host: "localhost:9001"}}, []string{":9000", "localhost:9001"}},
{"m1:9000", []*url.URL{{Host: "m1:9000"}, {Host: "m2:9000"}, {Host: "m3:9000"}}, []string{"m1:9000", "m2:9000", "m3:9000"}},
{"127.0.0.1:9000", mustGetNewEndpointList("/mnt/disk1"), []string{"127.0.0.1:9000"}},
{"127.0.0.1:9000", mustGetNewEndpointList("http://localhost:9001/d1"), []string{"127.0.0.1:9000", "localhost:9001"}},
{"example.org:9000", mustGetNewEndpointList("http://example.org:9000/d1", "http://example.com:9000/d1", "http://example.net:9000/d1", "http://example.edu:9000/d1"), []string{"example.org:9000", "example.com:9000", "example.edu:9000", "example.net:9000"}},
}
getPeersHelper := func(s3p s3Peers) []string {

359
cmd/server-main.go
View File

@ -18,19 +18,12 @@ package cmd
import (
"errors"
"fmt"
"net"
"net/url"
"os"
"path"
"path/filepath"
"sort"
"strconv"
"runtime"
"strings"
"time"
"runtime"
"github.com/minio/cli"
)
@ -110,249 +103,6 @@ func enableLoggers() {
log.SetConsoleTarget(consoleLogTarget)
}
type serverCmdConfig struct {
serverAddr string
endpoints []*url.URL
}
// Parse an array of end-points (from the command line)
func parseStorageEndpoints(eps []string) (endpoints []*url.URL, err error) {
for _, ep := range eps {
if ep == "" {
return nil, errInvalidArgument
}
var u *url.URL
u, err = url.Parse(ep)
if err != nil {
return nil, err
}
if u.Host != "" {
_, port, err := net.SplitHostPort(u.Host)
// Ignore the missing port error as the default port can be globalMinioPort.
if err != nil && !strings.Contains(err.Error(), "missing port in address") {
return nil, err
}
if globalMinioHost == "" {
// For ex.: minio server host1:port1 host2:port2...
// we return error as port is configurable only
// using "--address :port"
if port != "" {
return nil, fmt.Errorf("Invalid Argument %s, port configurable using --address :<port>", u.Host)
}
u.Host = net.JoinHostPort(u.Host, globalMinioPort)
} else {
// For ex.: minio server --address host:port host1:port1 host2:port2...
// i.e if "--address host:port" is specified
// port info in u.Host is mandatory else return error.
if port == "" {
return nil, fmt.Errorf("Invalid Argument %s, port mandatory when --address <host>:<port> is used", u.Host)
}
}
}
endpoints = append(endpoints, u)
}
return endpoints, nil
}
// Validate if input disks are sufficient for initializing XL.
func checkSufficientDisks(eps []*url.URL) error {
// Verify total number of disks.
total := len(eps)
if total > maxErasureBlocks {
return errXLMaxDisks
}
if total < minErasureBlocks {
return errXLMinDisks
}
// isEven function to verify if a given number if even.
isEven := func(number int) bool {
return number%2 == 0
}
// Verify if we have even number of disks.
// only combination of 4, 6, 8, 10, 12, 14, 16 are supported.
if !isEven(total) {
return errXLNumDisks
}
// Success.
return nil
}
// Returns if slice of disks is a distributed setup.
func isDistributedSetup(eps []*url.URL) bool {
// Validate if one the disks is not local.
for _, ep := range eps {
if !isLocalStorage(ep) {
// One or more disks supplied as arguments are
// not attached to the local node.
return true
}
}
return false
}
// Returns true if path is empty, or equals to '.', '/', '\' characters.
func isPathSentinel(path string) bool {
return path == "" || path == "." || path == "/" || path == `\`
}
// Returned when path is empty or root path.
var errEmptyRootPath = errors.New("Empty or root path is not allowed")
// Invalid scheme passed.
var errInvalidScheme = errors.New("Invalid scheme")
// Check if endpoint is in expected syntax by valid scheme/path across all platforms.
func checkEndpointURL(endpointURL *url.URL) (err error) {
// Applicable to all OS.
if endpointURL.Scheme == "" || endpointURL.Scheme == httpScheme || endpointURL.Scheme == httpsScheme {
if isPathSentinel(path.Clean(endpointURL.Path)) {
err = errEmptyRootPath
}
return err
}
// Applicable to Windows only.
if runtime.GOOS == globalWindowsOSName {
// On Windows, endpoint can be a path with drive eg. C:\Export and its URL.Scheme is 'C'.
// Check if URL.Scheme is a single letter alphabet to represent a drive.
// Note: URL.Parse() converts scheme into lower case always.
if len(endpointURL.Scheme) == 1 && endpointURL.Scheme[0] >= 'a' && endpointURL.Scheme[0] <= 'z' {
// If endpoint is C:\ or C:\export, URL.Path does not have path information like \ or \export
// hence we directly work with endpoint.
if isPathSentinel(strings.SplitN(path.Clean(endpointURL.String()), ":", 2)[1]) {
err = errEmptyRootPath
}
return err
}
}
return errInvalidScheme
}
// Check if endpoints are in expected syntax by valid scheme/path across all platforms.
func checkEndpointsSyntax(eps []*url.URL, disks []string) error {
for i, u := range eps {
if err := checkEndpointURL(u); err != nil {
return fmt.Errorf("%s: %s (%s)", err.Error(), u.Path, disks[i])
}
}
return nil
}
// Make sure all the command line parameters are OK and exit in case of invalid parameters.
func checkServerSyntax(endpoints []*url.URL, disks []string) {
// Validate if endpoints follow the expected syntax.
err := checkEndpointsSyntax(endpoints, disks)
fatalIf(err, "Invalid endpoints found %s", strings.Join(disks, " "))
// Validate for duplicate endpoints are supplied.
err = checkDuplicateEndpoints(endpoints)
fatalIf(err, "Duplicate entries in %s", strings.Join(disks, " "))
if len(endpoints) > 1 {
// Validate if we have sufficient disks for XL setup.
err = checkSufficientDisks(endpoints)
fatalIf(err, "Insufficient number of disks.")
} else {
// Validate if we have invalid disk for FS setup.
if endpoints[0].Host != "" && endpoints[0].Scheme != "" {
fatalIf(errInvalidArgument, "%s, FS setup expects a filesystem path", endpoints[0])
}
}
if !isDistributedSetup(endpoints) {
// for FS and singlenode-XL validation is done, return.
return
}
// Rest of the checks applies only to distributed XL setup.
if globalMinioHost != "" {
// We are here implies --address host:port is passed, hence the user is trying
// to run one minio process per export disk.
if globalMinioPort == "" {
fatalIf(errInvalidArgument, "Port missing, Host:Port should be specified for --address")
}
foundCnt := 0
for _, ep := range endpoints {
if ep.Host == globalMinioAddr {
foundCnt++
}
}
if foundCnt == 0 {
// --address host:port should be available in the XL disk list.
fatalIf(errInvalidArgument, "%s is not available in %s", globalMinioAddr, strings.Join(disks, " "))
}
if foundCnt > 1 {
// --address host:port should match exactly one entry in the XL disk list.
fatalIf(errInvalidArgument, "%s matches % entries in %s", globalMinioAddr, foundCnt, strings.Join(disks, " "))
}
}
for _, ep := range endpoints {
if ep.Scheme == httpsScheme && !globalIsSSL {
// Certificates should be provided for https configuration.
fatalIf(errInvalidArgument, "Certificates not provided for secure configuration")
}
}
}
// Checks if any of the endpoints supplied is local to this server.
func isAnyEndpointLocal(eps []*url.URL) bool {
anyLocalEp := false
for _, ep := range eps {
if isLocalStorage(ep) {
anyLocalEp = true
break
}
}
return anyLocalEp
}
// Returned when there are no ports.
var errEmptyPort = errors.New("Port cannot be empty or '0', please use `--address` to pick a specific port")
// Convert an input address of form host:port into, host and port, returns if any.
func getHostPort(address string) (host, port string, err error) {
// Check if requested port is available.
host, port, err = net.SplitHostPort(address)
if err != nil {
return "", "", err
}
// Empty ports.
if port == "0" || port == "" {
// Port zero or empty means use requested to choose any freely available
// port. Avoid this since it won't work with any configured clients,
// can lead to serious loss of availability.
return "", "", errEmptyPort
}
// Parse port.
if _, err = strconv.Atoi(port); err != nil {
return "", "", err
}
if runtime.GOOS == "darwin" {
// On macOS, if a process already listens on 127.0.0.1:PORT, net.Listen() falls back
// to IPv6 address ie minio will start listening on IPv6 address whereas another
// (non-)minio process is listening on IPv4 of given port.
// To avoid this error sutiation we check for port availability only for macOS.
if err = checkPortAvailability(port); err != nil {
return "", "", err
}
}
// Success.
return host, port, nil
}
func initConfig() {
// Config file does not exist, we create it fresh and return upon success.
if isFile(getConfigFile()) {
@ -365,6 +115,8 @@ func initConfig() {
}
func serverHandleCmdArgs(ctx *cli.Context) {
// Set configuration directory.
{
// Get configuration directory from command line argument.
configDir := ctx.String("config-dir")
if !ctx.IsSet("config-dir") && ctx.GlobalIsSet("config-dir") {
@ -375,52 +127,34 @@ func serverHandleCmdArgs(ctx *cli.Context) {
}
// Disallow relative paths, figure out absolute paths.
{
configDirAbs, err := filepath.Abs(configDir)
fatalIf(err, "Unable to fetch absolute path for config directory %s", configDir)
configDir = configDirAbs
setConfigDir(configDirAbs)
}
// Set configuration directory.
setConfigDir(configDir)
// Server address.
globalMinioAddr = ctx.String("address")
serverAddr := ctx.String("address")
fatalIf(CheckLocalServerAddr(serverAddr), "Invalid address ‘%s’ in command line argument.", serverAddr)
var setupType SetupType
var err error
globalMinioHost, globalMinioPort, err = getHostPort(globalMinioAddr)
fatalIf(err, "Unable to extract host and port %s", globalMinioAddr)
// Disks to be used in server init.
endpoints, err := parseStorageEndpoints(ctx.Args())
fatalIf(err, "Unable to parse storage endpoints %s", ctx.Args())
// Sort endpoints for consistent ordering across multiple
// nodes in a distributed setup. This is to avoid format.json
// corruption if the disks aren't supplied in the same order
// on all nodes.
sort.Sort(byHostPath(endpoints))
checkServerSyntax(endpoints, ctx.Args())
// Should exit gracefully if none of the endpoints passed
// as command line args are local to this server.
if !isAnyEndpointLocal(endpoints) {
fatalIf(errInvalidArgument, "None of the disks passed as command line args are local to this server.")
globalMinioAddr, globalEndpoints, setupType, err = CreateEndpoints(serverAddr, ctx.Args()...)
fatalIf(err, "Invalid command line arguments server=‘%s’, args=%s", serverAddr, ctx.Args())
globalMinioHost, globalMinioPort = mustSplitHostPort(globalMinioAddr)
if runtime.GOOS == "darwin" {
// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
// to IPv6 address ie minio will start listening on IPv6 address whereas another
// (non-)minio process is listening on IPv4 of given port.
// To avoid this error sutiation we check for port availability only for macOS.
fatalIf(checkPortAvailability(globalMinioPort), "Port %d already in use", globalMinioPort)
}
// Check if endpoints are part of distributed setup.
globalIsDistXL = isDistributedSetup(endpoints)
// Set globalIsXL if erasure code backend is about to be
// initialized for the given endpoints.
if len(endpoints) > 1 {
globalIsXL = (setupType == XLSetupType)
globalIsDistXL = (setupType == DistXLSetupType)
if globalIsDistXL {
globalIsXL = true
}
// Set endpoints of []*url.URL type to globalEndpoints.
globalEndpoints = endpoints
}
func serverHandleEnvVars() {
@ -497,11 +231,10 @@ func serverMain(ctx *cli.Context) {
if !quietFlag {
// Check for new updates from dl.minio.io.
mode := globalMinioModeFS
if globalIsXL {
mode = globalMinioModeXL
}
if globalIsDistXL {
mode = globalMinioModeDistXL
} else if globalIsXL {
mode = globalMinioModeXL
}
checkUpdate(mode)
}
@ -518,31 +251,18 @@ func serverMain(ctx *cli.Context) {
initNSLock(globalIsDistXL)
// Configure server.
srvConfig := serverCmdConfig{
serverAddr: globalMinioAddr,
endpoints: globalEndpoints,
}
// Configure server.
handler, err := configureServerHandler(srvConfig)
handler, err := configureServerHandler(globalEndpoints)
fatalIf(err, "Unable to configure one of server's RPC services.")
// Initialize a new HTTP server.
apiServer := NewServerMux(globalMinioAddr, handler)
// Set the global minio addr for this server.
globalMinioAddr = getLocalAddress(srvConfig)
// Initialize S3 Peers inter-node communication only in distributed setup.
initGlobalS3Peers(globalEndpoints)
// Initialize Admin Peers inter-node communication only in distributed setup.
initGlobalAdminPeers(globalEndpoints)
// Determine API endpoints where we are going to serve the S3 API from.
globalAPIEndpoints, err = finalizeAPIEndpoints(apiServer.Addr)
fatalIf(err, "Unable to finalize API endpoints for %s", apiServer.Addr)
// Start server, automatically configures TLS if certs are available.
go func() {
cert, key := "", ""
@ -552,7 +272,7 @@ func serverMain(ctx *cli.Context) {
fatalIf(apiServer.ListenAndServe(cert, key), "Failed to start minio server.")
}()
newObject, err := newObjectLayer(srvConfig)
newObject, err := newObjectLayer(globalEndpoints)
fatalIf(err, "Initializing object layer failed")
globalObjLayerMutex.Lock()
@ -560,7 +280,8 @@ func serverMain(ctx *cli.Context) {
globalObjLayerMutex.Unlock()
// Prints the formatted startup message once object layer is initialized.
printStartupMessage(globalAPIEndpoints)
apiEndpoints := getAPIEndpoints(apiServer.Addr)
printStartupMessage(apiEndpoints)
// Set uptime time after object layer has initialized.
globalBootTime = UTCNow()
@ -570,40 +291,26 @@ func serverMain(ctx *cli.Context) {
}
// Initialize object layer with the supplied disks, objectLayer is nil upon any error.
func newObjectLayer(srvCmdCfg serverCmdConfig) (newObject ObjectLayer, err error) {
func newObjectLayer(endpoints EndpointList) (newObject ObjectLayer, err error) {
// For FS only, directly use the disk.
isFS := len(srvCmdCfg.endpoints) == 1
isFS := len(endpoints) == 1
if isFS {
// Unescape is needed for some UNC paths on windows
// which are of this form \\127.0.0.1\\export\test.
var fsPath string
fsPath, err = url.QueryUnescape(srvCmdCfg.endpoints[0].String())
if err != nil {
return nil, err
}
// Initialize new FS object layer.
newObject, err = newFSObjectLayer(fsPath)
if err != nil {
return nil, err
return newFSObjectLayer(endpoints[0].Path)
}
// FS initialized, return.
return newObject, nil
}
// First disk argument check if it is local.
firstDisk := isLocalStorage(srvCmdCfg.endpoints[0])
// Initialize storage disks.
storageDisks, err := initStorageDisks(srvCmdCfg.endpoints)
storageDisks, err := initStorageDisks(endpoints)
if err != nil {
return nil, err
}
// Wait for formatting disks for XL backend.
var formattedDisks []StorageAPI
formattedDisks, err = waitForFormatXLDisks(firstDisk, srvCmdCfg.endpoints, storageDisks)
// First disk argument check if it is local.
firstDisk := endpoints[0].IsLocal
formattedDisks, err = waitForFormatXLDisks(firstDisk, endpoints, storageDisks)
if err != nil {
return nil, err
}

390
cmd/server-main_test.go
View File

@ -17,205 +17,10 @@
package cmd
import (
"errors"
"reflect"
"runtime"
"testing"
)
func TestGetListenIPs(t *testing.T) {
testCases := []struct {
addr string
port string
shouldPass bool
}{
{"127.0.0.1", "9000", true},
{"", "9000", true},
{"", "", false},
}
for _, test := range testCases {
var addr string
// Please keep this we need to do this because
// of odd https://play.golang.org/p/4dMPtM6Wdd
// implementation issue.
if test.port != "" {
addr = test.addr + ":" + test.port
}
hosts, port, err := getListenIPs(addr)
if !test.shouldPass && err == nil {
t.Fatalf("Test should fail but succeeded %s", err)
}
if test.shouldPass && err != nil {
t.Fatalf("Test should succeed but failed %s", err)
}
if test.shouldPass {
if port != test.port {
t.Errorf("Test expected %s, got %s", test.port, port)
}
if len(hosts) == 0 {
t.Errorf("Test unexpected value hosts cannot be empty %#v", test)
}
}
}
}
// Tests get host port.
func TestGetHostPort(t *testing.T) {
testCases := []struct {
addr string
err error
}{
// Test 1 - successful.
{
addr: ":" + getFreePort(),
err: nil,
},
// Test 2 port empty.
{
addr: ":0",
err: errEmptyPort,
},
// Test 3 port empty.
{
addr: ":",
err: errEmptyPort,
},
// Test 4 invalid port.
{
addr: "linux:linux",
err: errors.New("strconv.ParseInt: parsing \"linux\": invalid syntax"),
},
// Test 5 port not present.
{
addr: "hostname",
err: errors.New("missing port in address hostname"),
},
}
// Validate all tests.
for i, testCase := range testCases {
_, _, err := getHostPort(testCase.addr)
if err != nil {
if err.Error() != testCase.err.Error() {
t.Fatalf("Test %d: Error: %s", i+1, err)
}
}
}
}
// Tests finalize api endpoints.
func TestFinalizeAPIEndpoints(t *testing.T) {
testCases := []struct {
addr string
}{
{":80"},
{":80"},
{"127.0.0.1:80"},
{"127.0.0.1:80"},
}
for i, test := range testCases {
endPoints, err := finalizeAPIEndpoints(test.addr)
if err != nil && len(endPoints) <= 0 {
t.Errorf("Test case %d returned with no API end points for %s",
i+1, test.addr)
}
}
}
// Tests all the expected input disks for function checkSufficientDisks.
func TestCheckSufficientDisks(t *testing.T) {
var xlDisks []string
if runtime.GOOS == globalWindowsOSName {
xlDisks = []string{
"C:\\mnt\\backend1",
"C:\\mnt\\backend2",
"C:\\mnt\\backend3",
"C:\\mnt\\backend4",
"C:\\mnt\\backend5",
"C:\\mnt\\backend6",
"C:\\mnt\\backend7",
"C:\\mnt\\backend8",
"C:\\mnt\\backend9",
"C:\\mnt\\backend10",
"C:\\mnt\\backend11",
"C:\\mnt\\backend12",
"C:\\mnt\\backend13",
"C:\\mnt\\backend14",
"C:\\mnt\\backend15",
"C:\\mnt\\backend16",
"C:\\mnt\\backend17",
}
} else {
xlDisks = []string{
"/mnt/backend1",
"/mnt/backend2",
"/mnt/backend3",
"/mnt/backend4",
"/mnt/backend5",
"/mnt/backend6",
"/mnt/backend7",
"/mnt/backend8",
"/mnt/backend9",
"/mnt/backend10",
"/mnt/backend11",
"/mnt/backend12",
"/mnt/backend13",
"/mnt/backend14",
"/mnt/backend15",
"/mnt/backend16",
"/mnt/backend17",
}
}
// List of test cases fo sufficient disk verification.
testCases := []struct {
disks []string
expectedErr error
}{
// Even number of disks '6'.
{
xlDisks[0:6],
nil,
},
// Even number of disks '12'.
{
xlDisks[0:12],
nil,
},
// Even number of disks '16'.
{
xlDisks[0:16],
nil,
},
// Larger than maximum number of disks > 16.
{
xlDisks,
errXLMaxDisks,
},
// Lesser than minimum number of disks < 6.
{
xlDisks[0:3],
errXLMinDisks,
},
// Odd number of disks, not divisible by '2'.
{
append(xlDisks[0:10], xlDisks[11]),
errXLNumDisks,
},
}
// Validates different variations of input disks.
for i, testCase := range testCases {
endpoints, err := parseStorageEndpoints(testCase.disks)
if err != nil {
t.Fatalf("Unexpected error %s", err)
}
if checkSufficientDisks(endpoints) != testCase.expectedErr {
t.Errorf("Test %d expected to pass for disks %s", i+1, testCase.disks)
}
}
}
// Tests initializing new object layer.
func TestNewObjectLayer(t *testing.T) {
// Tests for FS object layer.
@ -226,15 +31,8 @@ func TestNewObjectLayer(t *testing.T) {
}
defer removeRoots(disks)
endpoints, err := parseStorageEndpoints(disks)
if err != nil {
t.Fatal("Unexpected parse error", err)
}
obj, err := newObjectLayer(serverCmdConfig{
serverAddr: ":9000",
endpoints: endpoints,
})
endpoints := mustGetNewEndpointList(disks...)
obj, err := newObjectLayer(endpoints)
if err != nil {
t.Fatal("Unexpected object layer initialization error", err)
}
@ -253,15 +51,8 @@ func TestNewObjectLayer(t *testing.T) {
}
defer removeRoots(disks)
endpoints, err = parseStorageEndpoints(disks)
if err != nil {
t.Fatal("Unexpected parse error", err)
}
obj, err = newObjectLayer(serverCmdConfig{
serverAddr: ":9000",
endpoints: endpoints,
})
endpoints = mustGetNewEndpointList(disks...)
obj, err = newObjectLayer(endpoints)
if err != nil {
t.Fatal("Unexpected object layer initialization error", err)
}
@ -271,176 +62,3 @@ func TestNewObjectLayer(t *testing.T) {
t.Fatal("Unexpected object layer detected", reflect.TypeOf(obj))
}
}
// Tests parsing various types of input endpoints and paths.
func TestParseStorageEndpoints(t *testing.T) {
testCases := []struct {
globalMinioHost string
host string
expectedErr error
}{
{"", "http://127.0.0.1/export", nil},
{
"testhost",
"http://127.0.0.1/export",
errors.New("Invalid Argument 127.0.0.1, port mandatory when --address <host>:<port> is used"),
},
{
"",
"http://127.0.0.1:9000/export",
errors.New("Invalid Argument 127.0.0.1:9000, port configurable using --address :<port>"),
},
{"testhost", "http://127.0.0.1:9000/export", nil},
}
for i, test := range testCases {
globalMinioHost = test.globalMinioHost
_, err := parseStorageEndpoints([]string{test.host})
if err != nil {
if err.Error() != test.expectedErr.Error() {
t.Errorf("Test %d : got %v, expected %v", i+1, err, test.expectedErr)
}
}
}
// Should be reset back to "" so that we don't affect other tests.
globalMinioHost = ""
}
// Test check endpoints syntax function for syntax verification
// across various scenarios of inputs.
func TestCheckEndpointsSyntax(t *testing.T) {
successCases := []string{
"export",
"/export",
"http://127.0.0.1/export",
"https://127.0.0.1/export",
}
failureCases := []string{
"/",
"http://127.0.0.1",
"http://127.0.0.1/",
"ftp://127.0.0.1/export",
"server:/export",
}
if runtime.GOOS == globalWindowsOSName {
successCases = append(successCases,
`\export`,
`D:\export`,
)
failureCases = append(failureCases,
"D:",
`D:\`,
`\`,
)
}
for _, disk := range successCases {
eps, err := parseStorageEndpoints([]string{disk})
if err != nil {
t.Fatalf("Unable to parse %s, error %s", disk, err)
}
if err = checkEndpointsSyntax(eps, []string{disk}); err != nil {
t.Errorf("expected: <nil>, got: %s", err)
}
}
for _, disk := range failureCases {
eps, err := parseStorageEndpoints([]string{disk})
if err != nil {
t.Fatalf("Unable to parse %s, error %s", disk, err)
}
if err = checkEndpointsSyntax(eps, []string{disk}); err == nil {
t.Errorf("expected: <error>, got: <nil>")
}
}
}
func TestIsDistributedSetup(t *testing.T) {
var testCases []struct {
disks []string
result bool
}
if runtime.GOOS == globalWindowsOSName {
testCases = []struct {
disks []string
result bool
}{
{[]string{`http://4.4.4.4/c:\mnt\disk1`, `http://4.4.4.4/c:\mnt\disk2`}, true},
{[]string{`http://4.4.4.4/c:\mnt\disk1`, `http://127.0.0.1/c:\mnt\disk2`}, true},
{[]string{`c:\mnt\disk1`, `c:\mnt\disk2`}, false},
}
} else {
testCases = []struct {
disks []string
result bool
}{
{[]string{"http://4.4.4.4/mnt/disk1", "http://4.4.4.4/mnt/disk2"}, true},
{[]string{"http://4.4.4.4/mnt/disk1", "http://127.0.0.1/mnt/disk2"}, true},
{[]string{"/mnt/disk1", "/mnt/disk2"}, false},
}
}
for i, test := range testCases {
endpoints, err := parseStorageEndpoints(test.disks)
if err != nil {
t.Fatalf("Test %d: Unexpected error: %s", i+1, err)
}
res := isDistributedSetup(endpoints)
if res != test.result {
t.Errorf("Test %d: expected result %t but received %t", i+1, test.result, res)
}
}
// Test cases when globalMinioHost is set
globalMinioHost = "testhost"
testCases = []struct {
disks []string
result bool
}{
{[]string{"http://127.0.0.1:9001/mnt/disk1", "http://127.0.0.1:9002/mnt/disk2", "http://127.0.0.1:9003/mnt/disk3", "http://127.0.0.1:9004/mnt/disk4"}, true},
{[]string{"/mnt/disk1", "/mnt/disk2"}, false},
}
for i, test := range testCases {
endpoints, err := parseStorageEndpoints(test.disks)
if err != nil {
t.Fatalf("Test %d: Unexpected error: %s", i+1, err)
}
res := isDistributedSetup(endpoints)
if res != test.result {
t.Errorf("Test %d: expected result %t but received %t", i+1, test.result, res)
}
}
// Reset so that we don't affect other tests.
globalMinioHost = ""
}
// Tests isAnyEndpointLocal function with inputs such that it returns true and false respectively.
func TestIsAnyEndpointLocal(t *testing.T) {
testCases := []struct {
disks []string
result bool
}{
{
disks: []string{"http://4.4.4.4/mnt/disk1",
"http://4.4.4.4/mnt/disk1"},
result: false,
},
{
disks: []string{"http://127.0.0.1/mnt/disk1",
"http://127.0.0.1/mnt/disk1"},
result: true,
},
}
for i, test := range testCases {
endpoints, err := parseStorageEndpoints(test.disks)
if err != nil {
t.Fatalf("Test %d - Failed to parse storage endpoints %v", i+1, err)
}
actual := isAnyEndpointLocal(endpoints)
if actual != test.result {
t.Errorf("Test %d - Expected %v but received %v", i+1, test.result, actual)
}
}
}

71
cmd/server-startup-utils.go
View File

@ -1,71 +0,0 @@
/*
* Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import (
"fmt"
"net"
)
// getListenIPs - gets all the ips to listen on.
func getListenIPs(serverAddr string) (hosts []string, port string, err error) {
var host string
host, port, err = net.SplitHostPort(serverAddr)
if err != nil {
return nil, port, fmt.Errorf("Unable to parse host address %s", err)
}
if host == "" {
var ipv4s []net.IP
ipv4s, err = getInterfaceIPv4s()
if err != nil {
return nil, port, fmt.Errorf("Unable reverse sort ips from hosts %s", err)
}
for _, ip := range ipv4s {
hosts = append(hosts, ip.String())
}
return hosts, port, nil
} // if host != "" {
// Proceed to append itself, since user requested a specific endpoint.
hosts = append(hosts, host)
// Success.
return hosts, port, nil
}
// Finalizes the API endpoints based on the host list and port.
func finalizeAPIEndpoints(addr string) (endPoints []string, err error) {
// Verify current scheme.
scheme := httpScheme
if globalIsSSL {
scheme = httpsScheme
}
// Get list of listen ips and port.
hosts, port, err1 := getListenIPs(addr)
if err1 != nil {
return nil, err1
}
// Construct proper endpoints.
for _, host := range hosts {
endPoints = append(endPoints, fmt.Sprintf("%s://%s:%s", scheme, host, port))
}
// Success.
return endPoints, nil
}

38
cmd/url-sort.go → cmd/setup-type.go
View File

@ -1,5 +1,5 @@
/*
* Minio Cloud Storage, (C) 2016 Minio, Inc.
* Minio Cloud Storage, (C) 2017 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,19 +16,29 @@
package cmd
import "net/url"
// SetupType - enum for setup type.
type SetupType int
type byHostPath []*url.URL
const (
// FSSetupType - FS setup type enum.
FSSetupType SetupType = iota + 1
func (s byHostPath) Swap(i, j int) {
s[i], s[j] = s[j], s[i]
}
func (s byHostPath) Len() int {
return len(s)
}
// Note: Host in url.URL includes the port too.
func (s byHostPath) Less(i, j int) bool {
return (s[i].Host + s[i].Path) < (s[j].Host + s[j].Path)
// XLSetupType - XL setup type enum.
XLSetupType
// DistXLSetupType - Distributed XL setup type enum.
DistXLSetupType
)
func (setupType SetupType) String() string {
switch setupType {
case FSSetupType:
return globalMinioModeFS
case XLSetupType:
return globalMinioModeXL
case DistXLSetupType:
return globalMinioModeDistXL
}
return ""
}

16
cmd/signature-v2.go
View File

@ -85,8 +85,12 @@ func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode {
cred := serverConfig.GetCredential()
// r.RequestURI will have raw encoded URI as sent by the client.
splits := splitStr(r.RequestURI, "?", 2)
encodedResource, encodedQuery := splits[0], splits[1]
tokens := strings.SplitN(r.RequestURI, "?", 2)
encodedResource := tokens[0]
encodedQuery := ""
if len(tokens) == 2 {
encodedQuery = tokens[1]
}
queries := strings.Split(encodedQuery, "&")
var filteredQueries []string
@ -206,8 +210,12 @@ func doesSignV2Match(r *http.Request) APIErrorCode {
}
// r.RequestURI will have raw encoded URI as sent by the client.
splits := splitStr(r.RequestURI, "?", 2)
encodedResource, encodedQuery := splits[0], splits[1]
tokens := strings.SplitN(r.RequestURI, "?", 2)
encodedResource := tokens[0]
encodedQuery := ""
if len(tokens) == 2 {
encodedQuery = tokens[1]
}
expectedAuth := signatureV2(r.Method, encodedResource, encodedQuery, r.Header)
if v2Auth != expectedAuth {

30
cmd/storage-rpc-client.go
View File

@ -21,7 +21,6 @@ import (
"io"
"net"
"net/rpc"
"net/url"
"path"
"strings"
@ -96,39 +95,22 @@ func toStorageErr(err error) error {
}
// Initialize new storage rpc client.
func newStorageRPC(ep *url.URL) (StorageAPI, error) {
if ep == nil {
return nil, errInvalidArgument
}
func newStorageRPC(endpoint Endpoint) StorageAPI {
// Dial minio rpc storage http path.
rpcPath := path.Join(minioReservedBucketPath, storageRPCPath, getPath(ep))
rpcAddr := ep.Host
rpcPath := path.Join(minioReservedBucketPath, storageRPCPath, endpoint.Path)
serverCred := serverConfig.GetCredential()
accessKey := serverCred.AccessKey
secretKey := serverCred.SecretKey
if ep.User != nil {
accessKey = ep.User.Username()
if password, ok := ep.User.Password(); ok {
secretKey = password
}
}
storageAPI := &networkStorage{
return &networkStorage{
rpcClient: newAuthRPCClient(authConfig{
accessKey: accessKey,
secretKey: secretKey,
serverAddr: rpcAddr,
accessKey: serverCred.AccessKey,
secretKey: serverCred.SecretKey,
serverAddr: endpoint.Host,
serviceEndpoint: rpcPath,
secureConn: globalIsSSL,
serviceName: "Storage",
disableReconnect: true,
}),
}
// Returns successfully here.
return storageAPI, nil
}
// Stringer interface compatible representation of network device.

23
cmd/storage-rpc-client_test.go
View File

@ -23,7 +23,6 @@ import (
"io"
"net"
"net/rpc"
"net/url"
"runtime"
"testing"
)
@ -146,25 +145,15 @@ func (s *TestRPCStorageSuite) SetUpSuite(c *testing.T) {
listenAddress := s.testServer.Server.Listener.Addr().String()
for _, ep := range s.testServer.Disks {
ep.Host = listenAddress
storageDisk, err := newStorageRPC(ep)
if err != nil {
c.Fatal("Unable to initialize RPC client", err)
// Eventhough s.testServer.Disks is EndpointList, we would need a URLEndpointType here.
endpoint := ep
if endpoint.Type() == PathEndpointType {
endpoint.Scheme = "http"
}
endpoint.Host = listenAddress
storageDisk := newStorageRPC(endpoint)
s.remoteDisks = append(s.remoteDisks, storageDisk)
}
_, err := newStorageRPC(nil)
if err != errInvalidArgument {
c.Fatalf("Unexpected error %s, expecting %s", err, errInvalidArgument)
}
u, err := url.Parse("http://abcd:abcd123@localhost/mnt/disk")
if err != nil {
c.Fatal("Unexpected error", err)
}
_, err = newStorageRPC(u)
if err != nil {
c.Fatal("Unexpected error", err)
}
}
// No longer used with gocheck, but used in explicit teardown code in

20
cmd/storage-rpc-server.go
View File

@ -197,30 +197,28 @@ func (s *storageServer) RenameFileHandler(args *RenameFileArgs, reply *AuthRPCRe
}
// Initialize new storage rpc.
func newRPCServer(srvConfig serverCmdConfig) (servers []*storageServer, err error) {
for _, ep := range srvConfig.endpoints {
// e.g server:/mnt/disk1
if isLocalStorage(ep) {
// Get the posix path.
path := getPath(ep)
var storage StorageAPI
storage, err = newPosix(path)
func newRPCServer(endpoints EndpointList) (servers []*storageServer, err error) {
for _, endpoint := range endpoints {
if endpoint.IsLocal {
storage, err := newPosix(endpoint.Path)
if err != nil && err != errDiskNotFound {
return nil, err
}
servers = append(servers, &storageServer{
storage: storage,
path: path,
path: endpoint.Path,
})
}
}
return servers, nil
}
// registerStorageRPCRouter - register storage rpc router.
func registerStorageRPCRouters(mux *router.Router, srvCmdConfig serverCmdConfig) error {
func registerStorageRPCRouters(mux *router.Router, endpoints EndpointList) error {
// Initialize storage rpc servers for every disk that is hosted on this node.
storageRPCs, err := newRPCServer(srvCmdConfig)
storageRPCs, err := newRPCServer(endpoints)
if err != nil {
return traceError(err)
}

9
cmd/storage-rpc-server_test.go
View File

@ -17,7 +17,6 @@
package cmd
import (
"net/url"
"testing"
"github.com/minio/minio/pkg/disk"
@ -30,7 +29,7 @@ type testStorageRPCServer struct {
token string
diskDirs []string
stServer *storageServer
endpoints []*url.URL
endpoints EndpointList
}
func createTestStorageServer(t *testing.T) *testStorageRPCServer {
@ -50,11 +49,7 @@ func createTestStorageServer(t *testing.T) *testStorageRPCServer {
t.Fatalf("unable to create FS backend, %s", err)
}
endpoints, err := parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatalf("unable to parse storage endpoints, %s", err)
}
endpoints := mustGetNewEndpointList(fsDirs...)
storageDisks, err := initStorageDisks(endpoints)
if err != nil {
t.Fatalf("unable to initialize storage disks, %s", err)

105
cmd/test-utils_test.go
View File

@ -91,11 +91,7 @@ func prepareXL() (ObjectLayer, []string, error) {
if err != nil {
return nil, nil, err
}
endpoints, err := parseStorageEndpoints(fsDirs)
if err != nil {
return nil, nil, err
}
obj, _, err := initObjectLayer(endpoints)
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
removeRoots(fsDirs)
return nil, nil, err
@ -180,12 +176,12 @@ func isSameType(obj1, obj2 interface{}) bool {
// defer s.Stop()
type TestServer struct {
Root string
Disks []*url.URL
Disks EndpointList
AccessKey string
SecretKey string
Server *httptest.Server
Obj ObjectLayer
SrvCmdCfg serverCmdConfig
endpoints EndpointList
}
// UnstartedTestServer - Configures a temp FS/XL backend,
@ -210,50 +206,31 @@ func UnstartedTestServer(t TestErrHandler, instanceType string) TestServer {
credentials := serverConfig.GetCredential()
testServer.Obj = objLayer
testServer.Disks, err = parseStorageEndpoints(disks)
if err != nil {
t.Fatalf("Unexpected error %v", err)
}
testServer.Disks = mustGetNewEndpointList(disks...)
testServer.Root = root
testServer.AccessKey = credentials.AccessKey
testServer.SecretKey = credentials.SecretKey
srvCmdCfg := serverCmdConfig{
endpoints: testServer.Disks,
}
httpHandler, err := configureServerHandler(
srvCmdCfg,
)
httpHandler, err := configureServerHandler(testServer.Disks)
if err != nil {
t.Fatalf("Failed to configure one of the RPC services <ERROR> %s", err)
}
// Run TestServer.
testServer.Server = httptest.NewUnstartedServer(httpHandler)
// obtain server address.
srvCmdCfg.serverAddr = testServer.Server.Listener.Addr().String()
globalObjLayerMutex.Lock()
globalObjectAPI = objLayer
globalObjLayerMutex.Unlock()
// initialize peer rpc
host, port, err := net.SplitHostPort(srvCmdCfg.serverAddr)
if err != nil {
t.Fatal("Early setup error:", err)
}
host, port := mustSplitHostPort(testServer.Server.Listener.Addr().String())
globalMinioHost = host
globalMinioPort = port
globalMinioAddr = getLocalAddress(srvCmdCfg)
endpoints, err := parseStorageEndpoints(disks)
if err != nil {
t.Fatal("Early setup error:", err)
}
initGlobalS3Peers(endpoints)
globalMinioAddr = getEndpointsLocalAddr(testServer.Disks)
initGlobalS3Peers(testServer.Disks)
return testServer
}
// testServerCertPEM and testServerKeyPEM are generated by
@ -339,10 +316,10 @@ func StartTestServer(t TestErrHandler, instanceType string) TestServer {
// Initializes storage RPC endpoints.
// The object Layer will be a temp back used for testing purpose.
func initTestStorageRPCEndPoint(srvCmdConfig serverCmdConfig) http.Handler {
func initTestStorageRPCEndPoint(endpoints EndpointList) http.Handler {
// Initialize router.
muxRouter := router.NewRouter()
registerStorageRPCRouters(muxRouter, srvCmdConfig)
registerStorageRPCRouters(muxRouter, endpoints)
return muxRouter
}
@ -354,10 +331,6 @@ func StartTestStorageRPCServer(t TestErrHandler, instanceType string, diskN int)
if err != nil {
t.Fatal("Failed to create disks for the backend")
}
endpoints, err := parseStorageEndpoints(disks)
if err != nil {
t.Fatalf("%s", err)
}
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
@ -369,15 +342,14 @@ func StartTestStorageRPCServer(t TestErrHandler, instanceType string, diskN int)
// Get credential.
credentials := serverConfig.GetCredential()
endpoints := mustGetNewEndpointList(disks...)
testRPCServer.Root = root
testRPCServer.Disks = endpoints
testRPCServer.AccessKey = credentials.AccessKey
testRPCServer.SecretKey = credentials.SecretKey
// Run TestServer.
testRPCServer.Server = httptest.NewServer(initTestStorageRPCEndPoint(serverCmdConfig{
endpoints: endpoints,
}))
testRPCServer.Server = httptest.NewServer(initTestStorageRPCEndPoint(endpoints))
return testRPCServer
}
@ -389,10 +361,6 @@ func StartTestPeersRPCServer(t TestErrHandler, instanceType string) TestServer {
if err != nil {
t.Fatal("Failed to create disks for the backend")
}
endpoints, err := parseStorageEndpoints(disks)
if err != nil {
t.Fatalf("%s", err)
}
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
@ -404,6 +372,7 @@ func StartTestPeersRPCServer(t TestErrHandler, instanceType string) TestServer {
// Get credential.
credentials := serverConfig.GetCredential()
endpoints := mustGetNewEndpointList(disks...)
testRPCServer.Root = root
testRPCServer.Disks = endpoints
testRPCServer.AccessKey = credentials.AccessKey
@ -420,13 +389,9 @@ func StartTestPeersRPCServer(t TestErrHandler, instanceType string) TestServer {
testRPCServer.Obj = objLayer
globalObjLayerMutex.Unlock()
srvCfg := serverCmdConfig{
endpoints: endpoints,
}
mux := router.NewRouter()
// need storage layer for bucket config storage.
registerStorageRPCRouters(mux, srvCfg)
registerStorageRPCRouters(mux, endpoints)
// need API layer to send requests, etc.
registerAPIRouter(mux)
// module being tested is Peer RPCs router.
@ -436,7 +401,7 @@ func StartTestPeersRPCServer(t TestErrHandler, instanceType string) TestServer {
testRPCServer.Server = httptest.NewServer(mux)
// initialize remainder of serverCmdConfig
testRPCServer.SrvCmdCfg = srvCfg
testRPCServer.endpoints = endpoints
return testRPCServer
}
@ -481,7 +446,7 @@ func resetGlobalEventnotify() {
}
func resetGlobalEndpoints() {
globalEndpoints = []*url.URL{}
globalEndpoints = EndpointList{}
}
func resetGlobalIsXL() {
@ -1659,7 +1624,7 @@ func getRandomDisks(N int) ([]string, error) {
}
// initObjectLayer - Instantiates object layer and returns it.
func initObjectLayer(endpoints []*url.URL) (ObjectLayer, []StorageAPI, error) {
func initObjectLayer(endpoints EndpointList) (ObjectLayer, []StorageAPI, error) {
storageDisks, err := initStorageDisks(endpoints)
if err != nil {
return nil, nil, err
@ -1738,12 +1703,8 @@ func prepareXLStorageDisks(t *testing.T) ([]StorageAPI, []string) {
if err != nil {
t.Fatal("Unexpected error: ", err)
}
endpoints, err := parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal("Unexpected error: ", err)
}
_, storageDisks, err := initObjectLayer(endpoints)
_, storageDisks, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
removeRoots(fsDirs)
t.Fatal("Unable to initialize storage disks", err)
@ -2077,11 +2038,7 @@ func ExecObjectLayerStaleFilesTest(t *testing.T, objTest objTestStaleFilesType)
if err != nil {
t.Fatalf("Initialization of disks for XL setup: %s", err)
}
endpoints, err := parseStorageEndpoints(erasureDisks)
if err != nil {
t.Fatalf("Initialization of disks for XL setup: %s", err)
}
objLayer, _, err := initObjectLayer(endpoints)
objLayer, _, err := initObjectLayer(mustGetNewEndpointList(erasureDisks...))
if err != nil {
t.Fatalf("Initialization of object layer failed for XL setup: %s", err)
}
@ -2380,3 +2337,27 @@ func generateTLSCertKey(host string) ([]byte, []byte, error) {
return certOut.Bytes(), keyOut.Bytes(), nil
}
func mustGetNewEndpointList(args ...string) (endpoints EndpointList) {
if len(args) == 1 {
endpoint, err := NewEndpoint(args[0])
fatalIf(err, "unable to create new endpoint")
endpoints = append(endpoints, endpoint)
} else {
var err error
endpoints, err = NewEndpointList(args...)
fatalIf(err, "unable to create new endpoint list")
}
return endpoints
}
func getEndpointsLocalAddr(endpoints EndpointList) string {
for _, endpoint := range endpoints {
if endpoint.IsLocal && endpoint.Type() == URLEndpointType {
return endpoint.Host
}
}
return globalMinioHost + ":" + globalMinioPort
}

34
cmd/tree-walk_test.go
View File

@ -164,10 +164,7 @@ func TestTreeWalk(t *testing.T) {
if err != nil {
t.Fatalf("Unable to create tmp directory: %s", err)
}
endpoints, err := parseStorageEndpoints([]string{fsDir})
if err != nil {
t.Fatalf("Unexpected error %s", err)
}
endpoints := mustGetNewEndpointList(fsDir)
disk, err := newStorageAPI(endpoints[0])
if err != nil {
t.Fatalf("Unable to create StorageAPI: %s", err)
@ -205,10 +202,7 @@ func TestTreeWalkTimeout(t *testing.T) {
if err != nil {
t.Fatalf("Unable to create tmp directory: %s", err)
}
endpoints, err := parseStorageEndpoints([]string{fsDir})
if err != nil {
t.Fatalf("Unexpected error %s", err)
}
endpoints := mustGetNewEndpointList(fsDir)
disk, err := newStorageAPI(endpoints[0])
if err != nil {
t.Fatalf("Unable to create StorageAPI: %s", err)
@ -285,18 +279,15 @@ func TestListDir(t *testing.T) {
t.Errorf("Unable to create tmp directory: %s", err)
}
endpoints, err := parseStorageEndpoints([]string{fsDir1, fsDir2})
if err != nil {
t.Fatalf("Unexpected error %s", err)
}
// Create two StorageAPIs disk1 and disk2.
endpoints := mustGetNewEndpointList(fsDir1)
disk1, err := newStorageAPI(endpoints[0])
if err != nil {
t.Errorf("Unable to create StorageAPI: %s", err)
}
disk2, err := newStorageAPI(endpoints[1])
endpoints = mustGetNewEndpointList(fsDir2)
disk2, err := newStorageAPI(endpoints[0])
if err != nil {
t.Errorf("Unable to create StorageAPI: %s", err)
}
@ -366,10 +357,7 @@ func TestRecursiveTreeWalk(t *testing.T) {
t.Fatalf("Unable to create tmp directory: %s", err)
}
endpoints, err := parseStorageEndpoints([]string{fsDir1})
if err != nil {
t.Fatalf("Unexpected error %s", err)
}
endpoints := mustGetNewEndpointList(fsDir1)
disk1, err := newStorageAPI(endpoints[0])
if err != nil {
t.Fatalf("Unable to create StorageAPI: %s", err)
@ -476,10 +464,7 @@ func TestSortedness(t *testing.T) {
t.Errorf("Unable to create tmp directory: %s", err)
}
endpoints, err := parseStorageEndpoints([]string{fsDir1})
if err != nil {
t.Fatalf("Unexpected error %s", err)
}
endpoints := mustGetNewEndpointList(fsDir1)
disk1, err := newStorageAPI(endpoints[0])
if err != nil {
t.Fatalf("Unable to create StorageAPI: %s", err)
@ -554,10 +539,7 @@ func TestTreeWalkIsEnd(t *testing.T) {
t.Errorf("Unable to create tmp directory: %s", err)
}
endpoints, err := parseStorageEndpoints([]string{fsDir1})
if err != nil {
t.Fatalf("Unexpected error %s", err)
}
endpoints := mustGetNewEndpointList(fsDir1)
disk1, err := newStorageAPI(endpoints[0])
if err != nil {
t.Fatalf("Unable to create StorageAPI: %s", err)

99
cmd/url-sort_test.go
View File

@ -1,99 +0,0 @@
/*
* Minio Cloud Storage, (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import (
"net/url"
"reflect"
"sort"
"testing"
)
// TestSortByHostPath - tests if ordering of urls are based on
// host+path concatenated.
func TestSortByHostPath(t *testing.T) {
testCases := []struct {
given []string
expected []*url.URL
}{
{
given: []string{
"http://abcd.com/a/b/d",
"http://abcd.com/a/b/c",
"http://abcd.com/a/b/e",
},
expected: []*url.URL{
{
Scheme: httpScheme,
Host: "abcd.com:9000",
Path: "/a/b/c",
},
{
Scheme: httpScheme,
Host: "abcd.com:9000",
Path: "/a/b/d",
},
{
Scheme: httpScheme,
Host: "abcd.com:9000",
Path: "/a/b/e",
},
},
},
{
given: []string{
"http://defg.com/a/b/c",
"http://abcd.com/a/b/c",
"http://hijk.com/a/b/c",
},
expected: []*url.URL{
{
Scheme: httpScheme,
Host: "abcd.com:9000",
Path: "/a/b/c",
},
{
Scheme: httpScheme,
Host: "defg.com:9000",
Path: "/a/b/c",
},
{
Scheme: httpScheme,
Host: "hijk.com:9000",
Path: "/a/b/c",
},
},
},
}
saveGlobalPort := globalMinioPort
globalMinioPort = "9000"
for i, test := range testCases {
eps, err := parseStorageEndpoints(test.given)
if err != nil {
t.Fatalf("Test %d - Failed to parse storage endpoint %v", i+1, err)
}
sort.Sort(byHostPath(eps))
if !sort.IsSorted(byHostPath(eps)) {
t.Errorf("Test %d - Expected order %v but got %v", i+1, test.expected, eps)
}
if !reflect.DeepEqual(eps, test.expected) {
t.Errorf("Test %d - Expected order %v but got %v", i+1, test.expected, eps)
}
}
globalMinioPort = saveGlobalPort
}

70
cmd/utils.go
View File

@ -45,44 +45,6 @@ func cloneHeader(h http.Header) http.Header {
return h2
}
// checkDuplicates - function to validate if there are duplicates in a slice of strings.
func checkDuplicateStrings(list []string) error {
// Empty lists are not allowed.
if len(list) == 0 {
return errInvalidArgument
}
// Empty keys are not allowed.
for _, key := range list {
if key == "" {
return errInvalidArgument
}
}
listMaps := make(map[string]int)
// Navigate through each configs and count the entries.
for _, key := range list {
listMaps[key]++
}
// Validate if there are any duplicate counts.
for key, count := range listMaps {
if count != 1 {
return fmt.Errorf("Duplicate key: \"%s\" found of count: \"%d\"", key, count)
}
}
// No duplicates.
return nil
}
// splitStr splits a string into n parts, empty strings are added
// if we are not able to reach n elements
func splitStr(path, sep string, n int) []string {
splits := strings.SplitN(path, sep, n)
// Add empty strings if we found elements less than nr
for i := n - len(splits); i > 0; i-- {
splits = append(splits, "")
}
return splits
}
// Convert url path into bucket and object name.
func urlPath2BucketObjectName(u *url.URL) (bucketName, objectName string) {
if u == nil {
@ -95,10 +57,11 @@ func urlPath2BucketObjectName(u *url.URL) (bucketName, objectName string) {
// Split urlpath using slash separator into a given number of
// expected tokens.
tokens := splitStr(urlPath, slashSeparator, 2)
// Extract bucket and objects.
bucketName, objectName = tokens[0], tokens[1]
tokens := strings.SplitN(urlPath, slashSeparator, 2)
bucketName = tokens[0]
if len(tokens) == 2 {
objectName = tokens[1]
}
// Success.
return bucketName, objectName
@ -110,29 +73,6 @@ const (
httpsScheme = "https"
)
// checkDuplicates - function to validate if there are duplicates in a slice of endPoints.
func checkDuplicateEndpoints(endpoints []*url.URL) error {
var strs []string
for _, ep := range endpoints {
strs = append(strs, ep.String())
}
return checkDuplicateStrings(strs)
}
// Find local node through the command line arguments. Returns in `host:port` format.
func getLocalAddress(srvCmdConfig serverCmdConfig) string {
if !globalIsDistXL {
return srvCmdConfig.serverAddr
}
for _, ep := range srvCmdConfig.endpoints {
// Validates if remote endpoint is local.
if isLocalStorage(ep) {
return ep.Host
}
}
return ""
}
// xmlDecoder provide decoded value in xml.
func xmlDecoder(body io.Reader, v interface{}, size int64) error {
var lbody io.Reader

170
cmd/utils_test.go
View File

@ -18,12 +18,9 @@ package cmd
import (
"encoding/json"
"fmt"
"net"
"net/http"
"net/url"
"reflect"
"runtime"
"strings"
"testing"
)
@ -52,57 +49,6 @@ func TestCloneHeader(t *testing.T) {
}
}
// Tests check duplicates function.
func TestCheckDuplicates(t *testing.T) {
tests := []struct {
list []string
err error
shouldPass bool
}{
// Test 1 - for '/tmp/1' repeated twice.
{
list: []string{"/tmp/1", "/tmp/1", "/tmp/2", "/tmp/3"},
err: fmt.Errorf("Duplicate key: \"/tmp/1\" found of count: \"2\""),
shouldPass: false,
},
// Test 2 - for '/tmp/1' repeated thrice.
{
list: []string{"/tmp/1", "/tmp/1", "/tmp/1", "/tmp/3"},
err: fmt.Errorf("Duplicate key: \"/tmp/1\" found of count: \"3\""),
shouldPass: false,
},
// Test 3 - empty string.
{
list: []string{""},
err: errInvalidArgument,
shouldPass: false,
},
// Test 4 - empty string.
{
list: nil,
err: errInvalidArgument,
shouldPass: false,
},
// Test 5 - non repeated strings.
{
list: []string{"/tmp/1", "/tmp/2", "/tmp/3"},
err: nil,
shouldPass: true,
},
}
// Validate if function runs as expected.
for i, test := range tests {
err := checkDuplicateStrings(test.list)
if test.shouldPass && err != test.err {
t.Errorf("Test: %d, Expected %s got %s", i+1, test.err, err)
}
if !test.shouldPass && err.Error() != test.err.Error() {
t.Errorf("Test: %d, Expected %s got %s", i+1, test.err, err)
}
}
}
// Tests maximum object size.
func TestMaxObjectSize(t *testing.T) {
sizes := []struct {
@ -275,122 +221,6 @@ func TestStartProfiler(t *testing.T) {
}
}
// Tests fetch local address.
func TestLocalAddress(t *testing.T) {
if runtime.GOOS == globalWindowsOSName {
return
}
currentIsDistXL := globalIsDistXL
defer func() {
globalIsDistXL = currentIsDistXL
}()
// need to set this to avoid stale values from other tests.
globalMinioPort = "9000"
globalMinioHost = ""
testCases := []struct {
isDistXL bool
srvCmdConfig serverCmdConfig
localAddr string
}{
// Test 1 - local address is found.
{
isDistXL: true,
srvCmdConfig: serverCmdConfig{
endpoints: []*url.URL{{
Scheme: httpScheme,
Host: "localhost:9000",
Path: "/mnt/disk1",
}, {
Scheme: httpScheme,
Host: "1.1.1.2:9000",
Path: "/mnt/disk2",
}, {
Scheme: httpScheme,
Host: "1.1.2.1:9000",
Path: "/mnt/disk3",
}, {
Scheme: httpScheme,
Host: "1.1.2.2:9000",
Path: "/mnt/disk4",
}},
},
localAddr: net.JoinHostPort("localhost", globalMinioPort),
},
// Test 2 - local address is everything.
{
isDistXL: false,
srvCmdConfig: serverCmdConfig{
serverAddr: net.JoinHostPort("", globalMinioPort),
endpoints: []*url.URL{{
Path: "/mnt/disk1",
}, {
Path: "/mnt/disk2",
}, {
Path: "/mnt/disk3",
}, {
Path: "/mnt/disk4",
}},
},
localAddr: net.JoinHostPort("", globalMinioPort),
},
// Test 3 - local address is not found.
{
isDistXL: true,
srvCmdConfig: serverCmdConfig{
endpoints: []*url.URL{{
Scheme: httpScheme,
Host: "1.1.1.1:9000",
Path: "/mnt/disk2",
}, {
Scheme: httpScheme,
Host: "1.1.1.2:9000",
Path: "/mnt/disk2",
}, {
Scheme: httpScheme,
Host: "1.1.2.1:9000",
Path: "/mnt/disk3",
}, {
Scheme: httpScheme,
Host: "1.1.2.2:9000",
Path: "/mnt/disk4",
}},
},
localAddr: "",
},
// Test 4 - in case of FS mode, with SSL, the host
// name is specified in the --address option on the
// server command line.
{
isDistXL: false,
srvCmdConfig: serverCmdConfig{
serverAddr: "play.minio.io:9000",
endpoints: []*url.URL{{
Path: "/mnt/disk1",
}, {
Path: "/mnt/disk2",
}, {
Path: "/mnt/disk3",
}, {
Path: "/mnt/disk4",
}},
},
localAddr: "play.minio.io:9000",
},
}
// Validates fetching local address.
for i, testCase := range testCases {
globalIsDistXL = testCase.isDistXL
localAddr := getLocalAddress(testCase.srvCmdConfig)
if localAddr != testCase.localAddr {
t.Fatalf("Test %d: Expected %s, got %s", i+1, testCase.localAddr, localAddr)
}
}
}
// TestCheckURL tests valid url.
func TestCheckURL(t *testing.T) {
testCases := []struct {

105
cmd/xl-v1-healing_test.go
View File

@ -37,13 +37,8 @@ func TestHealFormatXL(t *testing.T) {
t.Fatal(err)
}
endpoints, err := parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// Everything is fine, should return nil
obj, _, err := initObjectLayer(endpoints)
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -59,13 +54,8 @@ func TestHealFormatXL(t *testing.T) {
t.Fatal(err)
}
endpoints, err = parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// Disks 0..15 are nil
obj, _, err = initObjectLayer(endpoints)
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -84,13 +74,8 @@ func TestHealFormatXL(t *testing.T) {
t.Fatal(err)
}
endpoints, err = parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// One disk returns Faulty Disk
obj, _, err = initObjectLayer(endpoints)
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -112,13 +97,8 @@ func TestHealFormatXL(t *testing.T) {
t.Fatal(err)
}
endpoints, err = parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// One disk is not found, heal corrupted disks should return nil
obj, _, err = initObjectLayer(endpoints)
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -134,13 +114,8 @@ func TestHealFormatXL(t *testing.T) {
t.Fatal(err)
}
endpoints, err = parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// Remove format.json of all disks
obj, _, err = initObjectLayer(endpoints)
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -160,13 +135,8 @@ func TestHealFormatXL(t *testing.T) {
t.Fatal(err)
}
endpoints, err = parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// Corrupted format json in one disk
obj, _, err = initObjectLayer(endpoints)
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -186,13 +156,8 @@ func TestHealFormatXL(t *testing.T) {
t.Fatal(err)
}
endpoints, err = parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// Remove format.json on 3 disks.
obj, _, err = initObjectLayer(endpoints)
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -212,13 +177,8 @@ func TestHealFormatXL(t *testing.T) {
t.Fatal(err)
}
endpoints, err = parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// One disk is not found, heal corrupted disks should return nil
obj, _, err = initObjectLayer(endpoints)
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -246,13 +206,8 @@ func TestHealFormatXL(t *testing.T) {
t.Fatal(err)
}
endpoints, err = parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// One disk is not found, heal corrupted disks should return nil
obj, _, err = initObjectLayer(endpoints)
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -286,13 +241,8 @@ func TestUndoMakeBucket(t *testing.T) {
}
defer removeRoots(fsDirs)
endpoints, err := parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// Remove format.json on 16 disks.
obj, _, err := initObjectLayer(endpoints)
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -331,13 +281,8 @@ func TestQuickHeal(t *testing.T) {
}
defer removeRoots(fsDirs)
endpoints, err := parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// Remove format.json on 16 disks.
obj, _, err := initObjectLayer(endpoints)
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -382,13 +327,8 @@ func TestQuickHeal(t *testing.T) {
}
defer removeRoots(fsDirs)
endpoints, err = parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// One disk is not found, heal corrupted disks should return nil
obj, _, err = initObjectLayer(endpoints)
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -404,13 +344,8 @@ func TestQuickHeal(t *testing.T) {
}
defer removeRoots(fsDirs)
endpoints, err = parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// One disk is not found, heal corrupted disks should return nil
obj, _, err = initObjectLayer(endpoints)
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -441,12 +376,7 @@ func TestListBucketsHeal(t *testing.T) {
}
defer removeRoots(fsDirs)
endpoints, err := parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
obj, _, err := initObjectLayer(endpoints)
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
@ -505,13 +435,8 @@ func TestHealObjectXL(t *testing.T) {
defer removeRoots(fsDirs)
endpoints, err := parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatal(err)
}
// Everything is fine, should return nil
obj, _, err := initObjectLayer(endpoints)
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}

6
cmd/xl-v1-utils_test.go
View File

@ -392,11 +392,7 @@ func TestShuffleDisks(t *testing.T) {
if err != nil {
t.Fatal(err)
}
endpoints, err := parseStorageEndpoints(disks)
if err != nil {
t.Fatal(err)
}
objLayer, _, err := initObjectLayer(endpoints)
objLayer, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
if err != nil {
removeRoots(disks)
t.Fatal(err)

13
cmd/xl-v1_test.go
View File

@ -50,12 +50,7 @@ func TestStorageInfo(t *testing.T) {
t.Fatalf("Diskinfo total values should be greater 0")
}
endpoints, err := parseStorageEndpoints(fsDirs)
if err != nil {
t.Fatalf("Unexpected error %s", err)
}
storageDisks, err := initStorageDisks(endpoints)
storageDisks, err := initStorageDisks(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal("Unexpected error: ", err)
}
@ -145,11 +140,7 @@ func TestNewXL(t *testing.T) {
t.Fatalf("Unable to initialize erasure, %s", err)
}
endpoints, err := parseStorageEndpoints(erasureDisks)
if err != nil {
t.Fatalf("Unable to initialize erasure, %s", err)
}
endpoints := mustGetNewEndpointList(erasureDisks...)
storageDisks, err := initStorageDisks(endpoints)
if err != nil {
t.Fatal("Unexpected error: ", err)

8
vendor/github.com/minio/minio-go/pkg/set/stringset.go generated vendored
View File

@ -25,8 +25,8 @@ import (
// StringSet - uses map as set of strings.
type StringSet map[string]struct{}
// keys - returns StringSet keys.
func (set StringSet) keys() []string {
// ToSlice - returns StringSet as string slice.
func (set StringSet) ToSlice() []string {
keys := make([]string, 0, len(set))
for k := range set {
keys = append(keys, k)
@ -141,7 +141,7 @@ func (set StringSet) Union(sset StringSet) StringSet {
// MarshalJSON - converts to JSON data.
func (set StringSet) MarshalJSON() ([]byte, error) {
return json.Marshal(set.keys())
return json.Marshal(set.ToSlice())
}
// UnmarshalJSON - parses JSON data and creates new set with it.
@ -169,7 +169,7 @@ func (set *StringSet) UnmarshalJSON(data []byte) error {
// String - returns printable string of the set.
func (set StringSet) String() string {
return fmt.Sprintf("%s", set.keys())
return fmt.Sprintf("%s", set.ToSlice())
}
// NewStringSet - creates new string set.

6
vendor/vendor.json vendored
View File

@ -227,10 +227,10 @@
"revisionTime": "2016-12-20T20:43:13Z"
},
{
"checksumSHA1": "A8QOw1aWwc+RtjGozY0XeS5varo=",
"checksumSHA1": "maUy+dbN6VfTTnfErrAW2lLit1w=",
"path": "github.com/minio/minio-go/pkg/set",
"revision": "9e734013294ab153b0bdbe182738bcddd46f1947",
"revisionTime": "2016-08-18T00:31:20Z"
"revision": "7a3619e41885dcbcfafee193c10eb80530c2be53",
"revisionTime": "2017-02-17T20:03:45Z"
},
{
"checksumSHA1": "URVle4qtadmW9w9BulDRHY3kxnA=",