Add support for CopyObject across regions and multiple Minio IPs

This PR adds CopyObject support for objects residing in buckets
in different Minio instances (where Minio instances are part of
a federated setup).

Also, added support for multiple Minio domain IPs. This is required
for distributed deployments, where one deployment may have multiple
nodes, each with a different public IP.

cmd/bucket-handlers.go

@@ -31,12 +31,17 @@ import (
 	"sync"
 
 	"github.com/coreos/etcd/client"
+
 	"github.com/gorilla/mux"
+
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/policy"
 	"github.com/minio/minio/pkg/sync/errgroup"
+
+	"github.com/minio/minio-go/pkg/set"
 )
 
 // Check if there are buckets on server without corresponding entry in etcd backend and
@@ -47,7 +52,6 @@ import (
 // -- If yes, check if the IP of entry matches local IP. This means entry is for this instance.
 // -- If IP of the entry doesn't match, this means entry is for another instance. Log an error to console.
 func initFederatorBackend(objLayer ObjectLayer) {
-	// List all buckets
 	b, err := objLayer.ListBuckets(context.Background())
 	if err != nil {
 		logger.LogIf(context.Background(), err)
@@ -60,15 +64,14 @@ func initFederatorBackend(objLayer ObjectLayer) {
 		g.Go(func() error {
 			r, gerr := globalDNSConfig.Get(b[index].Name)
 			if gerr != nil {
-				if client.IsKeyNotFound(gerr) {
-					// Make a new entry
+				if client.IsKeyNotFound(gerr) || gerr == dns.ErrNoEntriesFound {
 					return globalDNSConfig.Put(b[index].Name)
 				}
 				return gerr
 			}
-			if r.Host != globalDomainIP {
-				// Log error that entry already present for different host
-				return fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use %s to access the bucket, or rename it to a unique value", b[index].Name, globalDomainIP)
+			if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+				// There is already an entry for this bucket, with all IP addresses different. This indicates a bucket name collision. Log an error and continue.
+				return fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", b[index].Name, globalDomainIPs.ToSlice())
 			}
 			return nil
 		}, index)
@@ -418,7 +421,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 
 	if globalDNSConfig != nil {
 		if _, err := globalDNSConfig.Get(bucket); err != nil {
-			if client.IsKeyNotFound(err) {
+			if client.IsKeyNotFound(err) || err == dns.ErrNoEntriesFound {
 				// Proceed to creating a bucket.
 				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
 					writeErrorResponse(w, toAPIErrorCode(err), r.URL)

cmd/common-main.go

@@ -28,10 +28,13 @@ import (
 	"time"
 
 	etcd "github.com/coreos/etcd/client"
+
 	"github.com/minio/cli"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/dns"
+
+	"github.com/minio/minio-go/pkg/set"
 )
 
 // Check for updates and print a notification message
@@ -59,7 +62,7 @@ func initConfig() {
 		} else {
 			if etcd.IsKeyNotFound(err) {
 				logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time.")
-				logger.Info("Created minio configuration file successfully at", globalEtcdClient.Endpoints())
+				logger.Info("Created minio configuration file successfully at %v", globalEtcdClient.Endpoints())
 			} else {
 				logger.FatalIf(err, "Unable to load config version: '%s'.", serverConfigVersion)
 			}
@@ -168,10 +171,20 @@ func handleCommonEnvVars() {
 		logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
 	}
 
-	globalDomainIP = os.Getenv("MINIO_PUBLIC_IP")
-	if globalDomainName != "" && globalDomainIP != "" && globalEtcdClient != nil {
+	minioEndpointsEnv, ok := os.LookupEnv("MINIO_PUBLIC_IPS")
+	if ok {
+		minioEndpoints := strings.Split(minioEndpointsEnv, ",")
+		globalDomainIPs = set.NewStringSet()
+		for i, ip := range minioEndpoints {
+			if net.ParseIP(ip) == nil {
+				logger.FatalIf(errInvalidArgument, "Unable to initialize Minio server with invalid MINIO_PUBLIC_IPS[%d]: %s", i, ip)
+			}
+			globalDomainIPs.Add(ip)
+		}
+	}
+	if globalDomainName != "" && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
 		var err error
-		globalDNSConfig, err = dns.NewCoreDNS(globalDomainName, globalDomainIP, globalMinioPort, globalEtcdClient)
+		globalDNSConfig, err = dns.NewCoreDNS(globalDomainName, globalDomainIPs, globalMinioPort, globalEtcdClient)
 		logger.FatalIf(err, "Unable to initialize DNS config for %s.", globalDomainName)
 	}
 

cmd/generic-handlers.go

@@ -26,9 +26,12 @@ import (
 	"strings"
 	"time"
 
+	"github.com/minio/minio-go/pkg/set"
+
 	"github.com/coreos/etcd/client"
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/sys"
 	"github.com/rs/cors"
@@ -629,29 +632,37 @@ func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 		return
 	}
 	bucket, object := urlPath2BucketObjectName(r.URL.Path)
-	// MakeBucket request
+	// MakeBucket requests should be handled at current endpoint
 	if r.Method == http.MethodPut && bucket != "" && object == "" {
 		f.handler.ServeHTTP(w, r)
 		return
 	}
-	// ListBucket request
+	// ListBucket requests should be handled at current endpoint as
+	// all buckets data can be fetched from here.
 	if r.Method == http.MethodGet && bucket == "" && object == "" {
 		f.handler.ServeHTTP(w, r)
 		return
 	}
+	// CopyObject requests should be handled at current endpoint as path style
+	// requests have target bucket and object in URI and source details are in
+	// header fields
+	if r.Method == http.MethodPut && r.Header.Get("X-Amz-Copy-Source") != "" {
+		f.handler.ServeHTTP(w, r)
+		return
+	}
 	sr, err := globalDNSConfig.Get(bucket)
 	if err != nil {
-		if client.IsKeyNotFound(err) {
+		if client.IsKeyNotFound(err) || err == dns.ErrNoEntriesFound {
 			writeErrorResponse(w, ErrNoSuchBucket, r.URL)
 		} else {
 			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		}
 		return
 	}
-	if sr.Host != globalDomainIP {
-		backendURL := fmt.Sprintf("http://%s:%d", sr.Host, sr.Port)
+	if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(sr)...)).IsEmpty() {
+		backendURL := fmt.Sprintf("http://%s:%d", sr[0].Host, sr[0].Port)
 		if globalIsSSL {
-			backendURL = fmt.Sprintf("https://%s:%d", sr.Host, sr.Port)
+			backendURL = fmt.Sprintf("https://%s:%d", sr[0].Host, sr[0].Port)
 		}
 		r.URL, err = url.Parse(backendURL)
 		if err != nil {

cmd/globals.go

@@ -22,6 +22,8 @@ import (
 	"runtime"
 	"time"
 
+	"github.com/minio/minio-go/pkg/set"
+
 	etcd "github.com/coreos/etcd/client"
 	humanize "github.com/dustin/go-humanize"
 	"github.com/fatih/color"
@@ -161,8 +163,8 @@ var (
 	globalPublicCerts []*x509.Certificate
 
 	globalIsEnvDomainName bool
-	globalDomainName      string // Root domain for virtual host style requests
-	globalDomainIP        string // Root domain IP address
+	globalDomainName      string        // Root domain for virtual host style requests
+	globalDomainIPs       set.StringSet // Root domain IP address(s) for a distributed Minio deployment
 
 	globalListingTimeout   = newDynamicTimeout( /*30*/ 600*time.Second /*5*/, 600*time.Second) // timeout for listing related ops
 	globalObjectTimeout    = newDynamicTimeout( /*1*/ 10*time.Minute /*10*/, 600*time.Second)  // timeout for Object API related ops

cmd/object-api-utils.go

@@ -20,12 +20,16 @@ import (
 	"context"
 	"encoding/hex"
 	"fmt"
+	"net"
 	"path"
 	"runtime"
+	"strconv"
 	"strings"
 	"unicode/utf8"
 
+	miniogo "github.com/minio/minio-go"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/dns"
 	"github.com/skyrings/skyring-common/tools/uuid"
 )
 
@@ -277,6 +281,40 @@ func isMinioReservedBucket(bucketName string) bool {
 	return bucketName == minioReservedBucket
 }
 
+// Returns a minio-go Client configured to access remote host described by destDNSRecord
+// Applicable only in a federated deployment
+func getRemoteInstanceClient(destDNSRecord dns.SrvRecord) (*miniogo.Core, error) {
+	// In a federated deployment, all the instances share config files and hence expected to have same
+	// credentials. So, access current instances creds and use it to create client for remote instance
+	client, err := miniogo.NewCore(net.JoinHostPort(destDNSRecord.Host, strconv.Itoa(destDNSRecord.Port)), globalServerConfig.Credential.AccessKey, globalServerConfig.Credential.SecretKey, globalIsSSL)
+	if err != nil {
+		return nil, err
+	}
+	return client, nil
+}
+
+// Checks if a remote putobject call is needed for CopyObject operation
+// 1. If source and destination bucket names are same, it means no call needed to etcd to get destination info
+// 2. If destination bucket doesn't exist locally, only then a etcd call is needed
+func isRemoteCallRequired(ctx context.Context, src, dst string, objAPI ObjectLayer) bool {
+	if src == dst {
+		return false
+	}
+	if _, err := objAPI.GetBucketInfo(ctx, dst); err == toObjectErr(errVolumeNotFound, dst) {
+		return true
+	}
+	return false
+}
+
+// returns a slice of hosts by reading a slice of DNS records
+func getHostsSlice(records []dns.SrvRecord) []string {
+	var hosts []string
+	for _, r := range records {
+		hosts = append(hosts, r.Host)
+	}
+	return hosts
+}
+
 // byBucketName is a collection satisfying sort.Interface.
 type byBucketName []BucketInfo
 

cmd/object-handlers.go

@@ -536,13 +536,51 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	// Copy source object to destination, if source and destination
-	// object is same then only metadata is updated.
-	objInfo, err := objectAPI.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo)
-	if err != nil {
-		pipeWriter.CloseWithError(err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
+	var objInfo ObjectInfo
+
+	// _, err = objectAPI.GetBucketInfo(ctx, dstBucket)
+	// if err == toObjectErr(errVolumeNotFound, dstBucket) && !cpSrcDstSame
+	if isRemoteCallRequired(ctx, srcBucket, dstBucket, objectAPI) {
+		if globalDNSConfig != nil {
+			if dstRecord, errEtcd := globalDNSConfig.Get(dstBucket); errEtcd == nil {
+				go func() {
+					if gerr := objectAPI.GetObject(ctx, srcBucket, srcObject, 0, srcInfo.Size, srcInfo.Writer, srcInfo.ETag); gerr != nil {
+						pipeWriter.CloseWithError(gerr)
+						writeErrorResponse(w, ErrInternalError, r.URL)
+						return
+					}
+					// Close writer explicitly to indicate data has been written
+					defer srcInfo.Writer.Close()
+				}()
+				// Send PutObject request to appropriate instance (in federated deployment)
+				client, rerr := getRemoteInstanceClient(dstRecord[0])
+				if rerr != nil {
+					pipeWriter.CloseWithError(rerr)
+					writeErrorResponse(w, ErrInternalError, r.URL)
+					return
+				}
+				remoteObjInfo, rerr := client.PutObject(dstBucket, dstObject, srcInfo.Reader, srcInfo.Size, "", "", srcInfo.UserDefined)
+				if rerr != nil {
+					pipeWriter.CloseWithError(rerr)
+					writeErrorResponse(w, ErrInternalError, r.URL)
+					return
+				}
+				objInfo.ETag = remoteObjInfo.ETag
+				objInfo.ModTime = remoteObjInfo.LastModified
+			}
+		} else {
+			writeErrorResponse(w, ErrNoSuchBucket, r.URL)
+			return
+		}
+	} else {
+		// Copy source object to destination, if source and destination
+		// object is same then only metadata is updated.
+		objInfo, err = objectAPI.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo)
+		if err != nil {
+			pipeWriter.CloseWithError(err)
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
 	}
 
 	pipeReader.Close()

cmd/server-main.go

@@ -91,7 +91,7 @@ ENVIRONMENT VARIABLES:
 
   BUCKET-DNS:
      MINIO_DOMAIN:    To enable bucket DNS requests, set this value to Minio host domain name.
-     MINIO_PUBLIC_IP: To enable bucket DNS requests, set this value to Minio host public IP.
+     MINIO_PUBLIC_IPS: To enable bucket DNS requests, set this value to list of Minio host public IP(s) delimited by ",".
      MINIO_ETCD_ENDPOINTS: To enable bucket DNS requests, set this value to list of etcd endpoints delimited by ",".
 
 EXAMPLES:

cmd/web-handlers.go

@@ -40,6 +40,7 @@ import (
 	"github.com/minio/minio/browser"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/policy"
@@ -138,7 +139,7 @@ func (web *webAPIHandlers) MakeBucket(r *http.Request, args *MakeBucketArgs, rep
 
 	if globalDNSConfig != nil {
 		if _, err := globalDNSConfig.Get(args.BucketName); err != nil {
-			if etcd.IsKeyNotFound(err) {
+			if etcd.IsKeyNotFound(err) || err == dns.ErrNoEntriesFound {
 				// Proceed to creating a bucket.
 				if err = objectAPI.MakeBucketWithLocation(context.Background(), args.BucketName, globalServerConfig.GetRegion()); err != nil {
 					return toJSONError(err)