From e7a724de0d3e30bdec81c98486145437b4ded0ff Mon Sep 17 00:00:00 2001 From: Krishna Srinivas Date: Tue, 14 Nov 2017 16:56:24 -0800 Subject: [PATCH] Virtual host style S3 requests (#5095) --- cmd/admin-handlers.go | 2 +- cmd/api-errors.go | 10 ++ cmd/api-router.go | 130 +++++++++--------- cmd/auth-handler.go | 6 +- cmd/bucket-handlers.go | 5 - cmd/common-main.go | 7 +- cmd/config-migrate.go | 115 +++++++++++++++- cmd/config-migrate_test.go | 2 +- cmd/config-old.go | 18 +++ cmd/{config-v19.go => config-v20.go} | 54 +++++--- ...{config-v19_test.go => config-v20_test.go} | 12 +- cmd/gateway-router.go | 124 +++++++++-------- cmd/generic-handlers.go | 5 - cmd/globals.go | 3 + cmd/handler-utils.go | 28 ++++ cmd/handler-utils_test.go | 23 ++++ cmd/signature-v2.go | 10 ++ docs/config/README.md | 14 ++ 18 files changed, 409 insertions(+), 159 deletions(-) rename cmd/{config-v19.go => config-v20.go} (86%) rename cmd/{config-v19_test.go => config-v20_test.go} (98%) diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index eae409186..27b7a8a06 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -985,7 +985,7 @@ func (adminAPI adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http return } - var config serverConfigV19 + var config serverConfigV20 err = json.Unmarshal(configBytes, &config) if err != nil { diff --git a/cmd/api-errors.go b/cmd/api-errors.go index 31c5494b3..ea4ed9c12 100644 --- a/cmd/api-errors.go +++ b/cmd/api-errors.go @@ -162,6 +162,7 @@ const ( ErrServerNotInitialized ErrOperationTimedOut ErrPartsSizeUnequal + ErrInvalidRequest // Add new extended error codes here. // Please open a https://github.com/minio/minio/issues before adding // new error codes here. @@ -731,6 +732,15 @@ var errorCodeResponse = map[APIErrorCode]APIError{ Description: "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds", HTTPStatusCode: http.StatusBadRequest, }, + // Generic Invalid-Request error. Should be used for response errors only for unlikely + // corner case errors for which introducing new APIErrorCode is not worth it. errorIf() + // should be used to log the error at the source of the error for debugging purposes. + ErrInvalidRequest: { + Code: "InvalidRequest", + Description: "Invalid Request", + HTTPStatusCode: http.StatusBadRequest, + }, + // Add your error structure here. } diff --git a/cmd/api-router.go b/cmd/api-router.go index 50844d201..c5ed69d54 100644 --- a/cmd/api-router.go +++ b/cmd/api-router.go @@ -16,9 +16,8 @@ package cmd -import ( - router "github.com/gorilla/mux" -) +import router "github.com/gorilla/mux" +import "net/http" // objectAPIHandler implements and provides http handlers for S3 API. type objectAPIHandlers struct { @@ -34,70 +33,75 @@ func registerAPIRouter(mux *router.Router) { // API Router apiRouter := mux.NewRoute().PathPrefix("/").Subrouter() + var routers []*router.Router + if globalDomainName != "" { + routers = append(routers, apiRouter.Host("{bucket:.+}."+globalDomainName).Subrouter()) + } + routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter()) - // Bucket router - bucket := apiRouter.PathPrefix("/{bucket}").Subrouter() + for _, bucket := range routers { + // Object operations + // HeadObject + bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler)) + // CopyObjectPart + bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") + // PutObjectPart + bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") + // ListObjectPxarts + bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}") + // CompleteMultipartUpload + bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}") + // NewMultipartUpload + bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "") + // AbortMultipartUpload + bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}") + // GetObject + bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler)) + // CopyObject + bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler)) + // PutObject + bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler)) + // DeleteObject + bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler)) - /// Object operations - - // HeadObject - bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler)) - // CopyObjectPart - bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") - // PutObjectPart - bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") - // ListObjectPxarts - bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}") - // CompleteMultipartUpload - bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}") - // NewMultipartUpload - bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "") - // AbortMultipartUpload - bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}") - // GetObject - bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler)) - // CopyObject - bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler)) - // PutObject - bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler)) - // DeleteObject - bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler)) - - /// Bucket operations - - // GetBucketLocation - bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "") - // GetBucketPolicy - bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "") - // GetBucketNotification - bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "") - // ListenBucketNotification - bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}") - // ListMultipartUploads - bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "") - // ListObjectsV2 - bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2") - // ListObjectsV1 (Legacy) - bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler)) - // PutBucketPolicy - bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "") - // PutBucketNotification - bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "") - // PutBucket - bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketHandler)) - // HeadBucket - bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler)) - // PostPolicy - bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceAll(api.PostPolicyBucketHandler)) - // DeleteMultipleObjects - bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)) - // DeleteBucketPolicy - bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "") - // DeleteBucket - bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketHandler)) + /// Bucket operations + // GetBucketLocation + bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "") + // GetBucketPolicy + bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "") + // GetBucketNotification + bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "") + // ListenBucketNotification + bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}") + // ListMultipartUploads + bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "") + // ListObjectsV2 + bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2") + // ListObjectsV1 (Legacy) + bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler)) + // PutBucketPolicy + bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "") + // PutBucketNotification + bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "") + // PutBucket + bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketHandler)) + // HeadBucket + bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler)) + // PostPolicy + bucket.Methods("POST").Path("/").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceAll(api.PostPolicyBucketHandler)) + // DeleteMultipleObjects + bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "") + // DeleteBucketPolicy + bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "") + // DeleteBucket + bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketHandler)) + } /// Root operation // ListBuckets - apiRouter.Methods("GET").HandlerFunc(httpTraceAll(api.ListBucketsHandler)) + apiRouter.Methods("GET").Path("/").HandlerFunc(httpTraceAll(api.ListBucketsHandler)) + + // If none of the routes match. + apiRouter.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler)) } diff --git a/cmd/auth-handler.go b/cmd/auth-handler.go index ae4a27975..a042942e1 100644 --- a/cmd/auth-handler.go +++ b/cmd/auth-handler.go @@ -127,7 +127,11 @@ func checkRequestAuthType(r *http.Request, bucket, policyAction, region string) if reqAuthType == authTypeAnonymous && policyAction != "" { // http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html sourceIP := getSourceIPAddress(r) - return enforceBucketPolicy(bucket, policyAction, r.URL.Path, + resource, err := getResource(r.URL.Path, r.Host, globalDomainName) + if err != nil { + return ErrInternalError + } + return enforceBucketPolicy(bucket, policyAction, resource, r.Referer(), sourceIP, r.URL.Query()) } diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index b150f32bf..627f9f947 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -24,7 +24,6 @@ import ( "net/http" "net/url" "path" - "path/filepath" "reflect" "strings" "sync" @@ -441,10 +440,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h // Make sure that the URL does not contain object name. bucket := mux.Vars(r)["bucket"] - if bucket != filepath.Clean(r.URL.Path[1:]) { - writeErrorResponse(w, ErrMethodNotAllowed, r.URL) - return - } // Require Content-Length to be set in the request size := r.ContentLength diff --git a/cmd/common-main.go b/cmd/common-main.go index c3f6fc1fb..9717da555 100644 --- a/cmd/common-main.go +++ b/cmd/common-main.go @@ -57,7 +57,7 @@ func initConfig() { // Config file does not exist, we create it fresh and return upon success. if isFile(getConfigFile()) { fatalIf(migrateConfig(), "Config migration failed.") - fatalIf(loadConfig(), "Unable to load config version: '%s'.", v19) + fatalIf(loadConfig(), "Unable to load config version: '%s'.", v20) } else { fatalIf(newConfig(), "Unable to initialize minio config for the first time.") log.Println("Created minio configuration file successfully at " + getConfigDir()) @@ -117,4 +117,9 @@ func handleCommonEnvVars() { } globalHTTPTrace = os.Getenv("MINIO_HTTP_TRACE") != "" + + globalDomainName = os.Getenv("MINIO_DOMAIN") + if globalDomainName != "" { + globalIsEnvDomainName = true + } } diff --git a/cmd/config-migrate.go b/cmd/config-migrate.go index bda24047a..947b1438b 100644 --- a/cmd/config-migrate.go +++ b/cmd/config-migrate.go @@ -148,7 +148,12 @@ func migrateConfig() error { return err } fallthrough - case v19: + case "19": + if err = migrateV19ToV20(); err != nil { + return err + } + fallthrough + case "20": // No migration needed. this always points to current version. err = nil } @@ -1479,3 +1484,111 @@ func migrateV18ToV19() error { log.Printf(configMigrateMSGTemplate, configFile, cv18.Version, srvConfig.Version) return nil } + +func migrateV19ToV20() error { + configFile := getConfigFile() + + cv19 := &serverConfigV19{} + _, err := quick.Load(configFile, cv19) + if os.IsNotExist(err) { + return nil + } else if err != nil { + return fmt.Errorf("Unable to load config version ‘18’. %v", err) + } + if cv19.Version != "19" { + return nil + } + + // Copy over fields from V19 into V20 config struct + srvConfig := &serverConfigV20{ + Logger: &loggers{}, + Notify: ¬ifier{}, + } + srvConfig.Version = "20" + srvConfig.Credential = cv19.Credential + srvConfig.Region = cv19.Region + if srvConfig.Region == "" { + // Region needs to be set for AWS Signature Version 4. + srvConfig.Region = globalMinioDefaultRegion + } + + srvConfig.Logger.Console = cv19.Logger.Console + srvConfig.Logger.File = cv19.Logger.File + + // check and set notifiers config + if len(cv19.Notify.AMQP) == 0 { + srvConfig.Notify.AMQP = make(map[string]amqpNotify) + srvConfig.Notify.AMQP["1"] = amqpNotify{} + } else { + // New deliveryMode parameter is added for AMQP, + // default value is already 0, so nothing to + // explicitly migrate here. + srvConfig.Notify.AMQP = cv19.Notify.AMQP + } + if len(cv19.Notify.ElasticSearch) == 0 { + srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify) + srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{ + Format: formatNamespace, + } + } else { + srvConfig.Notify.ElasticSearch = cv19.Notify.ElasticSearch + } + if len(cv19.Notify.Redis) == 0 { + srvConfig.Notify.Redis = make(map[string]redisNotify) + srvConfig.Notify.Redis["1"] = redisNotify{ + Format: formatNamespace, + } + } else { + srvConfig.Notify.Redis = cv19.Notify.Redis + } + if len(cv19.Notify.PostgreSQL) == 0 { + srvConfig.Notify.PostgreSQL = make(map[string]postgreSQLNotify) + srvConfig.Notify.PostgreSQL["1"] = postgreSQLNotify{ + Format: formatNamespace, + } + } else { + srvConfig.Notify.PostgreSQL = cv19.Notify.PostgreSQL + } + if len(cv19.Notify.Kafka) == 0 { + srvConfig.Notify.Kafka = make(map[string]kafkaNotify) + srvConfig.Notify.Kafka["1"] = kafkaNotify{} + } else { + srvConfig.Notify.Kafka = cv19.Notify.Kafka + } + if len(cv19.Notify.NATS) == 0 { + srvConfig.Notify.NATS = make(map[string]natsNotify) + srvConfig.Notify.NATS["1"] = natsNotify{} + } else { + srvConfig.Notify.NATS = cv19.Notify.NATS + } + if len(cv19.Notify.Webhook) == 0 { + srvConfig.Notify.Webhook = make(map[string]webhookNotify) + srvConfig.Notify.Webhook["1"] = webhookNotify{} + } else { + srvConfig.Notify.Webhook = cv19.Notify.Webhook + } + if len(cv19.Notify.MySQL) == 0 { + srvConfig.Notify.MySQL = make(map[string]mySQLNotify) + srvConfig.Notify.MySQL["1"] = mySQLNotify{ + Format: formatNamespace, + } + } else { + srvConfig.Notify.MySQL = cv19.Notify.MySQL + } + if len(cv19.Notify.MQTT) == 0 { + srvConfig.Notify.MQTT = make(map[string]mqttNotify) + srvConfig.Notify.MQTT["1"] = mqttNotify{} + } else { + srvConfig.Notify.MQTT = cv19.Notify.MQTT + } + + // Load browser config from existing config in the file. + srvConfig.Browser = cv19.Browser + + if err = quick.Save(configFile, srvConfig); err != nil { + return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv19.Version, srvConfig.Version, err) + } + + log.Printf(configMigrateMSGTemplate, configFile, cv19.Version, srvConfig.Version) + return nil +} diff --git a/cmd/config-migrate_test.go b/cmd/config-migrate_test.go index 2a8ffdff5..1f0004874 100644 --- a/cmd/config-migrate_test.go +++ b/cmd/config-migrate_test.go @@ -169,7 +169,7 @@ func TestServerConfigMigrateV2toV19(t *testing.T) { } // Check the version number in the upgraded config file - expectedVersion := v19 + expectedVersion := v20 if serverConfig.Version != expectedVersion { t.Fatalf("Expect version "+expectedVersion+", found: %v", serverConfig.Version) } diff --git a/cmd/config-old.go b/cmd/config-old.go index e2919c9d9..53abaee3f 100644 --- a/cmd/config-old.go +++ b/cmd/config-old.go @@ -472,3 +472,21 @@ type serverConfigV18 struct { // Notification queue configuration. Notify *notifier `json:"notify"` } + +// serverConfigV19 server configuration version '19' which is like +// version '18' except it adds support for MQTT notifications. +type serverConfigV19 struct { + sync.RWMutex + Version string `json:"version"` + + // S3 API configuration. + Credential auth.Credentials `json:"credential"` + Region string `json:"region"` + Browser BrowserFlag `json:"browser"` + + // Additional error logging configuration. + Logger *loggers `json:"logger"` + + // Notification queue configuration. + Notify *notifier `json:"notify"` +} diff --git a/cmd/config-v19.go b/cmd/config-v20.go similarity index 86% rename from cmd/config-v19.go rename to cmd/config-v20.go index c81e836c1..c8aaa112a 100644 --- a/cmd/config-v19.go +++ b/cmd/config-v20.go @@ -28,17 +28,17 @@ import ( ) // Config version -const v19 = "19" +const v20 = "20" var ( // serverConfig server config. - serverConfig *serverConfigV19 + serverConfig *serverConfigV20 serverConfigMu sync.RWMutex ) -// serverConfigV19 server configuration version '19' which is like -// version '18' except it adds support for MQTT notifications. -type serverConfigV19 struct { +// serverConfigV20 server configuration version '20' which is like +// version '19' except it adds support for VirtualHostDomain +type serverConfigV20 struct { sync.RWMutex Version string `json:"version"` @@ -46,6 +46,7 @@ type serverConfigV19 struct { Credential auth.Credentials `json:"credential"` Region string `json:"region"` Browser BrowserFlag `json:"browser"` + Domain string `json:"domain"` // Additional error logging configuration. Logger *loggers `json:"logger"` @@ -55,7 +56,7 @@ type serverConfigV19 struct { } // GetVersion get current config version. -func (s *serverConfigV19) GetVersion() string { +func (s *serverConfigV20) GetVersion() string { s.RLock() defer s.RUnlock() @@ -63,7 +64,7 @@ func (s *serverConfigV19) GetVersion() string { } // SetRegion set a new region. -func (s *serverConfigV19) SetRegion(region string) { +func (s *serverConfigV20) SetRegion(region string) { s.Lock() defer s.Unlock() @@ -72,7 +73,7 @@ func (s *serverConfigV19) SetRegion(region string) { } // GetRegion get current region. -func (s *serverConfigV19) GetRegion() string { +func (s *serverConfigV20) GetRegion() string { s.RLock() defer s.RUnlock() @@ -80,7 +81,7 @@ func (s *serverConfigV19) GetRegion() string { } // SetCredentials set new credentials. SetCredential returns the previous credential. -func (s *serverConfigV19) SetCredential(creds auth.Credentials) (prevCred auth.Credentials) { +func (s *serverConfigV20) SetCredential(creds auth.Credentials) (prevCred auth.Credentials) { s.Lock() defer s.Unlock() @@ -95,7 +96,7 @@ func (s *serverConfigV19) SetCredential(creds auth.Credentials) (prevCred auth.C } // GetCredentials get current credentials. -func (s *serverConfigV19) GetCredential() auth.Credentials { +func (s *serverConfigV20) GetCredential() auth.Credentials { s.RLock() defer s.RUnlock() @@ -103,7 +104,7 @@ func (s *serverConfigV19) GetCredential() auth.Credentials { } // SetBrowser set if browser is enabled. -func (s *serverConfigV19) SetBrowser(b bool) { +func (s *serverConfigV20) SetBrowser(b bool) { s.Lock() defer s.Unlock() @@ -112,7 +113,7 @@ func (s *serverConfigV19) SetBrowser(b bool) { } // GetCredentials get current credentials. -func (s *serverConfigV19) GetBrowser() bool { +func (s *serverConfigV20) GetBrowser() bool { s.RLock() defer s.RUnlock() @@ -120,7 +121,7 @@ func (s *serverConfigV19) GetBrowser() bool { } // Save config. -func (s *serverConfigV19) Save() error { +func (s *serverConfigV20) Save() error { s.RLock() defer s.RUnlock() @@ -128,9 +129,9 @@ func (s *serverConfigV19) Save() error { return quick.Save(getConfigFile(), s) } -func newServerConfigV19() *serverConfigV19 { - srvCfg := &serverConfigV19{ - Version: v19, +func newServerConfigV20() *serverConfigV20 { + srvCfg := &serverConfigV20{ + Version: v20, Credential: auth.MustGetNewCredentials(), Region: globalMinioDefaultRegion, Browser: true, @@ -168,7 +169,7 @@ func newServerConfigV19() *serverConfigV19 { // found, otherwise use default parameters func newConfig() error { // Initialize server config. - srvCfg := newServerConfigV19() + srvCfg := newServerConfigV20() // If env is set override the credentials from config file. if globalIsEnvCreds { @@ -183,6 +184,10 @@ func newConfig() error { srvCfg.SetRegion(globalServerRegion) } + if globalIsEnvDomainName { + srvCfg.Domain = globalDomainName + } + // hold the mutex lock before a new config is assigned. // Save the new config globally. // unlock the mutex. @@ -246,8 +251,8 @@ func checkDupJSONKeys(json string) error { } // getValidConfig - returns valid server configuration -func getValidConfig() (*serverConfigV19, error) { - srvCfg := &serverConfigV19{ +func getValidConfig() (*serverConfigV20, error) { + srvCfg := &serverConfigV20{ Region: globalMinioDefaultRegion, Browser: true, } @@ -257,8 +262,8 @@ func getValidConfig() (*serverConfigV19, error) { return nil, err } - if srvCfg.Version != v19 { - return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", v19, srvCfg.Version) + if srvCfg.Version != v20 { + return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", v20, srvCfg.Version) } // Load config file json and check for duplication json keys @@ -312,6 +317,10 @@ func loadConfig() error { srvCfg.SetRegion(globalServerRegion) } + if globalIsEnvDomainName { + srvCfg.Domain = globalDomainName + } + // hold the mutex lock before a new config is assigned. serverConfigMu.Lock() serverConfig = srvCfg @@ -324,6 +333,9 @@ func loadConfig() error { if !globalIsEnvRegion { globalServerRegion = serverConfig.GetRegion() } + if !globalIsEnvDomainName { + globalDomainName = serverConfig.Domain + } serverConfigMu.Unlock() return nil diff --git a/cmd/config-v19_test.go b/cmd/config-v20_test.go similarity index 98% rename from cmd/config-v19_test.go rename to cmd/config-v20_test.go index d19a6739a..c0bc1347a 100644 --- a/cmd/config-v19_test.go +++ b/cmd/config-v20_test.go @@ -117,8 +117,8 @@ func TestServerConfig(t *testing.T) { serverConfig.Logger.SetFile(fileLogger) // Match version. - if serverConfig.GetVersion() != v19 { - t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v19) + if serverConfig.GetVersion() != v20 { + t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v20) } // Attempt to save. @@ -149,6 +149,9 @@ func TestServerConfigWithEnvs(t *testing.T) { os.Setenv("MINIO_REGION", "us-west-1") defer os.Unsetenv("MINIO_REGION") + os.Setenv("MINIO_DOMAIN", "domain.com") + defer os.Unsetenv("MINIO_DOMAIN") + defer resetGlobalIsEnvs() // Get test root. @@ -189,6 +192,9 @@ func TestServerConfigWithEnvs(t *testing.T) { t.Errorf("Expecting access key to be `minio123` found %s", cred.SecretKey) } + if serverConfig.Domain != "domain.com" { + t.Errorf("Expecting Domain to be `domain.com` found " + serverConfig.Domain) + } } func TestCheckDupJSONKeys(t *testing.T) { @@ -231,7 +237,7 @@ func TestValidateConfig(t *testing.T) { configPath := filepath.Join(rootPath, minioConfigFile) - v := v19 + v := v20 testCases := []struct { configData string diff --git a/cmd/gateway-router.go b/cmd/gateway-router.go index 35ff7da4e..cd3221772 100644 --- a/cmd/gateway-router.go +++ b/cmd/gateway-router.go @@ -18,6 +18,7 @@ package cmd import ( "io" + "net/http" router "github.com/gorilla/mux" "github.com/minio/minio-go/pkg/policy" @@ -62,69 +63,78 @@ func registerGatewayAPIRouter(mux *router.Router, gw GatewayLayer) { // API Router apiRouter := mux.NewRoute().PathPrefix("/").Subrouter() - // Bucket router - bucket := apiRouter.PathPrefix("/{bucket}").Subrouter() + var routers []*router.Router + if globalDomainName != "" { + routers = append(routers, apiRouter.Host("{bucket:.+}."+globalDomainName).Subrouter()) + } + routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter()) - /// Object operations + // Object operations + for _, bucket := range routers { + /// Object operations - // HeadObject - bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler)) - // CopyObjectPart - bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") - // PutObjectPart - bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") - // ListObjectPxarts - bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}") - // CompleteMultipartUpload - bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}") - // NewMultipartUpload - bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "") - // AbortMultipartUpload - bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}") - // GetObject - bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler)) - // CopyObject - bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler)) - // PutObject - bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler)) - // DeleteObject - bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler)) + // HeadObject + bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler)) + // CopyObjectPart + bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") + // PutObjectPart + bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") + // ListObjectPxarts + bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}") + // CompleteMultipartUpload + bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}") + // NewMultipartUpload + bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "") + // AbortMultipartUpload + bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}") + // GetObject + bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler)) + // CopyObject + bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler)) + // PutObject + bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler)) + // DeleteObject + bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler)) - /// Bucket operations + /// Bucket operations - // GetBucketLocation - bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "") - // GetBucketPolicy - bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "") - // GetBucketNotification - bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "") - // ListenBucketNotification - bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}") - // ListMultipartUploads - bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "") - // ListObjectsV2 - bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2") - // ListObjectsV1 (Legacy) - bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler)) - // PutBucketPolicy - bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "") - // PutBucketNotification - bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "") - // PutBucket - bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketHandler)) - // HeadBucket - bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler)) - // PostPolicy - bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceAll(api.PostPolicyBucketHandler)) - // DeleteMultipleObjects - bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)) - // DeleteBucketPolicy - bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "") - // DeleteBucket - bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketHandler)) + // GetBucketLocation + bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "") + // GetBucketPolicy + bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "") + // GetBucketNotification + bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "") + // ListenBucketNotification + bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}") + // ListMultipartUploads + bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "") + // ListObjectsV2 + bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2") + // ListObjectsV1 (Legacy) + bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler)) + // PutBucketPolicy + bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "") + // PutBucketNotification + bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "") + // PutBucket + bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketHandler)) + // HeadBucket + bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler)) + // PostPolicy + bucket.Methods("POST").Path("/").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceAll(api.PostPolicyBucketHandler)) + // DeleteMultipleObjects + bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "") + // DeleteBucketPolicy + bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "") + // DeleteBucket + bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketHandler)) + } /// Root operation // ListBuckets - apiRouter.Methods("GET").HandlerFunc(httpTraceAll(api.ListBucketsHandler)) + apiRouter.Methods("GET").Path("/").HandlerFunc(httpTraceAll(api.ListBucketsHandler)) + + // If none of the routes match. + apiRouter.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler)) } diff --git a/cmd/generic-handlers.go b/cmd/generic-handlers.go index 97f19ad4a..647fb68f1 100644 --- a/cmd/generic-handlers.go +++ b/cmd/generic-handlers.go @@ -440,11 +440,6 @@ func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } } - // A put method on path "/" doesn't make sense, ignore it. - if r.Method == httpPUT && r.URL.Path == "/" && r.Header.Get(minioAdminOpHeader) == "" { - writeErrorResponse(w, ErrNotImplemented, r.URL) - return - } // Serve HTTP. h.handler.ServeHTTP(w, r) diff --git a/cmd/globals.go b/cmd/globals.go index d38eda338..f0cd3c65f 100644 --- a/cmd/globals.go +++ b/cmd/globals.go @@ -146,6 +146,9 @@ var ( globalActiveCred auth.Credentials globalPublicCerts []*x509.Certificate globalXLObjCacheDisabled bool + + globalIsEnvDomainName bool + globalDomainName string // Root domain for virtual host style requests // Add new variable global values here. globalListingTimeout = newDynamicTimeout( /*30*/ 600*time.Second /*5*/, 600*time.Second) // timeout for listing related ops diff --git a/cmd/handler-utils.go b/cmd/handler-utils.go index 175291704..84d5b4c95 100644 --- a/cmd/handler-utils.go +++ b/cmd/handler-utils.go @@ -19,6 +19,7 @@ package cmd import ( "io" "mime/multipart" + "net" "net/http" "net/url" "os" @@ -259,3 +260,30 @@ func httpTraceHdrs(f http.HandlerFunc) http.HandlerFunc { } return httptracer.TraceReqHandlerFunc(f, os.Stdout, false) } + +// Returns "/bucketName/objectName" for path-style or virtual-host-style requests. +func getResource(path string, host string, domain string) (string, error) { + if domain == "" { + return path, nil + } + // If virtual-host-style is enabled construct the "resource" properly. + if strings.Contains(host, ":") { + // In bucket.mydomain.com:9000, strip out :9000 + var err error + if host, _, err = net.SplitHostPort(host); err != nil { + errorIf(err, "Unable to split %s", host) + return "", err + } + } + if !strings.HasSuffix(host, "."+domain) { + return path, nil + } + bucket := strings.TrimSuffix(host, "."+domain) + return slashSeparator + pathJoin(bucket, path), nil +} + +// If none of the http routes match respond with MethodNotAllowed +func notFoundHandler(w http.ResponseWriter, r *http.Request) { + writeErrorResponse(w, ErrMethodNotAllowed, r.URL) + return +} diff --git a/cmd/handler-utils_test.go b/cmd/handler-utils_test.go index 10576b989..8110cd25c 100644 --- a/cmd/handler-utils_test.go +++ b/cmd/handler-utils_test.go @@ -190,3 +190,26 @@ func TestExtractMetadataHeaders(t *testing.T) { } } } + +// Test getResource() +func TestGetResource(t *testing.T) { + testCases := []struct { + p string + host string + domain string + expectedResource string + }{ + {"/a/b/c", "test.mydomain.com", "mydomain.com", "/test/a/b/c"}, + {"/a/b/c", "test.mydomain.com", "notmydomain.com", "/a/b/c"}, + {"/a/b/c", "test.mydomain.com", "", "/a/b/c"}, + } + for i, test := range testCases { + gotResource, err := getResource(test.p, test.host, test.domain) + if err != nil { + t.Fatal(err) + } + if gotResource != test.expectedResource { + t.Fatalf("test %d: expected %s got %s", i+1, test.expectedResource, gotResource) + } + } +} diff --git a/cmd/signature-v2.go b/cmd/signature-v2.go index 71250abc5..de17525ab 100644 --- a/cmd/signature-v2.go +++ b/cmd/signature-v2.go @@ -157,6 +157,11 @@ func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode { return ErrExpiredPresignRequest } + encodedResource, err = getResource(encodedResource, r.Host, globalDomainName) + if err != nil { + return ErrInvalidRequest + } + expectedSignature := preSignatureV2(r.Method, encodedResource, strings.Join(filteredQueries, "&"), r.Header, expires) if gotSignature != expectedSignature { return ErrSignatureDoesNotMatch @@ -237,6 +242,11 @@ func doesSignV2Match(r *http.Request) APIErrorCode { return ErrInvalidQueryParams } + encodedResource, err = getResource(encodedResource, r.Host, globalDomainName) + if err != nil { + return ErrInvalidRequest + } + expectedAuth := signatureV2(r.Method, encodedResource, strings.Join(unescapedQueries, "&"), r.Header) if v2Auth != expectedAuth { return ErrSignatureDoesNotMatch diff --git a/docs/config/README.md b/docs/config/README.md index 34be115cb..6aa40a376 100644 --- a/docs/config/README.md +++ b/docs/config/README.md @@ -69,6 +69,20 @@ export MINIO_BROWSER=off minio server /data ``` +### Domain +|Field|Type|Description| +|:---|:---|:---| +|``domain``| _string_ | Enable virtual-host-style requests i.e http://bucket.mydomain.com/object| + +By default, Minio supports path-style requests which look like http://mydomain.com/bucket/object. MINIO_DOMAIN environmental varialble (or `domain` in config.json) can be used to enable virtual-host-style requests. If the request `Host` header matches with `(.+).mydomain.com` then the mattched pattern `$1` is used as bucket and the path is used as object. More information on path-style and virtual-host-style [here](http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAPI.html) + +Example: + +```sh +export MINIO_DOMAIN=mydomain.com +minio server /data +``` + #### Logger |Field|Type|Description| |:---|:---|:---|