From f99f2189999e7a51e5efcb89e18a7f17781ed34a Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 9 Jun 2017 19:50:51 -0700 Subject: [PATCH] Add support for reading and saving config on Gateway. (#4463) This is also a first step towards supporting bucket notification for gateway. --- cmd/common-main.go | 117 +++++++++++++++++ cmd/gateway-azure.go | 25 ++-- cmd/gateway-gcs-errors.go | 6 +- cmd/gateway-gcs.go | 23 +--- cmd/gateway-handlers.go | 140 +++++++------------- cmd/gateway-main.go | 222 ++++++++++++-------------------- cmd/gateway-main_test.go | 23 ---- cmd/gateway-s3.go | 32 ++--- cmd/gateway-startup-msg.go | 31 ++--- cmd/gateway-startup-msg_test.go | 20 ++- cmd/server-main.go | 96 ++------------ cmd/server-startup-msg.go | 8 +- cmd/server-startup-msg_test.go | 2 +- 13 files changed, 325 insertions(+), 420 deletions(-) create mode 100644 cmd/common-main.go diff --git a/cmd/common-main.go b/cmd/common-main.go new file mode 100644 index 000000000..c9041f8f4 --- /dev/null +++ b/cmd/common-main.go @@ -0,0 +1,117 @@ +/* + * Minio Cloud Storage, (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "errors" + "os" + "path/filepath" + "strings" + "time" + + "github.com/minio/cli" +) + +// Check for updates and print a notification message +func checkUpdate(mode string) { + // Its OK to ignore any errors during getUpdateInfo() here. + if older, downloadURL, err := getUpdateInfo(1*time.Second, mode); err == nil { + if updateMsg := computeUpdateMessage(downloadURL, older); updateMsg != "" { + log.Println(updateMsg) + } + } +} + +func enableLoggers() { + fileLogTarget := serverConfig.Logger.GetFile() + if fileLogTarget.Enable { + err := InitFileLogger(&fileLogTarget) + fatalIf(err, "Unable to initialize file logger") + log.AddTarget(fileLogTarget) + } + + consoleLogTarget := serverConfig.Logger.GetConsole() + if consoleLogTarget.Enable { + InitConsoleLogger(&consoleLogTarget) + } + + log.SetConsoleTarget(consoleLogTarget) +} + +func initConfig() { + // Config file does not exist, we create it fresh and return upon success. + if isFile(getConfigFile()) { + fatalIf(migrateConfig(), "Config migration failed.") + fatalIf(loadConfig(), "Unable to load config version: '%s'.", v19) + } else { + fatalIf(newConfig(), "Unable to initialize minio config for the first time.") + log.Println("Created minio configuration file successfully at " + getConfigDir()) + } +} + +func handleCommonCmdArgs(ctx *cli.Context) { + // Set configuration directory. + { + // Get configuration directory from command line argument. + configDir := ctx.String("config-dir") + if !ctx.IsSet("config-dir") && ctx.GlobalIsSet("config-dir") { + configDir = ctx.GlobalString("config-dir") + } + if configDir == "" { + fatalIf(errors.New("empty directory"), "Configuration directory cannot be empty.") + } + + // Disallow relative paths, figure out absolute paths. + configDirAbs, err := filepath.Abs(configDir) + fatalIf(err, "Unable to fetch absolute path for config directory %s", configDir) + + setConfigDir(configDirAbs) + } +} + +func handleCommonEnvVars() { + // Start profiler if env is set. + if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" { + globalProfiler = startProfiler(profiler) + } + + // Check if object cache is disabled. + globalXLObjCacheDisabled = strings.EqualFold(os.Getenv("_MINIO_CACHE"), "off") + + accessKey := os.Getenv("MINIO_ACCESS_KEY") + secretKey := os.Getenv("MINIO_SECRET_KEY") + if accessKey != "" && secretKey != "" { + cred, err := createCredential(accessKey, secretKey) + fatalIf(err, "Invalid access/secret Key set in environment.") + + // credential Envs are set globally. + globalIsEnvCreds = true + globalActiveCred = cred + } + + if browser := os.Getenv("MINIO_BROWSER"); browser != "" { + browserFlag, err := ParseBrowserFlag(browser) + if err != nil { + fatalIf(errors.New("invalid value"), "Unknown value ‘%s’ in MINIO_BROWSER environment variable.", browser) + } + + // browser Envs are set globally, this does not represent + // if browser is turned off or on. + globalIsEnvBrowser = true + globalIsBrowserEnabled = bool(browserFlag) + } +} diff --git a/cmd/gateway-azure.go b/cmd/gateway-azure.go index b0434f101..2fa5664c5 100644 --- a/cmd/gateway-azure.go +++ b/cmd/gateway-azure.go @@ -26,14 +26,12 @@ import ( "io" "net/http" "net/url" - "os" "strconv" "strings" "sync" "time" "github.com/Azure/azure-sdk-for-go/storage" - "github.com/minio/cli" "github.com/minio/minio-go/pkg/policy" "github.com/minio/sha256-simd" ) @@ -157,32 +155,30 @@ func azureToObjectError(err error, params ...string) error { } // Inits azure blob storage client and returns AzureObjects. -func newAzureLayer(args cli.Args) (GatewayLayer, error) { +func newAzureLayer(host string) (GatewayLayer, error) { var err error - - // Default endpoint parameters - endPoint := storage.DefaultBaseURL - secure := true + var endpoint = storage.DefaultBaseURL + var secure = true // If user provided some parameters - if args.Present() { - endPoint, secure, err = parseGatewayEndpoint(args.First()) + if host != "" { + endpoint, secure, err = parseGatewayEndpoint(host) if err != nil { return nil, err } } - account := os.Getenv("MINIO_ACCESS_KEY") - key := os.Getenv("MINIO_SECRET_KEY") - if account == "" || key == "" { - return nil, errors.New("No Azure account and key set") + creds := serverConfig.GetCredential() + if !creds.IsValid() && !globalIsEnvCreds { + return nil, errors.New("Azure backend account and secret keys should be set through ENVs") } - c, err := storage.NewClient(account, key, endPoint, globalAzureAPIVersion, secure) + c, err := storage.NewClient(creds.AccessKey, creds.SecretKey, endpoint, globalAzureAPIVersion, secure) if err != nil { return &azureObjects{}, err } + return &azureObjects{ client: c.GetBlobService(), metaInfo: azureMultipartMetaInfo{ @@ -195,7 +191,6 @@ func newAzureLayer(args cli.Args) (GatewayLayer, error) { // Shutdown - save any gateway metadata to disk // if necessary and reload upon next restart. func (a *azureObjects) Shutdown() error { - // TODO return nil } diff --git a/cmd/gateway-gcs-errors.go b/cmd/gateway-gcs-errors.go index 591d76d45..bd71fc359 100644 --- a/cmd/gateway-gcs-errors.go +++ b/cmd/gateway-gcs-errors.go @@ -19,9 +19,9 @@ package cmd import "errors" var ( - // Project is format is not valid - errGCSInvalidProjectID = errors.New("invalid project id") + // ProjectID format is not valid. + errGCSInvalidProjectID = errors.New("GCS project id is either empty or invalid") - // Multipart identifier is not in the correct form + // Multipart identifier is not in the correct form. errGCSNotValidMultipartIdentifier = errors.New("Not a valid multipart identifier") ) diff --git a/cmd/gateway-gcs.go b/cmd/gateway-gcs.go index 45a3b9db9..f60a9bd9c 100644 --- a/cmd/gateway-gcs.go +++ b/cmd/gateway-gcs.go @@ -33,7 +33,6 @@ import ( "google.golang.org/api/googleapi" "google.golang.org/api/iterator" - "github.com/minio/cli" minio "github.com/minio/minio-go" "github.com/minio/minio-go/pkg/policy" ) @@ -175,29 +174,20 @@ type gcsGateway struct { ctx context.Context } +const googleStorageEndpoint = "storage.googleapis.com" + // newGCSGateway returns gcs gatewaylayer -func newGCSGateway(args cli.Args) (GatewayLayer, error) { - if !args.Present() { - return nil, fmt.Errorf("ProjectID expected") - } - - endpoint := "storage.googleapis.com" - secure := true - projectID := args.First() - - if !isValidGCSProjectID(projectID) { - fatalIf(errGCSInvalidProjectID, "Unable to initialize GCS gateway") - } - +func newGCSGateway(projectID string) (GatewayLayer, error) { ctx := context.Background() - // Creates a client. + // Initialize a GCS client. client, err := storage.NewClient(ctx) if err != nil { return nil, err } - anonClient, err := minio.NewCore(endpoint, "", "", secure) + // Initialize a anonymous client with minio core APIs. + anonClient, err := minio.NewCore(googleStorageEndpoint, "", "", true) if err != nil { return nil, err } @@ -213,7 +203,6 @@ func newGCSGateway(args cli.Args) (GatewayLayer, error) { // Shutdown - save any gateway metadata to disk // if necessary and reload upon next restart. func (l *gcsGateway) Shutdown() error { - // TODO return nil } diff --git a/cmd/gateway-handlers.go b/cmd/gateway-handlers.go index d343f6e61..5cc4b5730 100644 --- a/cmd/gateway-handlers.go +++ b/cmd/gateway-handlers.go @@ -19,12 +19,12 @@ package cmd import ( "io" "io/ioutil" + "net" "net/http" "strconv" "encoding/hex" "encoding/json" - "encoding/xml" router "github.com/gorilla/mux" "github.com/minio/minio-go/pkg/policy" @@ -140,7 +140,7 @@ func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Re } // Reads the object at startOffset and writes to mw. - if err := getObject(bucket, object, startOffset, length, writer); err != nil { + if err = getObject(bucket, object, startOffset, length, writer); err != nil { errorIf(err, "Unable to write to client.") if !dataWritten { // Error response only if no data has been written to client yet. i.e if @@ -157,6 +157,23 @@ func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Re // call wrter.Write(nil) to set appropriate headers. writer.Write(nil) } + + // Get host and port from Request.RemoteAddr. + host, port, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + host, port = "", "" + } + + // Notify object accessed via a GET request. + eventNotify(eventData{ + Type: ObjectAccessedGet, + Bucket: bucket, + ObjInfo: objInfo, + ReqParams: extractReqParams(r), + UserAgent: r.UserAgent(), + Host: host, + Port: port, + }) } // PutObjectHandler - PUT Object @@ -293,6 +310,23 @@ func (api gatewayAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Re w.Header().Set("ETag", "\""+objInfo.ETag+"\"") writeSuccessResponseHeadersOnly(w) + + // Get host and port from Request.RemoteAddr. + host, port, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + host, port = "", "" + } + + // Notify object created event. + eventNotify(eventData{ + Type: ObjectCreatedPut, + Bucket: bucket, + ObjInfo: objInfo, + ReqParams: extractReqParams(r), + UserAgent: r.UserAgent(), + Host: host, + Port: port, + }) } // HeadObjectHandler - HEAD Object @@ -361,97 +395,23 @@ func (api gatewayAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.R // Successful response. w.WriteHeader(http.StatusOK) -} -// DeleteMultipleObjectsHandler - deletes multiple objects. -func (api gatewayAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) { - vars := router.Vars(r) - bucket := vars["bucket"] - - objectAPI := api.ObjectAPI() - if objectAPI == nil { - writeErrorResponse(w, ErrServerNotInitialized, r.URL) - return + // Get host and port from Request.RemoteAddr. + host, port, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + host, port = "", "" } - if s3Error := checkRequestAuthType(r, bucket, "s3:DeleteObject", serverConfig.GetRegion()); s3Error != ErrNone { - writeErrorResponse(w, s3Error, r.URL) - return - } - - // Content-Length is required and should be non-zero - // http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html - if r.ContentLength <= 0 { - writeErrorResponse(w, ErrMissingContentLength, r.URL) - return - } - - // Content-Md5 is requied should be set - // http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html - if _, ok := r.Header["Content-Md5"]; !ok { - writeErrorResponse(w, ErrMissingContentMD5, r.URL) - return - } - - // Allocate incoming content length bytes. - deleteXMLBytes := make([]byte, r.ContentLength) - - // Read incoming body XML bytes. - if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil { - errorIf(err, "Unable to read HTTP body.") - writeErrorResponse(w, ErrInternalError, r.URL) - return - } - - // Unmarshal list of keys to be deleted. - deleteObjects := &DeleteObjectsRequest{} - if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil { - errorIf(err, "Unable to unmarshal delete objects request XML.") - writeErrorResponse(w, ErrMalformedXML, r.URL) - return - } - - var dErrs = make([]error, len(deleteObjects.Objects)) - - // Delete all requested objects in parallel. - for index, object := range deleteObjects.Objects { - dErr := objectAPI.DeleteObject(bucket, object.ObjectName) - if dErr != nil { - dErrs[index] = dErr - } - } - - // Collect deleted objects and errors if any. - var deletedObjects []ObjectIdentifier - var deleteErrors []DeleteError - for index, err := range dErrs { - object := deleteObjects.Objects[index] - // Success deleted objects are collected separately. - if err == nil { - deletedObjects = append(deletedObjects, object) - continue - } - if _, ok := errorCause(err).(ObjectNotFound); ok { - // If the object is not found it should be - // accounted as deleted as per S3 spec. - deletedObjects = append(deletedObjects, object) - continue - } - errorIf(err, "Unable to delete object. %s", object.ObjectName) - // Error during delete should be collected separately. - deleteErrors = append(deleteErrors, DeleteError{ - Code: errorCodeResponse[toAPIErrorCode(err)].Code, - Message: errorCodeResponse[toAPIErrorCode(err)].Description, - Key: object.ObjectName, - }) - } - - // Generate response - response := generateMultiDeleteResponse(deleteObjects.Quiet, deletedObjects, deleteErrors) - encodedSuccessResponse := encodeResponse(response) - - // Write success response. - writeSuccessResponseXML(w, encodedSuccessResponse) + // Notify object accessed via a HEAD request. + eventNotify(eventData{ + Type: ObjectAccessedHead, + Bucket: bucket, + ObjInfo: objInfo, + ReqParams: extractReqParams(r), + UserAgent: r.UserAgent(), + Host: host, + Port: port, + }) } // PutBucketPolicyHandler - PUT Bucket policy diff --git a/cmd/gateway-main.go b/cmd/gateway-main.go index 3ede59ab8..a9888bb09 100644 --- a/cmd/gateway-main.go +++ b/cmd/gateway-main.go @@ -20,7 +20,6 @@ import ( "errors" "fmt" "net/url" - "os" "runtime" "strings" @@ -60,20 +59,6 @@ EXAMPLES: $ {{.HelpName}} https://azure.example.com ` -var azureBackendCmd = cli.Command{ - Name: "azure", - Usage: "Microsoft Azure Blob Storage.", - Action: azureGatewayMain, - CustomHelpTemplate: azureGatewayTemplate, - Flags: append(serverFlags, - cli.BoolFlag{ - Name: "quiet", - Usage: "Disable startup banner.", - }, - ), - HideHelpCommand: true, -} - const s3GatewayTemplate = `NAME: {{.HelpName}} - {{.Usage}} @@ -134,40 +119,41 @@ EXAMPLES: ` -var s3BackendCmd = cli.Command{ - Name: "s3", - Usage: "Amazon Simple Storage Service (S3).", - Action: s3GatewayMain, - CustomHelpTemplate: s3GatewayTemplate, - Flags: append(serverFlags, - cli.BoolFlag{ - Name: "quiet", - Usage: "Disable startup banner.", - }, - ), - HideHelpCommand: true, -} +var ( + azureBackendCmd = cli.Command{ + Name: "azure", + Usage: "Microsoft Azure Blob Storage.", + Action: azureGatewayMain, + CustomHelpTemplate: azureGatewayTemplate, + Flags: append(serverFlags, globalFlags...), + HideHelpCommand: true, + } -var gcsBackendCmd = cli.Command{ - Name: "gcs", - Usage: "Google Cloud Storage.", - Action: gcsGatewayMain, - CustomHelpTemplate: gcsGatewayTemplate, - Flags: append(serverFlags, - cli.BoolFlag{ - Name: "quiet", - Usage: "Disable startup banner.", - }, - ), - HideHelpCommand: true, -} + s3BackendCmd = cli.Command{ + Name: "s3", + Usage: "Amazon Simple Storage Service (S3).", + Action: s3GatewayMain, + CustomHelpTemplate: s3GatewayTemplate, + Flags: append(serverFlags, globalFlags...), + HideHelpCommand: true, + } + gcsBackendCmd = cli.Command{ + Name: "gcs", + Usage: "Google Cloud Storage.", + Action: gcsGatewayMain, + CustomHelpTemplate: gcsGatewayTemplate, + Flags: append(serverFlags, globalFlags...), + HideHelpCommand: true, + } -var gatewayCmd = cli.Command{ - Name: "gateway", - Usage: "Start object storage gateway.", - HideHelpCommand: true, - Subcommands: []cli.Command{azureBackendCmd, s3BackendCmd, gcsBackendCmd}, -} + gatewayCmd = cli.Command{ + Name: "gateway", + Usage: "Start object storage gateway.", + Flags: append(serverFlags, globalFlags...), + HideHelpCommand: true, + Subcommands: []cli.Command{azureBackendCmd, s3BackendCmd, gcsBackendCmd}, + } +) // Represents the type of the gateway backend. type gatewayBackend string @@ -179,38 +165,6 @@ const ( // Add more backends here. ) -// Returns access and secretkey set from environment variables. -func mustGetGatewayConfigFromEnv() (string, string, string) { - // Fetch access keys from environment variables. - accessKey := os.Getenv("MINIO_ACCESS_KEY") - secretKey := os.Getenv("MINIO_SECRET_KEY") - if accessKey == "" || secretKey == "" { - fatalIf(errors.New("Missing credentials"), "Access and secret keys are mandatory to run Minio gateway server.") - } - - region := globalMinioDefaultRegion - if v := os.Getenv("MINIO_REGION"); v != "" { - region = v - } - - return accessKey, secretKey, region -} - -// Set browser setting from environment variables -func mustSetBrowserSettingFromEnv() { - if browser := os.Getenv("MINIO_BROWSER"); browser != "" { - browserFlag, err := ParseBrowserFlag(browser) - if err != nil { - fatalIf(errors.New("invalid value"), "Unknown value ‘%s’ in MINIO_BROWSER environment variable.", browser) - } - - // browser Envs are set globally, this does not represent - // if browser is turned off or on. - globalIsEnvBrowser = true - globalIsBrowserEnabled = bool(browserFlag) - } -} - // Initialize gateway layer depending on the backend type. // Supported backend types are // @@ -218,46 +172,19 @@ func mustSetBrowserSettingFromEnv() { // - AWS S3. // - Google Cloud Storage. // - Add your favorite backend here. -func newGatewayLayer(backendType string, args cli.Args) (GatewayLayer, error) { - switch gatewayBackend(backendType) { +func newGatewayLayer(backendType gatewayBackend, arg string) (GatewayLayer, error) { + switch backendType { case azureBackend: - return newAzureLayer(args) + return newAzureLayer(arg) case s3Backend: - return newS3Gateway(args) + return newS3Gateway(arg) case gcsBackend: - return newGCSGateway(args) + return newGCSGateway(arg) } return nil, fmt.Errorf("Unrecognized backend type %s", backendType) } -// Initialize a new gateway config. -// -// DO NOT save this config, this is meant to be -// only used in memory. -func newGatewayConfig(accessKey, secretKey, region string) error { - // Initialize server config. - srvCfg := newServerConfigV19() - - // If env is set for a fresh start, save them to config file. - srvCfg.SetCredential(credential{ - AccessKey: accessKey, - SecretKey: secretKey, - }) - - // Set custom region. - srvCfg.SetRegion(region) - - // hold the mutex lock before a new config is assigned. - // Save the new config globally. - // unlock the mutex. - serverConfigMu.Lock() - serverConfig = srvCfg - serverConfigMu.Unlock() - - return nil -} - // Return endpoint. func parseGatewayEndpoint(arg string) (endPoint string, secure bool, err error) { schemeSpecified := len(strings.Split(arg, "://")) > 1 @@ -317,6 +244,9 @@ func azureGatewayMain(ctx *cli.Context) { cli.ShowCommandHelpAndExit(ctx, "azure", 1) } + // Validate gateway arguments. + fatalIf(validateGatewayArguments(ctx.String("address"), ctx.Args().First()), "Invalid argument") + gatewayMain(ctx, azureBackend) } @@ -326,13 +256,21 @@ func s3GatewayMain(ctx *cli.Context) { cli.ShowCommandHelpAndExit(ctx, "s3", 1) } + // Validate gateway arguments. + fatalIf(validateGatewayArguments(ctx.String("address"), ctx.Args().First()), "Invalid argument") + gatewayMain(ctx, s3Backend) } // Handler for 'minio gateway gcs' command line func gcsGatewayMain(ctx *cli.Context) { if ctx.Args().Present() && ctx.Args().First() == "help" { - cli.ShowCommandHelpAndExit(ctx, "s3", 1) + cli.ShowCommandHelpAndExit(ctx, "gcs", 1) + } + + if !isValidGCSProjectID(ctx.Args().First()) { + errorIf(errGCSInvalidProjectID, "Unable to start GCS gateway with %s", ctx.Args().First()) + cli.ShowCommandHelpAndExit(ctx, "gcs", 1) } gatewayMain(ctx, gcsBackend) @@ -340,43 +278,45 @@ func gcsGatewayMain(ctx *cli.Context) { // Handler for 'minio gateway'. func gatewayMain(ctx *cli.Context, backendType gatewayBackend) { - // Fetch access and secret key from env. - accessKey, secretKey, region := mustGetGatewayConfigFromEnv() - - // Fetch browser env setting - mustSetBrowserSettingFromEnv() - - // Initialize new gateway config. - newGatewayConfig(accessKey, secretKey, region) - // Get quiet flag from command line argument. quietFlag := ctx.Bool("quiet") || ctx.GlobalBool("quiet") if quietFlag { log.EnableQuiet() } - serverAddr := ctx.String("address") - endpointAddr := ctx.Args().First() - err := validateGatewayArguments(serverAddr, endpointAddr) - fatalIf(err, "Invalid argument") + // Handle common command args. + handleCommonCmdArgs(ctx) - // Create certs path for SSL configuration. - fatalIf(createConfigDir(), "Unable to create configuration directory") + // Handle common env vars. + handleCommonEnvVars() + + // Create certs path. + fatalIf(createConfigDir(), "Unable to create configuration directories.") + + // Initialize gateway config. + initConfig() + + // Enable loggers as per configuration file. + enableLoggers() + + // Init the error tracing module. + initError() + + // Check and load SSL certificates. + var err error + globalPublicCerts, globalRootCAs, globalIsSSL, err = getSSLConfig() + fatalIf(err, "Invalid SSL key file") initNSLock(false) // Enable local namespace lock. - newObject, err := newGatewayLayer(backendType, ctx.Args()[1:]) + newObject, err := newGatewayLayer(backendType, ctx.Args().First()) fatalIf(err, "Unable to initialize gateway layer") router := mux.NewRouter().SkipClean(true) - // credentials Envs are set globally. - globalIsEnvCreds = true - // Register web router when its enabled. if globalIsBrowserEnabled { - aerr := registerWebRouter(router) - fatalIf(aerr, "Unable to configure web browser") + fatalIf(registerWebRouter(router), "Unable to configure web browser") } registerGatewayAPIRouter(router, newObject) @@ -409,10 +349,7 @@ func gatewayMain(ctx *cli.Context, backendType gatewayBackend) { } - apiServer := NewServerMux(serverAddr, registerHandlers(router, handlerFns...)) - - _, _, globalIsSSL, err = getSSLConfig() - fatalIf(err, "Invalid SSL key file") + apiServer := NewServerMux(ctx.String("address"), registerHandlers(router, handlerFns...)) // Start server, automatically configures TLS if certs are available. go func() { @@ -420,9 +357,7 @@ func gatewayMain(ctx *cli.Context, backendType gatewayBackend) { if globalIsSSL { cert, key = getPublicCertFile(), getPrivateKeyFile() } - - aerr := apiServer.ListenAndServe(cert, key) - fatalIf(aerr, "Failed to start minio server") + fatalIf(apiServer.ListenAndServe(cert, key), "Failed to start minio server") }() // Once endpoints are finalized, initialize the new object api. @@ -441,9 +376,12 @@ func gatewayMain(ctx *cli.Context, backendType gatewayBackend) { case s3Backend: mode = globalMinioModeGatewayS3 } + + // Check update mode. checkUpdate(mode) - apiEndpoints := getAPIEndpoints(apiServer.Addr) - printGatewayStartupMessage(apiEndpoints, accessKey, secretKey, backendType) + + // Print gateway startup message. + printGatewayStartupMessage(getAPIEndpoints(apiServer.Addr), backendType) } <-globalServiceDoneCh diff --git a/cmd/gateway-main_test.go b/cmd/gateway-main_test.go index b80d0292b..fdab9580f 100644 --- a/cmd/gateway-main_test.go +++ b/cmd/gateway-main_test.go @@ -17,7 +17,6 @@ package cmd import ( - "os" "strings" "testing" ) @@ -53,28 +52,6 @@ func TestParseGatewayEndpoint(t *testing.T) { } } -func TestSetBrowserFromEnv(t *testing.T) { - browser := os.Getenv("MINIO_BROWSER") - - os.Setenv("MINIO_BROWSER", "on") - mustSetBrowserSettingFromEnv() - if globalIsBrowserEnabled != true { - t.Errorf("Expected the response status to be `%t`, but instead found `%t`", globalIsBrowserEnabled, false) - } - - os.Setenv("MINIO_BROWSER", "off") - mustSetBrowserSettingFromEnv() - if globalIsBrowserEnabled != false { - t.Errorf("Expected the response status to be `%t`, but instead found `%t`", globalIsBrowserEnabled, true) - } - os.Setenv("MINIO_BROWSER", "") - mustSetBrowserSettingFromEnv() - if globalIsBrowserEnabled != false { - t.Errorf("Expected the response status to be `%t`, but instead found `%t`", globalIsBrowserEnabled, true) - } - os.Setenv("MINIO_BROWSER", browser) -} - // Test validateGatewayArguments func TestValidateGatewayArguments(t *testing.T) { nonLoopBackIPs := localIP4.FuncMatch(func(ip string, matchString string) bool { diff --git a/cmd/gateway-s3.go b/cmd/gateway-s3.go index f6f7410ab..98bf96e08 100644 --- a/cmd/gateway-s3.go +++ b/cmd/gateway-s3.go @@ -20,12 +20,10 @@ import ( "errors" "io" "net/http" - "os" "path" "encoding/hex" - "github.com/minio/cli" minio "github.com/minio/minio-go" "github.com/minio/minio-go/pkg/policy" ) @@ -101,31 +99,33 @@ type s3Objects struct { } // newS3Gateway returns s3 gatewaylayer -func newS3Gateway(args cli.Args) (GatewayLayer, error) { +func newS3Gateway(host string) (GatewayLayer, error) { var err error + var endpoint string + var secure = true - // Default endpoint parameters - endpoint := "s3.amazonaws.com" - secure := true - - // Check if user provided some parameters - if args.Present() { - // Override default params if the endpoint is provided - endpoint, secure, err = parseGatewayEndpoint(args.First()) + // Validate host parameters. + if host != "" { + // Override default params if the host is provided + endpoint, secure, err = parseGatewayEndpoint(host) if err != nil { return nil, err } } - accessKey := os.Getenv("MINIO_ACCESS_KEY") - secretKey := os.Getenv("MINIO_SECRET_KEY") - if accessKey == "" || secretKey == "" { - return nil, errors.New("No S3 access and secret key") + // Default endpoint parameters + if endpoint == "" { + endpoint = "s3.amazonaws.com" + } + + creds := serverConfig.GetCredential() + if !creds.IsValid() && !globalIsEnvCreds { + return nil, errors.New("S3 backend account and secret keys should be set through ENVs") } // Initialize minio client object. - client, err := minio.NewCore(endpoint, accessKey, secretKey, secure) + client, err := minio.NewCore(endpoint, creds.AccessKey, creds.SecretKey, secure) if err != nil { return nil, err } diff --git a/cmd/gateway-startup-msg.go b/cmd/gateway-startup-msg.go index b89a753f3..e17a3e4d5 100644 --- a/cmd/gateway-startup-msg.go +++ b/cmd/gateway-startup-msg.go @@ -18,28 +18,19 @@ package cmd import ( "fmt" - "runtime" "strings" ) // Prints the formatted startup message. -func printGatewayStartupMessage(apiEndPoints []string, accessKey, secretKey string, backendType gatewayBackend) { +func printGatewayStartupMessage(apiEndPoints []string, backendType gatewayBackend) { + strippedAPIEndpoints := stripStandardPorts(apiEndPoints) + // Prints credential. - printGatewayCommonMsg(apiEndPoints, accessKey, secretKey) + printGatewayCommonMsg(strippedAPIEndpoints) // Prints `mc` cli configuration message chooses // first endpoint as default. - endPoint := apiEndPoints[0] - - // Configure 'mc', following block prints platform specific information for minio client. - log.Println(colorBlue("\nCommand-line Access: ") + mcQuickStartGuide) - if runtime.GOOS == globalWindowsOSName { - mcMessage := fmt.Sprintf("$ mc.exe config host add my%s %s %s %s", backendType, endPoint, accessKey, secretKey) - log.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage)) - } else { - mcMessage := fmt.Sprintf("$ mc config host add my%s %s %s %s", backendType, endPoint, accessKey, secretKey) - log.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage)) - } + printCLIAccessMsg(strippedAPIEndpoints[0], fmt.Sprintf("my%s", backendType)) // Prints documentation message. printObjectAPIMsg() @@ -52,10 +43,16 @@ func printGatewayStartupMessage(apiEndPoints []string, accessKey, secretKey stri } // Prints common server startup message. Prints credential, region and browser access. -func printGatewayCommonMsg(apiEndpoints []string, accessKey, secretKey string) { +func printGatewayCommonMsg(apiEndpoints []string) { + // Get saved credentials. + cred := serverConfig.GetCredential() + apiEndpointStr := strings.Join(apiEndpoints, " ") // Colorize the message and print. log.Println(colorBlue("\nEndpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr))) - log.Println(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", accessKey))) - log.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", secretKey))) + log.Println(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey))) + log.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey))) + + log.Println(colorBlue("\nBrowser Access:")) + log.Println(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 3), apiEndpointStr)) } diff --git a/cmd/gateway-startup-msg_test.go b/cmd/gateway-startup-msg_test.go index c3bf2e6cb..bda258fca 100644 --- a/cmd/gateway-startup-msg_test.go +++ b/cmd/gateway-startup-msg_test.go @@ -20,12 +20,24 @@ import "testing" // Test printing Gateway common message. func TestPrintGatewayCommonMessage(t *testing.T) { - apiEndpoints := []string{"127.0.0.1:9000"} - printGatewayCommonMsg(apiEndpoints, "abcd1", "abcd123") + root, err := newTestConfig(globalMinioDefaultRegion) + if err != nil { + t.Fatal(err) + } + defer removeAll(root) + + apiEndpoints := []string{"http://127.0.0.1:9000"} + printGatewayCommonMsg(apiEndpoints) } // Test print gateway startup message. func TestPrintGatewayStartupMessage(t *testing.T) { - apiEndpoints := []string{"127.0.0.1:9000"} - printGatewayStartupMessage(apiEndpoints, "abcd1", "abcd123", "azure") + root, err := newTestConfig(globalMinioDefaultRegion) + if err != nil { + t.Fatal(err) + } + defer removeAll(root) + + apiEndpoints := []string{"http://127.0.0.1:9000"} + printGatewayStartupMessage(apiEndpoints, "azure") } diff --git a/cmd/server-main.go b/cmd/server-main.go index 1a84f6250..1f66e8c7f 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -17,12 +17,8 @@ package cmd import ( - "errors" "os" - "path/filepath" "runtime" - "strings" - "time" "github.com/minio/cli" "github.com/minio/dsync" @@ -78,61 +74,9 @@ EXAMPLES: `, } -// Check for updates and print a notification message -func checkUpdate(mode string) { - // Its OK to ignore any errors during getUpdateInfo() here. - if older, downloadURL, err := getUpdateInfo(1*time.Second, mode); err == nil { - if updateMsg := computeUpdateMessage(downloadURL, older); updateMsg != "" { - log.Println(updateMsg) - } - } -} - -func enableLoggers() { - fileLogTarget := serverConfig.Logger.GetFile() - if fileLogTarget.Enable { - err := InitFileLogger(&fileLogTarget) - fatalIf(err, "Unable to initialize file logger") - log.AddTarget(fileLogTarget) - } - - consoleLogTarget := serverConfig.Logger.GetConsole() - if consoleLogTarget.Enable { - InitConsoleLogger(&consoleLogTarget) - } - - log.SetConsoleTarget(consoleLogTarget) -} - -func initConfig() { - // Config file does not exist, we create it fresh and return upon success. - if isFile(getConfigFile()) { - fatalIf(migrateConfig(), "Config migration failed.") - fatalIf(loadConfig(), "Unable to load config version: '%s'.", v19) - } else { - fatalIf(newConfig(), "Unable to initialize minio config for the first time.") - log.Println("Created minio configuration file successfully at " + getConfigDir()) - } -} - func serverHandleCmdArgs(ctx *cli.Context) { - // Set configuration directory. - { - // Get configuration directory from command line argument. - configDir := ctx.String("config-dir") - if !ctx.IsSet("config-dir") && ctx.GlobalIsSet("config-dir") { - configDir = ctx.GlobalString("config-dir") - } - if configDir == "" { - fatalIf(errors.New("empty directory"), "Configuration directory cannot be empty.") - } - - // Disallow relative paths, figure out absolute paths. - configDirAbs, err := filepath.Abs(configDir) - fatalIf(err, "Unable to fetch absolute path for config directory %s", configDir) - - setConfigDir(configDirAbs) - } + // Handle common command args. + handleCommonCmdArgs(ctx) // Server address. serverAddr := ctx.String("address") @@ -159,36 +103,8 @@ func serverHandleCmdArgs(ctx *cli.Context) { } func serverHandleEnvVars() { - // Start profiler if env is set. - if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" { - globalProfiler = startProfiler(profiler) - } - - // Check if object cache is disabled. - globalXLObjCacheDisabled = strings.EqualFold(os.Getenv("_MINIO_CACHE"), "off") - - accessKey := os.Getenv("MINIO_ACCESS_KEY") - secretKey := os.Getenv("MINIO_SECRET_KEY") - if accessKey != "" && secretKey != "" { - cred, err := createCredential(accessKey, secretKey) - fatalIf(err, "Invalid access/secret Key set in environment.") - - // credential Envs are set globally. - globalIsEnvCreds = true - globalActiveCred = cred - } - - if browser := os.Getenv("MINIO_BROWSER"); browser != "" { - browserFlag, err := ParseBrowserFlag(browser) - if err != nil { - fatalIf(errors.New("invalid value"), "Unknown value ‘%s’ in MINIO_BROWSER environment variable.", browser) - } - - // browser Envs are set globally, this does not represent - // if browser is turned off or on. - globalIsEnvBrowser = true - globalIsBrowserEnabled = bool(browserFlag) - } + // Handle common environment variables. + handleCommonEnvVars() if serverRegion := os.Getenv("MINIO_REGION"); serverRegion != "" { // region Envs are set globally. @@ -210,12 +126,16 @@ func serverMain(ctx *cli.Context) { log.EnableQuiet() } + // Handle all server command args. serverHandleCmdArgs(ctx) + + // Handle all server environment vars. serverHandleEnvVars() // Create certs path. fatalIf(createConfigDir(), "Unable to create configuration directories.") + // Initialize server config. initConfig() // Enable loggers as per configuration file. diff --git a/cmd/server-startup-msg.go b/cmd/server-startup-msg.go index 9f4549eee..7e1cc6934 100644 --- a/cmd/server-startup-msg.go +++ b/cmd/server-startup-msg.go @@ -52,7 +52,7 @@ func printStartupMessage(apiEndPoints []string) { // Prints `mc` cli configuration message chooses // first endpoint as default. - printCLIAccessMsg(strippedAPIEndpoints[0]) + printCLIAccessMsg(strippedAPIEndpoints[0], "myminio") // Prints documentation message. printObjectAPIMsg() @@ -141,17 +141,17 @@ func printEventNotifiers() { // Prints startup message for command line access. Prints link to our documentation // and custom platform specific message. -func printCLIAccessMsg(endPoint string) { +func printCLIAccessMsg(endPoint string, alias string) { // Get saved credentials. cred := serverConfig.GetCredential() // Configure 'mc', following block prints platform specific information for minio client. log.Println(colorBlue("\nCommand-line Access: ") + mcQuickStartGuide) if runtime.GOOS == globalWindowsOSName { - mcMessage := fmt.Sprintf("$ mc.exe config host add myminio %s %s %s", endPoint, cred.AccessKey, cred.SecretKey) + mcMessage := fmt.Sprintf("$ mc.exe config host add %s %s %s %s", alias, endPoint, cred.AccessKey, cred.SecretKey) log.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage)) } else { - mcMessage := fmt.Sprintf("$ mc config host add myminio %s %s %s", endPoint, cred.AccessKey, cred.SecretKey) + mcMessage := fmt.Sprintf("$ mc config host add %s %s %s %s", alias, endPoint, cred.AccessKey, cred.SecretKey) log.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage)) } } diff --git a/cmd/server-startup-msg_test.go b/cmd/server-startup-msg_test.go index 2da77901c..5c4090328 100644 --- a/cmd/server-startup-msg_test.go +++ b/cmd/server-startup-msg_test.go @@ -140,7 +140,7 @@ func TestPrintCLIAccessMsg(t *testing.T) { defer removeAll(root) apiEndpoints := []string{"http://127.0.0.1:9000"} - printCLIAccessMsg(apiEndpoints[0]) + printCLIAccessMsg(apiEndpoints[0], "myminio") } // Test print startup message.