Add new console/http loggers (#6066)

- Add console target logging, enabled by default.
- Add http target logging, which supports an endpoint
  with basic authentication (username/password are passed
  in the endpoint url itself)
- HTTP target logging is asynchronous and some logs can be
  dropped if channel buffer (10000) is full

cmd/admin-handlers_test.go

@@ -38,7 +38,7 @@ import (
 
 var (
 	configJSON = []byte(`{
-	"version": "26",
+	"version": "27",
 	"credential": {
 		"accessKey": "minio",
 		"secretKey": "minio123"
@@ -162,8 +162,20 @@ var (
 				"endpoint": ""
 			}
 		}
-	}
-}`)
+	    },
+	    "logger": {
+		"console": {
+		    "enabled": true
+		},
+		"http": {
+		    "1": {
+			"enabled": false,
+			"endpoint": "http://user:example@localhost:9001/api/endpoint"
+		    }
+		}
+	    }
+
+	}`)
 )
 
 // adminXLTestBed - encapsulates subsystems that need to be setup for

cmd/common-main.go

@@ -81,6 +81,20 @@ func initConfig() {
 	}
 }
 
+// Load logger targets based on user's configuration
+func loadLoggers() {
+	if globalServerConfig.Logger.Console.Enabled {
+		// Enable console logging
+		logger.AddTarget(logger.NewConsole())
+	}
+	for _, l := range globalServerConfig.Logger.HTTP {
+		if l.Enabled {
+			// Enable http logging
+			logger.AddTarget(logger.NewHTTP(l.Endpoint, NewCustomHTTPTransport()))
+		}
+	}
+}
+
 func handleCommonCmdArgs(ctx *cli.Context) {
 
 	var configDir string

cmd/config-current.go

@@ -41,9 +41,9 @@ import (
 // 6. Make changes in config-current_test.go for any test change
 
 // Config version
-const serverConfigVersion = "26"
+const serverConfigVersion = "27"
 
-type serverConfig = serverConfigV26
+type serverConfig = serverConfigV27
 
 var (
 	// globalServerConfig server config.
@@ -260,6 +260,8 @@ func (s *serverConfig) ConfigDiff(t *serverConfig) string {
 		return "MySQL Notification configuration differs"
 	case !reflect.DeepEqual(s.Notify.MQTT, t.Notify.MQTT):
 		return "MQTT Notification configuration differs"
+	case !reflect.DeepEqual(s.Logger, t.Logger):
+		return "Logger configuration differs"
 	case reflect.DeepEqual(s, t):
 		return ""
 	default:
@@ -315,6 +317,13 @@ func newServerConfig() *serverConfig {
 	srvCfg.Cache.Exclude = make([]string, 0)
 	srvCfg.Cache.Expiry = globalCacheExpiry
 	srvCfg.Cache.MaxUse = globalCacheMaxUse
+
+	// Console logging is on by default
+	srvCfg.Logger.Console.Enabled = true
+	// Create an example of HTTP logger
+	srvCfg.Logger.HTTP = make(map[string]loggerHTTP)
+	srvCfg.Logger.HTTP["target1"] = loggerHTTP{Endpoint: "https://username:password@example.com/api"}
+
 	return srvCfg
 }
 

cmd/config-current_test.go

@@ -319,6 +319,18 @@ func TestConfigDiff(t *testing.T) {
 			&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: false}}}},
 			"MQTT Notification configuration differs",
 		},
+		// 16
+		{
+			&serverConfig{Logger: loggerConfig{
+				Console: loggerConsole{Enabled: true},
+				HTTP:    map[string]loggerHTTP{"1": {Endpoint: "http://address1"}},
+			}},
+			&serverConfig{Logger: loggerConfig{
+				Console: loggerConsole{Enabled: true},
+				HTTP:    map[string]loggerHTTP{"1": {Endpoint: "http://address2"}},
+			}},
+			"Logger configuration differs",
+		},
 	}
 
 	for i, testCase := range testCases {

cmd/config-migrate.go

@@ -188,6 +188,11 @@ func migrateConfig() error {
 			return err
 		}
 		fallthrough
+	case "26":
+		if err = migrateV26ToV27(); err != nil {
+			return err
+		}
+		fallthrough
 	case serverConfigVersion:
 		// No migration needed. this always points to current version.
 		err = nil
@@ -2317,3 +2322,36 @@ func migrateV25ToV26() error {
 	logger.Info(configMigrateMSGTemplate, configFile, cv25.Version, srvConfig.Version)
 	return nil
 }
+
+func migrateV26ToV27() error {
+	configFile := getConfigFile()
+
+	// config V27 is backward compatible with V26, load the old
+	// config file in serverConfigV27 struct and put some examples
+	// in the new `logger` field
+	srvConfig := &serverConfigV27{}
+	_, err := quick.LoadConfig(configFile, globalEtcdClient, srvConfig)
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config file. %v", err)
+	}
+
+	if srvConfig.Version != "26" {
+		return nil
+	}
+
+	srvConfig.Version = "27"
+	// Enable console logging by default to avoid breaking users
+	// current deployments
+	srvConfig.Logger.Console.Enabled = true
+	srvConfig.Logger.HTTP = make(map[string]loggerHTTP)
+	srvConfig.Logger.HTTP["1"] = loggerHTTP{}
+
+	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘26’ to ‘27’. %v", err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, "26", "27")
+	return nil
+}

cmd/config-migrate_test.go

@@ -134,10 +134,25 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 	if err := migrateV21ToV22(); err != nil {
 		t.Fatal("migrate v21 to v22 should succeed when no config file is found")
 	}
+	if err := migrateV22ToV23(); err != nil {
+		t.Fatal("migrate v22 to v23 should succeed when no config file is found")
+	}
+	if err := migrateV23ToV24(); err != nil {
+		t.Fatal("migrate v23 to v24 should succeed when no config file is found")
+	}
+	if err := migrateV24ToV25(); err != nil {
+		t.Fatal("migrate v24 to v25 should succeed when no config file is found")
+	}
+	if err := migrateV25ToV26(); err != nil {
+		t.Fatal("migrate v25 to v26 should succeed when no config file is found")
+	}
+	if err := migrateV26ToV27(); err != nil {
+		t.Fatal("migrate v26 to v27 should succeed when no config file is found")
+	}
 }
 
-// Test if a config migration from v2 to v23 is successfully done
-func TestServerConfigMigrateV2toV23(t *testing.T) {
+// Test if a config migration from v2 to v27 is successfully done
+func TestServerConfigMigrateV2toV27(t *testing.T) {
 	rootPath, err := newTestConfig(globalMinioDefaultRegion)
 	if err != nil {
 		t.Fatalf("Init Test config failed")
@@ -272,6 +287,18 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 	if err := migrateV22ToV23(); err == nil {
 		t.Fatal("migrateConfigV22ToV23() should fail with a corrupted json")
 	}
+	if err := migrateV23ToV24(); err == nil {
+		t.Fatal("migrateConfigV23ToV24() should fail with a corrupted json")
+	}
+	if err := migrateV24ToV25(); err == nil {
+		t.Fatal("migrateConfigV24ToV25() should fail with a corrupted json")
+	}
+	if err := migrateV25ToV26(); err == nil {
+		t.Fatal("migrateConfigV25ToV26() should fail with a corrupted json")
+	}
+	if err := migrateV26ToV27(); err == nil {
+		t.Fatal("migrateConfigV26ToV27() should fail with a corrupted json")
+	}
 }
 
 // Test if all migrate code returns error with corrupted config files

cmd/config-versions.go

@@ -657,9 +657,6 @@ type serverConfigV25 struct {
 
 // serverConfigV26 is just like version '25', stores additionally
 // cache max use value in 'CacheConfig'.
-//
-// IMPORTANT NOTE: When updating this struct make sure that
-// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV26 struct {
 	quick.Config `json:"-"` // ignore interfaces
 
@@ -681,3 +678,47 @@ type serverConfigV26 struct {
 	// Notification queue configuration.
 	Notify notifier `json:"notify"`
 }
+
+type loggerConsole struct {
+	Enabled bool `json:"enabled"`
+}
+
+type loggerHTTP struct {
+	Enabled  bool   `json:"enabled"`
+	Endpoint string `json:"endpoint"`
+}
+
+type loggerConfig struct {
+	Console loggerConsole         `json:"console"`
+	HTTP    map[string]loggerHTTP `json:"http"`
+}
+
+// serverConfigV27 is just like version '26', stores additionally
+// the logger field
+//
+// IMPORTANT NOTE: When updating this struct make sure that
+// serverConfig.ConfigDiff() is updated as necessary.
+type serverConfigV27 struct {
+	quick.Config `json:"-"` // ignore interfaces
+
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BoolFlag         `json:"browser"`
+	Worm       BoolFlag         `json:"worm"`
+	Domain     string           `json:"domain"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+
+	// Logger configuration
+	Logger loggerConfig `json:"logger"`
+}

cmd/gateway-main.go

@@ -159,6 +159,9 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Initialize gateway config.
 	initConfig()
 
+	// Load logger subsystem
+	loadLoggers()
+
 	// Check and load SSL certificates.
 	var err error
 	globalPublicCerts, globalRootCAs, globalTLSCerts, globalIsSSL, err = getSSLConfig()

cmd/logger/console.go

@@ -0,0 +1,101 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package logger
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// ConsoleTarget implements loggerTarget to send log
+// in plain or json format to the standard output.
+type ConsoleTarget struct{}
+
+func (c *ConsoleTarget) send(entry logEntry) error {
+	if jsonFlag {
+		logJSON, err := json.Marshal(&entry)
+		if err != nil {
+			return err
+		}
+		fmt.Println(string(logJSON))
+		return nil
+	}
+
+	trace := make([]string, len(entry.Trace.Source))
+
+	// Add a sequence number and formatting for each stack trace
+	// No formatting is required for the first entry
+	for i, element := range entry.Trace.Source {
+		trace[i] = fmt.Sprintf("%8v: %s", i+1, element)
+	}
+
+	tagString := ""
+	for key, value := range entry.Trace.Variables {
+		if value != "" {
+			if tagString != "" {
+				tagString += ", "
+			}
+			tagString += key + "=" + value
+		}
+	}
+
+	apiString := "API: " + entry.API.Name + "("
+	if entry.API.Args != nil && entry.API.Args.Bucket != "" {
+		apiString = apiString + "bucket=" + entry.API.Args.Bucket
+	}
+	if entry.API.Args != nil && entry.API.Args.Object != "" {
+		apiString = apiString + ", object=" + entry.API.Args.Object
+	}
+	apiString += ")"
+	timeString := "Time: " + time.Now().Format(loggerTimeFormat)
+
+	var requestID string
+	if entry.RequestID != "" {
+		requestID = "\nRequestID: " + entry.RequestID
+	}
+
+	var remoteHost string
+	if entry.RemoteHost != "" {
+		remoteHost = "\nRemoteHost: " + entry.RemoteHost
+	}
+
+	var userAgent string
+	if entry.UserAgent != "" {
+		userAgent = "\nUserAgent: " + entry.UserAgent
+	}
+
+	if len(entry.Trace.Variables) > 0 {
+		tagString = "\n       " + tagString
+	}
+
+	var msg = colorFgRed(colorBold(entry.Trace.Message))
+	var output = fmt.Sprintf("\n%s\n%s%s%s%s\nError: %s%s\n%s",
+		apiString, timeString, requestID, remoteHost, userAgent,
+		msg, tagString, strings.Join(trace, "\n"))
+
+	fmt.Println(output)
+	return nil
+}
+
+// NewConsole initializes a new logger target
+// which prints log directly in the standard
+// output.
+func NewConsole() LoggingTarget {
+	return &ConsoleTarget{}
+}

cmd/logger/http.go

@@ -0,0 +1,88 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package logger
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"net/http"
+)
+
+// HTTPTarget implements loggerTarget and sends the json
+// format of a log entry to the configured http endpoint.
+// An internal buffer of logs is maintained but when the
+// buffer is full, new logs are just ignored and an error
+// is returned to the caller.
+type HTTPTarget struct {
+	// Channel of log entries
+	logCh chan logEntry
+	// HTTP(s) endpoint
+	endpoint string
+	client   http.Client
+}
+
+func (h *HTTPTarget) startHTTPLogger() {
+	// Create a routine which sends json logs received
+	// from an internal channel.
+	go func() {
+		for entry := range h.logCh {
+			logJSON, err := json.Marshal(&entry)
+			if err != nil {
+				continue
+			}
+
+			req, err := http.NewRequest("POST", h.endpoint, bytes.NewBuffer(logJSON))
+			req.Header.Set("Content-Type", "application/json")
+
+			resp, err := h.client.Do(req)
+			if err != nil {
+				continue
+			}
+			if resp.Body != nil {
+				resp.Body.Close()
+			}
+		}
+	}()
+}
+
+// NewHTTP initializes a new logger target which
+// sends log over http to the specified endpoint
+func NewHTTP(endpoint string, transport *http.Transport) LoggingTarget {
+	h := HTTPTarget{
+		endpoint: endpoint,
+		client: http.Client{
+			Transport: transport,
+		},
+		logCh: make(chan logEntry, 10000),
+	}
+
+	h.startHTTPLogger()
+	return &h
+}
+
+func (h *HTTPTarget) send(entry logEntry) error {
+	select {
+	case h.logCh <- entry:
+	default:
+		// log channel is full, do not wait and return
+		// an error immediately to the caller
+		return errors.New("log buffer full")
+	}
+
+	return nil
+}

cmd/logger/logger.go

@@ -299,74 +299,21 @@ func LogIf(ctx context.Context, err error) {
 	// Get the cause for the Error
 	message := err.Error()
 
-	// Output the formatted log message at console
-	var output string
-	if jsonFlag {
-		logJSON, err := json.Marshal(&logEntry{
-			Level:      ErrorLvl.String(),
-			RemoteHost: req.RemoteHost,
-			RequestID:  req.RequestID,
-			UserAgent:  req.UserAgent,
-			Time:       time.Now().UTC().Format(time.RFC3339Nano),
-			API:        &api{Name: API, Args: &args{Bucket: req.BucketName, Object: req.ObjectName}},
-			Trace:      &traceEntry{Message: message, Source: trace, Variables: tags},
-		})
-		if err != nil {
-			panic(err)
-		}
-		output = string(logJSON)
-	} else {
-		// Add a sequence number and formatting for each stack trace
-		// No formatting is required for the first entry
-		for i, element := range trace {
-			trace[i] = fmt.Sprintf("%8v: %s", i+1, element)
-		}
-
-		tagString := ""
-		for key, value := range tags {
-			if value != "" {
-				if tagString != "" {
-					tagString += ", "
-				}
-				tagString += key + "=" + value
-			}
-		}
-
-		apiString := "API: " + API + "("
-		if req.BucketName != "" {
-			apiString = apiString + "bucket=" + req.BucketName
-		}
-		if req.ObjectName != "" {
-			apiString = apiString + ", object=" + req.ObjectName
-		}
-		apiString += ")"
-		timeString := "Time: " + time.Now().Format(loggerTimeFormat)
-
-		var requestID string
-		if req.RequestID != "" {
-			requestID = "\nRequestID: " + req.RequestID
-		}
-
-		var remoteHost string
-		if req.RemoteHost != "" {
-			remoteHost = "\nRemoteHost: " + req.RemoteHost
-		}
-
-		var userAgent string
-		if req.UserAgent != "" {
-			userAgent = "\nUserAgent: " + req.UserAgent
-		}
-
-		if len(tags) > 0 {
-			tagString = "\n       " + tagString
-		}
-
-		var msg = colorFgRed(colorBold(message))
-		output = fmt.Sprintf("\n%s\n%s%s%s%s\nError: %s%s\n%s",
-			apiString, timeString, requestID, remoteHost, userAgent,
-			msg, tagString, strings.Join(trace, "\n"))
+	entry := logEntry{
+		DeploymentID: deploymentID,
+		Level:        ErrorLvl.String(),
+		RemoteHost:   req.RemoteHost,
+		RequestID:    req.RequestID,
+		UserAgent:    req.UserAgent,
+		Time:         time.Now().UTC().Format(time.RFC3339Nano),
+		API:          &api{Name: API, Args: &args{Bucket: req.BucketName, Object: req.ObjectName}},
+		Trace:        &traceEntry{Message: message, Source: trace, Variables: tags},
+	}
+
+	// Iterate over all logger targets to send the log entry
+	for _, t := range Targets {
+		t.send(entry)
 	}
-	fmt.Println(output)
 }
 
 // ErrCritical is the value panic'd whenever CriticalIf is called.

cmd/logger/targets.go

@@ -0,0 +1,33 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package logger
+
+// LoggingTarget is the entity that we will receive
+// a single log entry and send it to the log target
+//   e.g. send the log to a http server
+type LoggingTarget interface {
+	send(entry logEntry) error
+}
+
+// Targets is the set of enabled loggers
+var Targets = []LoggingTarget{}
+
+// AddTarget adds a new logger target to the
+// list of enabled loggers
+func AddTarget(t LoggingTarget) {
+	Targets = append(Targets, t)
+}

cmd/server-main.go

@@ -221,6 +221,9 @@ func serverMain(ctx *cli.Context) {
 	// Initialize server config.
 	initConfig()
 
+	// Load logger subsystem
+	loadLoggers()
+
 	// Check and load SSL certificates.
 	var err error
 	globalPublicCerts, globalRootCAs, globalTLSCerts, globalIsSSL, err = getSSLConfig()