Add new console/http loggers (#6066)

- Add console target logging, enabled by default.
- Add http target logging, which supports an endpoint
  with basic authentication (username/password are passed
  in the endpoint url itself)
- HTTP target logging is asynchronous and some logs can be
  dropped if channel buffer (10000) is full

cmd/logger/console.go

@@ -0,0 +1,101 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package logger
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// ConsoleTarget implements loggerTarget to send log
+// in plain or json format to the standard output.
+type ConsoleTarget struct{}
+
+func (c *ConsoleTarget) send(entry logEntry) error {
+	if jsonFlag {
+		logJSON, err := json.Marshal(&entry)
+		if err != nil {
+			return err
+		}
+		fmt.Println(string(logJSON))
+		return nil
+	}
+
+	trace := make([]string, len(entry.Trace.Source))
+
+	// Add a sequence number and formatting for each stack trace
+	// No formatting is required for the first entry
+	for i, element := range entry.Trace.Source {
+		trace[i] = fmt.Sprintf("%8v: %s", i+1, element)
+	}
+
+	tagString := ""
+	for key, value := range entry.Trace.Variables {
+		if value != "" {
+			if tagString != "" {
+				tagString += ", "
+			}
+			tagString += key + "=" + value
+		}
+	}
+
+	apiString := "API: " + entry.API.Name + "("
+	if entry.API.Args != nil && entry.API.Args.Bucket != "" {
+		apiString = apiString + "bucket=" + entry.API.Args.Bucket
+	}
+	if entry.API.Args != nil && entry.API.Args.Object != "" {
+		apiString = apiString + ", object=" + entry.API.Args.Object
+	}
+	apiString += ")"
+	timeString := "Time: " + time.Now().Format(loggerTimeFormat)
+
+	var requestID string
+	if entry.RequestID != "" {
+		requestID = "\nRequestID: " + entry.RequestID
+	}
+
+	var remoteHost string
+	if entry.RemoteHost != "" {
+		remoteHost = "\nRemoteHost: " + entry.RemoteHost
+	}
+
+	var userAgent string
+	if entry.UserAgent != "" {
+		userAgent = "\nUserAgent: " + entry.UserAgent
+	}
+
+	if len(entry.Trace.Variables) > 0 {
+		tagString = "\n       " + tagString
+	}
+
+	var msg = colorFgRed(colorBold(entry.Trace.Message))
+	var output = fmt.Sprintf("\n%s\n%s%s%s%s\nError: %s%s\n%s",
+		apiString, timeString, requestID, remoteHost, userAgent,
+		msg, tagString, strings.Join(trace, "\n"))
+
+	fmt.Println(output)
+	return nil
+}
+
+// NewConsole initializes a new logger target
+// which prints log directly in the standard
+// output.
+func NewConsole() LoggingTarget {
+	return &ConsoleTarget{}
+}

cmd/logger/http.go

@@ -0,0 +1,88 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package logger
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"net/http"
+)
+
+// HTTPTarget implements loggerTarget and sends the json
+// format of a log entry to the configured http endpoint.
+// An internal buffer of logs is maintained but when the
+// buffer is full, new logs are just ignored and an error
+// is returned to the caller.
+type HTTPTarget struct {
+	// Channel of log entries
+	logCh chan logEntry
+	// HTTP(s) endpoint
+	endpoint string
+	client   http.Client
+}
+
+func (h *HTTPTarget) startHTTPLogger() {
+	// Create a routine which sends json logs received
+	// from an internal channel.
+	go func() {
+		for entry := range h.logCh {
+			logJSON, err := json.Marshal(&entry)
+			if err != nil {
+				continue
+			}
+
+			req, err := http.NewRequest("POST", h.endpoint, bytes.NewBuffer(logJSON))
+			req.Header.Set("Content-Type", "application/json")
+
+			resp, err := h.client.Do(req)
+			if err != nil {
+				continue
+			}
+			if resp.Body != nil {
+				resp.Body.Close()
+			}
+		}
+	}()
+}
+
+// NewHTTP initializes a new logger target which
+// sends log over http to the specified endpoint
+func NewHTTP(endpoint string, transport *http.Transport) LoggingTarget {
+	h := HTTPTarget{
+		endpoint: endpoint,
+		client: http.Client{
+			Transport: transport,
+		},
+		logCh: make(chan logEntry, 10000),
+	}
+
+	h.startHTTPLogger()
+	return &h
+}
+
+func (h *HTTPTarget) send(entry logEntry) error {
+	select {
+	case h.logCh <- entry:
+	default:
+		// log channel is full, do not wait and return
+		// an error immediately to the caller
+		return errors.New("log buffer full")
+	}
+
+	return nil
+}

cmd/logger/logger.go

@@ -299,74 +299,21 @@ func LogIf(ctx context.Context, err error) {
 	// Get the cause for the Error
 	message := err.Error()
 
-	// Output the formatted log message at console
-	var output string
-	if jsonFlag {
-		logJSON, err := json.Marshal(&logEntry{
-			Level:      ErrorLvl.String(),
-			RemoteHost: req.RemoteHost,
-			RequestID:  req.RequestID,
-			UserAgent:  req.UserAgent,
-			Time:       time.Now().UTC().Format(time.RFC3339Nano),
-			API:        &api{Name: API, Args: &args{Bucket: req.BucketName, Object: req.ObjectName}},
-			Trace:      &traceEntry{Message: message, Source: trace, Variables: tags},
-		})
-		if err != nil {
-			panic(err)
-		}
-		output = string(logJSON)
-	} else {
-		// Add a sequence number and formatting for each stack trace
-		// No formatting is required for the first entry
-		for i, element := range trace {
-			trace[i] = fmt.Sprintf("%8v: %s", i+1, element)
-		}
-
-		tagString := ""
-		for key, value := range tags {
-			if value != "" {
-				if tagString != "" {
-					tagString += ", "
-				}
-				tagString += key + "=" + value
-			}
-		}
-
-		apiString := "API: " + API + "("
-		if req.BucketName != "" {
-			apiString = apiString + "bucket=" + req.BucketName
-		}
-		if req.ObjectName != "" {
-			apiString = apiString + ", object=" + req.ObjectName
-		}
-		apiString += ")"
-		timeString := "Time: " + time.Now().Format(loggerTimeFormat)
-
-		var requestID string
-		if req.RequestID != "" {
-			requestID = "\nRequestID: " + req.RequestID
-		}
-
-		var remoteHost string
-		if req.RemoteHost != "" {
-			remoteHost = "\nRemoteHost: " + req.RemoteHost
-		}
-
-		var userAgent string
-		if req.UserAgent != "" {
-			userAgent = "\nUserAgent: " + req.UserAgent
-		}
-
-		if len(tags) > 0 {
-			tagString = "\n       " + tagString
-		}
-
-		var msg = colorFgRed(colorBold(message))
-		output = fmt.Sprintf("\n%s\n%s%s%s%s\nError: %s%s\n%s",
-			apiString, timeString, requestID, remoteHost, userAgent,
-			msg, tagString, strings.Join(trace, "\n"))
+	entry := logEntry{
+		DeploymentID: deploymentID,
+		Level:        ErrorLvl.String(),
+		RemoteHost:   req.RemoteHost,
+		RequestID:    req.RequestID,
+		UserAgent:    req.UserAgent,
+		Time:         time.Now().UTC().Format(time.RFC3339Nano),
+		API:          &api{Name: API, Args: &args{Bucket: req.BucketName, Object: req.ObjectName}},
+		Trace:        &traceEntry{Message: message, Source: trace, Variables: tags},
+	}
+
+	// Iterate over all logger targets to send the log entry
+	for _, t := range Targets {
+		t.send(entry)
 	}
-	fmt.Println(output)
 }
 
 // ErrCritical is the value panic'd whenever CriticalIf is called.

cmd/logger/targets.go

@@ -0,0 +1,33 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package logger
+
+// LoggingTarget is the entity that we will receive
+// a single log entry and send it to the log target
+//   e.g. send the log to a http server
+type LoggingTarget interface {
+	send(entry logEntry) error
+}
+
+// Targets is the set of enabled loggers
+var Targets = []LoggingTarget{}
+
+// AddTarget adds a new logger target to the
+// list of enabled loggers
+func AddTarget(t LoggingTarget) {
+	Targets = append(Targets, t)
+}