MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-01-11 23:13:23 -05:00
Code Issues Packages Projects Releases Wiki Activity

move HTTP recorder to an internal library (#16128)

Browse Source
This commit is contained in:
Anis Elleuch 2022-11-28 19:20:27 +01:00 committed by GitHub
parent 98a67a3776
commit 1f1dcdce65
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 215 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/generic-handlers.go
View File

@ -549,7 +549,7 @@ func addCustomHeaders(h http.Handler) http.Handler {
// part of the log entry, Error response XML and auditing. // part of the log entry, Error response XML and auditing.
// Set custom headers such as x-amz-request-id for each request. // Set custom headers such as x-amz-request-id for each request.
w.Header().Set(xhttp.AmzRequestID, mustGetRequestID(UTCNow())) w.Header().Set(xhttp.AmzRequestID, mustGetRequestID(UTCNow()))
h.ServeHTTP(logger.NewResponseWriter(w), r) h.ServeHTTP(xhttp.NewResponseRecorder(w), r)
}) })
} }

2
cmd/handler-utils.go
View File

@ -350,7 +350,7 @@ func collectAPIStats(api string, f http.HandlerFunc) http.HandlerFunc {
globalHTTPStats.currentS3Requests.Inc(api) globalHTTPStats.currentS3Requests.Inc(api)
defer globalHTTPStats.currentS3Requests.Dec(api) defer globalHTTPStats.currentS3Requests.Dec(api)
statsWriter := logger.NewResponseWriter(w) statsWriter := xhttp.NewResponseRecorder(w)
f.ServeHTTP(statsWriter, r) f.ServeHTTP(statsWriter, r)

4
cmd/http-stats.go
View File

@ -23,7 +23,7 @@ import (
"sync" "sync"
"sync/atomic" "sync/atomic"
"github.com/minio/minio/internal/logger" xhttp "github.com/minio/minio/internal/http"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
) )
@ -295,7 +295,7 @@ func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats {
} }
// Update statistics from http request and response data // Update statistics from http request and response data
func (st *HTTPStats) updateStats(api string, r *http.Request, w *logger.ResponseWriter) { func (st *HTTPStats) updateStats(api string, r *http.Request, w *xhttp.ResponseRecorder) {
// Ignore non S3 requests // Ignore non S3 requests
if strings.HasSuffix(r.URL.Path, minioReservedBucketPathWithSlash) { if strings.HasSuffix(r.URL.Path, minioReservedBucketPathWithSlash) {
return return

59
cmd/http-tracer.go
View File

@ -18,9 +18,7 @@
package cmd package cmd
import ( import (
"bytes"
"context" "context"
"io"
"net" "net"
"net/http" "net/http"
"reflect" "reflect"
@ -32,54 +30,9 @@ import (
"github.com/minio/madmin-go" "github.com/minio/madmin-go"
"github.com/minio/minio/internal/handlers" "github.com/minio/minio/internal/handlers"
"github.com/minio/minio/internal/logger" xhttp "github.com/minio/minio/internal/http"
) )
// recordRequest - records the first recLen bytes
// of a given io.Reader
type recordRequest struct {
// Data source to record
io.Reader
// Response body should be logged
logBody bool
// Internal recording buffer
buf bytes.Buffer
// total bytes read including header size
bytesRead int
}
func (r *recordRequest) Close() error {
// no-op
return nil
}
func (r *recordRequest) Read(p []byte) (n int, err error) {
n, err = r.Reader.Read(p)
r.bytesRead += n
if r.logBody {
r.buf.Write(p[:n])
}
if err != nil {
return n, err
}
return n, err
}
func (r *recordRequest) BodySize() int {
return r.bytesRead
}
// Return the bytes that were recorded.
func (r *recordRequest) Data() []byte {
// If body logging is enabled then we return the actual body
if r.logBody {
return r.buf.Bytes()
}
// ... otherwise we return <BODY> placeholder
return logger.BodyPlaceHolder
}
var ldapPwdRegex = regexp.MustCompile("(^.*?)LDAPPassword=([^&]*?)(&(.*?))?$") var ldapPwdRegex = regexp.MustCompile("(^.*?)LDAPPassword=([^&]*?)(&(.*?))?$")
// redact LDAP password if part of string // redact LDAP password if part of string
@ -116,8 +69,8 @@ const contextTraceReqKey = contextTraceReqType("request-trace-info")
// Hold related tracing data of a http request, any handler // Hold related tracing data of a http request, any handler
// can modify this struct to modify the trace information . // can modify this struct to modify the trace information .
type traceCtxt struct { type traceCtxt struct {
requestRecorder *recordRequest requestRecorder *xhttp.RequestRecorder
responseRecorder *logger.ResponseWriter responseRecorder *xhttp.ResponseRecorder
funcName string funcName string
} }
@ -136,8 +89,8 @@ func httpTracer(h http.Handler) http.Handler {
r = r.WithContext(ctx) r = r.WithContext(ctx)
// Setup a http request and response body recorder // Setup a http request and response body recorder
reqRecorder := &recordRequest{Reader: r.Body} reqRecorder := &xhttp.RequestRecorder{Reader: r.Body}
respRecorder := logger.NewResponseWriter(w) respRecorder := xhttp.NewResponseRecorder(w)
tc.requestRecorder = reqRecorder tc.requestRecorder = reqRecorder
tc.responseRecorder = respRecorder tc.responseRecorder = respRecorder
@ -240,7 +193,7 @@ func httpTrace(f http.HandlerFunc, logBody bool) http.HandlerFunc {
} }
tc.funcName = getOpName(runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()) tc.funcName = getOpName(runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name())
tc.requestRecorder.logBody = logBody tc.requestRecorder.LogBody = logBody
tc.responseRecorder.LogAllBody = logBody tc.responseRecorder.LogAllBody = logBody
tc.responseRecorder.LogErrBody = true tc.responseRecorder.LogErrBody = true

2
cmd/object-handlers.go
View File

@ -3542,7 +3542,7 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
return return
} }
nr := httptest.NewRecorder() nr := httptest.NewRecorder()
rw := logger.NewResponseWriter(nr) rw := xhttp.NewResponseRecorder(nr)
rw.LogErrBody = true rw.LogErrBody = true
rw.LogAllBody = true rw.LogAllBody = true
rreq.SelectParameters.Evaluate(rw) rreq.SelectParameters.Evaluate(rw)

72
internal/http/request-recorder.go Normal file
View File

@ -0,0 +1,72 @@
// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package http
import (
"bytes"
"io"
)
// RequestRecorder - records the
// of a given io.Reader
type RequestRecorder struct {
// Data source to record
io.Reader
// Response body should be logged
LogBody bool
// internal recording buffer
buf bytes.Buffer
// total bytes read including header size
bytesRead int
}
// Close is a no operation closer
func (r *RequestRecorder) Close() error {
// no-op
return nil
}
// Read reads from the internal reader and counts/save the body in the memory
func (r *RequestRecorder) Read(p []byte) (n int, err error) {
n, err = r.Reader.Read(p)
r.bytesRead += n
if r.LogBody {
r.buf.Write(p[:n])
}
if err != nil {
return n, err
}
return n, err
}
// BodySize returns the body size if the currently read object
func (r *RequestRecorder) BodySize() int {
return r.bytesRead
}
// Data returns the bytes that were recorded.
func (r *RequestRecorder) Data() []byte {
// If body logging is enabled then we return the actual body
if r.LogBody {
return r.buf.Bytes()
}
// ... otherwise we return <BODY> placeholder
return BodyPlaceHolder
}

129
internal/http/response-recorder.go Normal file
View File

@ -0,0 +1,129 @@
// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package http
import (
"bytes"
"fmt"
"io"
"net/http"
"time"
)
// ResponseRecorder - is a wrapper to trap the http response
// status code and to record the response body
type ResponseRecorder struct {
http.ResponseWriter
StatusCode int
// Log body of 4xx or 5xx responses
LogErrBody bool
// Log body of all responses
LogAllBody bool
TimeToFirstByte time.Duration
StartTime time.Time
// number of bytes written
bytesWritten int
// number of bytes of response headers written
headerBytesWritten int
// Internal recording buffer
headers bytes.Buffer
body bytes.Buffer
// Indicate if headers are written in the log
headersLogged bool
}
// NewResponseRecorder - returns a wrapped response writer to trap
// http status codes for auditing purposes.
func NewResponseRecorder(w http.ResponseWriter) *ResponseRecorder {
return &ResponseRecorder{
ResponseWriter: w,
StatusCode: http.StatusOK,
StartTime: time.Now().UTC(),
}
}
func (lrw *ResponseRecorder) Write(p []byte) (int, error) {
if !lrw.headersLogged {
// We assume the response code to be '200 OK' when WriteHeader() is not called,
// that way following Golang HTTP response behavior.
lrw.WriteHeader(http.StatusOK)
}
n, err := lrw.ResponseWriter.Write(p)
lrw.bytesWritten += n
if lrw.TimeToFirstByte == 0 {
lrw.TimeToFirstByte = time.Now().UTC().Sub(lrw.StartTime)
}
if (lrw.LogErrBody && lrw.StatusCode >= http.StatusBadRequest) || lrw.LogAllBody {
// Always logging error responses.
lrw.body.Write(p)
}
if err != nil {
return n, err
}
return n, err
}
// Write the headers into the given buffer
func (lrw *ResponseRecorder) writeHeaders(w io.Writer, statusCode int, headers http.Header) {
n, _ := fmt.Fprintf(w, "%d %s\n", statusCode, http.StatusText(statusCode))
lrw.headerBytesWritten += n
for k, v := range headers {
n, _ := fmt.Fprintf(w, "%s: %s\n", k, v[0])
lrw.headerBytesWritten += n
}
}
// BodyPlaceHolder returns a dummy body placeholder
var BodyPlaceHolder = []byte("<BODY>")
// Body - Return response body.
func (lrw *ResponseRecorder) Body() []byte {
// If there was an error response or body logging is enabled
// then we return the body contents
if (lrw.LogErrBody && lrw.StatusCode >= http.StatusBadRequest) || lrw.LogAllBody {
return lrw.body.Bytes()
}
// ... otherwise we return the <BODY> place holder
return BodyPlaceHolder
}
// WriteHeader - writes http status code
func (lrw *ResponseRecorder) WriteHeader(code int) {
if !lrw.headersLogged {
lrw.StatusCode = code
lrw.writeHeaders(&lrw.headers, code, lrw.ResponseWriter.Header())
lrw.headersLogged = true
lrw.ResponseWriter.WriteHeader(code)
}
}
// Flush - Calls the underlying Flush.
func (lrw *ResponseRecorder) Flush() {
lrw.ResponseWriter.(http.Flusher).Flush()
}
// Size - returns the number of bytes written
func (lrw *ResponseRecorder) Size() int {
return lrw.bytesWritten
}
// HeaderSize - returns the number of bytes of response headers written
func (lrw *ResponseRecorder) HeaderSize() int {
return lrw.headerBytesWritten
}

110
internal/logger/audit.go
View File

@ -18,10 +18,8 @@
package logger package logger
import ( import (
"bytes"
"context" "context"
"fmt" "fmt"
"io"
"net/http" "net/http"
"strconv" "strconv"
"time" "time"
@ -32,108 +30,6 @@ import (
"github.com/minio/minio/internal/logger/message/audit" "github.com/minio/minio/internal/logger/message/audit"
) )
// ResponseWriter - is a wrapper to trap the http response status code.
type ResponseWriter struct {
http.ResponseWriter
StatusCode int
// Log body of 4xx or 5xx responses
LogErrBody bool
// Log body of all responses
LogAllBody bool
TimeToFirstByte time.Duration
StartTime time.Time
// number of bytes written
bytesWritten int
// number of bytes of response headers written
headerBytesWritten int
// Internal recording buffer
headers bytes.Buffer
body bytes.Buffer
// Indicate if headers are written in the log
headersLogged bool
}
// NewResponseWriter - returns a wrapped response writer to trap
// http status codes for auditing purposes.
func NewResponseWriter(w http.ResponseWriter) *ResponseWriter {
return &ResponseWriter{
ResponseWriter: w,
StatusCode: http.StatusOK,
StartTime: time.Now().UTC(),
}
}
func (lrw *ResponseWriter) Write(p []byte) (int, error) {
if !lrw.headersLogged {
// We assume the response code to be '200 OK' when WriteHeader() is not called,
// that way following Golang HTTP response behavior.
lrw.WriteHeader(http.StatusOK)
}
n, err := lrw.ResponseWriter.Write(p)
lrw.bytesWritten += n
if lrw.TimeToFirstByte == 0 {
lrw.TimeToFirstByte = time.Now().UTC().Sub(lrw.StartTime)
}
if (lrw.LogErrBody && lrw.StatusCode >= http.StatusBadRequest) || lrw.LogAllBody {
// Always logging error responses.
lrw.body.Write(p)
}
if err != nil {
return n, err
}
return n, err
}
// Write the headers into the given buffer
func (lrw *ResponseWriter) writeHeaders(w io.Writer, statusCode int, headers http.Header) {
n, _ := fmt.Fprintf(w, "%d %s\n", statusCode, http.StatusText(statusCode))
lrw.headerBytesWritten += n
for k, v := range headers {
n, _ := fmt.Fprintf(w, "%s: %s\n", k, v[0])
lrw.headerBytesWritten += n
}
}
// BodyPlaceHolder returns a dummy body placeholder
var BodyPlaceHolder = []byte("<BODY>")
// Body - Return response body.
func (lrw *ResponseWriter) Body() []byte {
// If there was an error response or body logging is enabled
// then we return the body contents
if (lrw.LogErrBody && lrw.StatusCode >= http.StatusBadRequest) || lrw.LogAllBody {
return lrw.body.Bytes()
}
// ... otherwise we return the <BODY> place holder
return BodyPlaceHolder
}
// WriteHeader - writes http status code
func (lrw *ResponseWriter) WriteHeader(code int) {
if !lrw.headersLogged {
lrw.StatusCode = code
lrw.writeHeaders(&lrw.headers, code, lrw.ResponseWriter.Header())
lrw.headersLogged = true
lrw.ResponseWriter.WriteHeader(code)
}
}
// Flush - Calls the underlying Flush.
func (lrw *ResponseWriter) Flush() {
lrw.ResponseWriter.(http.Flusher).Flush()
}
// Size - returns the number of bytes written
func (lrw *ResponseWriter) Size() int {
return lrw.bytesWritten
}
// HeaderSize - returns the number of bytes of response headers written
func (lrw *ResponseWriter) HeaderSize() int {
return lrw.headerBytesWritten
}
const contextAuditKey = contextKeyType("audit-entry") const contextAuditKey = contextKeyType("audit-entry")
// SetAuditEntry sets Audit info in the context. // SetAuditEntry sets Audit info in the context.
@ -197,13 +93,13 @@ func AuditLog(ctx context.Context, w http.ResponseWriter, r *http.Request, reqCl
headerBytes int64 headerBytes int64
) )
var st *ResponseWriter var st *xhttp.ResponseRecorder
switch v := w.(type) { switch v := w.(type) {
case *ResponseWriter: case *xhttp.ResponseRecorder:
st = v st = v
case *gzhttp.GzipResponseWriter: case *gzhttp.GzipResponseWriter:
// the writer may be obscured by gzip response writer // the writer may be obscured by gzip response writer
if rw, ok := v.ResponseWriter.(*ResponseWriter); ok { if rw, ok := v.ResponseWriter.(*xhttp.ResponseRecorder); ok {
st = rw st = rw
} }
} }