MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-09 05:34:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: Add support for kakfa audit logger target (#12678)

Browse Source
This commit is contained in:
Harshavardhana
2021-07-13 09:39:13 -07:00
committed by GitHub
parent 559d075627
commit e316873f84
14 changed files with 811 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
cmd/config-current.go
View File

@@ -43,6 +43,7 @@ import (
"github.com/minio/minio/internal/kms"
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/logger/target/http"
"github.com/minio/minio/internal/logger/target/kafka"
"github.com/minio/pkg/env"
)
@@ -58,7 +59,8 @@ func initHelp() {
config.APISubSys: api.DefaultKVS,
config.CredentialsSubSys: config.DefaultCredentialKVS,
config.LoggerWebhookSubSys: logger.DefaultKVS,
config.AuditWebhookSubSys: logger.DefaultAuditKVS,
config.AuditWebhookSubSys: logger.DefaultAuditWebhookKVS,
config.AuditKafkaSubSys: logger.DefaultAuditKafkaKVS,
config.HealSubSys: heal.DefaultKVS,
config.ScannerSubSys: scanner.DefaultKVS,
}
@@ -122,6 +124,11 @@ func initHelp() {
Description: "send audit logs to webhook endpoints",
MultipleTargets: true,
},
config.HelpKV{
Key: config.AuditKafkaSubSys,
Description: "send audit logs to kafka endpoints",
MultipleTargets: true,
},
config.HelpKV{
Key: config.NotifyWebhookSubSys,
Description: "publish bucket notifications to webhook endpoints",
@@ -197,7 +204,8 @@ func initHelp() {
config.IdentityLDAPSubSys: xldap.Help,
config.PolicyOPASubSys: opa.Help,
config.LoggerWebhookSubSys: logger.Help,
config.AuditWebhookSubSys: logger.HelpAudit,
config.AuditWebhookSubSys: logger.HelpWebhook,
config.AuditKafkaSubSys: logger.HelpKafka,
config.NotifyAMQPSubSys: notify.HelpAMQP,
config.NotifyKafkaSubSys: notify.HelpKafka,
config.NotifyMQTTSubSys: notify.HelpMQTT,
@@ -478,42 +486,40 @@ func lookupConfigs(s config.Config, setDriveCounts []int) {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize logger: %w", err))
}
for k, l := range loggerCfg.HTTP {
for _, l := range loggerCfg.HTTP {
if l.Enabled {
l.LogOnce = logger.LogOnceIf
l.UserAgent = loggerUserAgent
l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
// Enable http logging
if err = logger.AddTarget(
http.New(
http.WithTargetName(k),
http.WithEndpoint(l.Endpoint),
http.WithAuthToken(l.AuthToken),
http.WithUserAgent(loggerUserAgent),
http.WithLogKind(string(logger.All)),
http.WithTransport(NewGatewayHTTPTransport()),
),
); err != nil {
if err = logger.AddTarget(http.New(l)); err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize console HTTP target: %w", err))
}
}
}
for k, l := range loggerCfg.Audit {
for _, l := range loggerCfg.AuditWebhook {
if l.Enabled {
l.LogOnce = logger.LogOnceIf
l.UserAgent = loggerUserAgent
l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
// Enable http audit logging
if err = logger.AddAuditTarget(
http.New(
http.WithTargetName(k),
http.WithEndpoint(l.Endpoint),
http.WithAuthToken(l.AuthToken),
http.WithUserAgent(loggerUserAgent),
http.WithLogKind(string(logger.All)),
http.WithTransport(NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)),
),
); err != nil {
if err = logger.AddAuditTarget(http.New(l)); err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize audit HTTP target: %w", err))
}
}
}
for _, l := range loggerCfg.AuditKafka {
if l.Enabled {
l.LogOnce = logger.LogOnceIf
// Enable Kafka audit logging
if err = logger.AddAuditTarget(kafka.New(l)); err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize audit Kafka target: %w", err))
}
}
}
globalConfigTargetList, err = notify.GetNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), false)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))

7
cmd/config-migrate.go
View File

@@ -41,6 +41,7 @@ import (
"github.com/minio/minio/internal/event/target"
"github.com/minio/minio/internal/kms"
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/logger/target/http"
xnet "github.com/minio/pkg/net"
"github.com/minio/pkg/quick"
)
@@ -2383,8 +2384,8 @@ func migrateV26ToV27() error {
// Enable console logging by default to avoid breaking users
// current deployments
srvConfig.Logger.Console.Enabled = true
srvConfig.Logger.HTTP = make(map[string]logger.HTTP)
srvConfig.Logger.HTTP["1"] = logger.HTTP{}
srvConfig.Logger.HTTP = make(map[string]http.Config)
srvConfig.Logger.HTTP["1"] = http.Config{}
if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
return fmt.Errorf("Failed to migrate config from ‘26’ to ‘27’. %w", err)
@@ -2748,7 +2749,7 @@ func migrateMinioSysConfigToKV(objAPI ObjectLayer) error {
for k, loggerArgs := range cfg.Logger.HTTP {
logger.SetLoggerHTTP(newCfg, k, loggerArgs)
}
for k, auditArgs := range cfg.Logger.Audit {
for k, auditArgs := range cfg.Logger.AuditWebhook {
logger.SetLoggerHTTPAudit(newCfg, k, auditArgs)
}

4
cmd/consolelogger.go
View File

@@ -117,8 +117,8 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh <-chan
sys.pubsub.Subscribe(subCh, doneCh, filter)
}
// Validate if HTTPConsoleLoggerSys is valid, always returns nil right now
func (sys *HTTPConsoleLoggerSys) Validate() error {
// Init if HTTPConsoleLoggerSys is valid, always returns nil right now
func (sys *HTTPConsoleLoggerSys) Init() error {
return nil
}

2
cmd/data-update-tracker_test.go
View File

@@ -51,7 +51,7 @@ func (t *testingLogger) String() string {
return ""
}
func (t *testingLogger) Validate() error {
func (t *testingLogger) Init() error {
return nil
}