tune-kafka targets to ensure timeout triggers on hung brokers (#17898)

hung brokers can cause slowness to the entire system
when many callers are hung, leading to large goroutine
build-up.

internal/logger/target/kafka/kafka.go

@@ -32,8 +32,8 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/Shopify/sarama"
-	saramatls "github.com/Shopify/sarama/tools/tls"
+	"github.com/IBM/sarama"
+	saramatls "github.com/IBM/sarama/tools/tls"
 	"github.com/tidwall/gjson"
 
 	"github.com/minio/minio/internal/logger/target/types"
@@ -287,9 +287,20 @@ func (h *Target) init() error {
 	sconfig.Net.TLS.Config.ClientAuth = h.kconfig.TLS.ClientAuth
 	sconfig.Net.TLS.Config.RootCAs = h.kconfig.TLS.RootCAs
 
-	sconfig.Producer.RequiredAcks = sarama.WaitForAll
-	sconfig.Producer.Retry.Max = 10
+	// These settings are needed to ensure that kafka client doesn't hang on brokers
+	// refer https://github.com/IBM/sarama/issues/765#issuecomment-254333355
+	sconfig.Producer.Retry.Max = 2
+	sconfig.Producer.Retry.Backoff = (10 * time.Second)
 	sconfig.Producer.Return.Successes = true
+	sconfig.Producer.Return.Errors = true
+	sconfig.Producer.RequiredAcks = 1
+	sconfig.Producer.Timeout = (10 * time.Second)
+	sconfig.Net.ReadTimeout = (10 * time.Second)
+	sconfig.Net.DialTimeout = (10 * time.Second)
+	sconfig.Net.WriteTimeout = (10 * time.Second)
+	sconfig.Metadata.Retry.Max = 1
+	sconfig.Metadata.Retry.Backoff = (10 * time.Second)
+	sconfig.Metadata.RefreshFrequency = (15 * time.Minute)
 
 	h.config = sconfig
 

internal/logger/target/kafka/kafka_scram_client_contrib.go

@@ -20,7 +20,7 @@ package kafka
 import (
 	"crypto/sha512"
 
-	"github.com/Shopify/sarama"
+	"github.com/IBM/sarama"
 	"github.com/xdg/scram"
 
 	"github.com/minio/minio/internal/hash/sha256"