fix: re-use connections in webhook/elasticsearch (#9461)

- elasticsearch client should rely on the SDK helpers
  instead of pure HTTP calls.
- webhook shouldn't need to check for IsActive() for
  all notifications, failure should be delayed.
- Remove DialHTTP as its never used properly

Fixes #9460

pkg/event/target/webhook.go

@@ -28,6 +28,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/minio/minio/pkg/event"
 	xnet "github.com/minio/minio/pkg/net"
@@ -94,16 +95,27 @@ func (target *WebhookTarget) HasQueueStore() bool {
 
 // IsActive - Return true if target is up and active
 func (target *WebhookTarget) IsActive() (bool, error) {
-	u, pErr := xnet.ParseHTTPURL(target.args.Endpoint.String())
-	if pErr != nil {
-		return false, pErr
-	}
-	if dErr := u.DialHTTP(nil); dErr != nil {
-		if xnet.IsNetworkOrHostDown(dErr) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	req, err := http.NewRequest(http.MethodHead, target.args.Endpoint.String(), nil)
+	if err != nil {
+		if xnet.IsNetworkOrHostDown(err) {
 			return false, errNotConnected
 		}
-		return false, dErr
+		return false, err
 	}
+
+	resp, err := target.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		if xnet.IsNetworkOrHostDown(err) || err == context.DeadlineExceeded {
+			return false, errNotConnected
+		}
+		return false, err
+	}
+	io.Copy(ioutil.Discard, resp.Body)
+	resp.Body.Close()
+	// No network failure i.e response from the target means its up
 	return true, nil
 }
 
@@ -112,11 +124,13 @@ func (target *WebhookTarget) Save(eventData event.Event) error {
 	if target.store != nil {
 		return target.store.Put(eventData)
 	}
-	_, err := target.IsActive()
+	err := target.send(eventData)
 	if err != nil {
-		return err
+		if xnet.IsNetworkOrHostDown(err) {
+			return errNotConnected
+		}
 	}
-	return target.send(eventData)
+	return err
 }
 
 // send - sends an event to the webhook.
@@ -161,10 +175,6 @@ func (target *WebhookTarget) send(eventData event.Event) error {
 
 // Send - reads an event from store and sends it to webhook.
 func (target *WebhookTarget) Send(eventKey string) error {
-	_, err := target.IsActive()
-	if err != nil {
-		return err
-	}
 	eventData, eErr := target.store.Get(eventKey)
 	if eErr != nil {
 		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
@@ -217,7 +227,8 @@ func NewWebhookTarget(id string, args WebhookArgs, doneCh <-chan struct{}, logge
 		target.store = store
 	}
 
-	if _, err := target.IsActive(); err != nil {
+	_, err := target.IsActive()
+	if err != nil {
 		if target.store == nil || err != errNotConnected {
 			target.loggerOnce(context.Background(), err, target.ID())
 			return target, err