fix: select always online peers for remote listing (#11153)

always find the right set of online peers for remote listing,
this may have an effect on listing if the server is down - we
should do this to avoid always performing transient operations
on bucket->peerClient that is permanently or down for a long
period.

cmd/notification.go

@@ -1258,14 +1258,26 @@ func (sys *NotificationSys) GetLocalDiskIDs(ctx context.Context) (localDiskIDs [
 	return localDiskIDs
 }
 
+// returns all the peers that are currently online.
+func (sys *NotificationSys) getOnlinePeers() []*peerRESTClient {
+	var peerClients []*peerRESTClient
+	for _, peerClient := range sys.allPeerClients {
+		if peerClient != nil && peerClient.IsOnline() {
+			peerClients = append(peerClients, peerClient)
+		}
+	}
+	return peerClients
+}
+
 // restClientFromHash will return a deterministic peerRESTClient based on s.
 // Will return nil if client is local.
 func (sys *NotificationSys) restClientFromHash(s string) (client *peerRESTClient) {
 	if len(sys.peerClients) == 0 {
 		return nil
 	}
-	idx := xxhash.Sum64String(s) % uint64(len(sys.allPeerClients))
-	return sys.allPeerClients[idx]
+	peerClients := sys.getOnlinePeers()
+	idx := xxhash.Sum64String(s) % uint64(len(peerClients))
+	return peerClients[idx]
 }
 
 // NewNotificationSys - creates new notification system object.

cmd/peer-rest-client.go

@@ -78,6 +78,11 @@ func (client *peerRESTClient) String() string {
 	return client.host.String()
 }
 
+// IsOnline returns true if the peer client is online.
+func (client *peerRESTClient) IsOnline() bool {
+	return client.restClient.IsOnline()
+}
+
 // Close - marks the client as closed.
 func (client *peerRESTClient) Close() error {
 	client.restClient.Close()