Fix etcd watch regression in IAM subsystem (#8224)

Fixes #8223

cmd/iam-etcd-store.go

@@ -503,28 +503,35 @@ func (ies *IAMEtcdStore) deleteGroupInfo(name string) error {
 
 func (ies *IAMEtcdStore) watch(sys *IAMSys) {
 	watchEtcd := func() {
-		// Refresh IAMSys with etcd watch.
 		for {
+		outerLoop:
+			// Refresh IAMSys with etcd watch.
 			watchCh := ies.client.Watch(context.Background(),
 				iamConfigPrefix, etcd.WithPrefix(), etcd.WithKeysOnly())
-			select {
+			for {
-			case <-GlobalServiceDoneCh:
+				select {
-				return
+				case <-GlobalServiceDoneCh:
-			case watchResp, ok := <-watchCh:
+					return
-				if !ok {
+				case watchResp, ok := <-watchCh:
-					time.Sleep(1 * time.Second)
+					if !ok {
-					continue
+						time.Sleep(1 * time.Second)
-				}
+						// Upon an error on watch channel
-				if err := watchResp.Err(); err != nil {
+						// re-init the watch channel.
-					logger.LogIf(context.Background(), err)
+						goto outerLoop
-					// log and retry.
+					}
-					time.Sleep(1 * time.Second)
+					if err := watchResp.Err(); err != nil {
-					continue
+						logger.LogIf(context.Background(), err)
-				}
+						// log and retry.
-				for _, event := range watchResp.Events {
+						time.Sleep(1 * time.Second)
-					sys.Lock()
+						// Upon an error on watch channel
-					ies.reloadFromEvent(sys, event)
+						// re-init the watch channel.
-					sys.Unlock()
+						goto outerLoop
+					}
+					for _, event := range watchResp.Events {
+						sys.Lock()
+						ies.reloadFromEvent(sys, event)
+						sys.Unlock()
+					}
 				}
 			}
 		}