retry each object decom upto 3 times, in-case of failure (#14861)

cmd/erasure-server-pool-decom.go

@@ -687,6 +687,9 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					continue
 				}
 
+				var failure bool
+				// gr.Close() is ensured by decommissionObject().
+				for try := 0; try < 3; try++ {
 					gr, err := set.GetObjectNInfo(ctx,
 						bName,
 						encodeDirObject(version.Name),
@@ -697,17 +700,17 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 							VersionID: version.VersionID,
 						})
 					if err != nil {
-					logger.LogIf(ctx, err)
-					z.poolMetaMutex.Lock()
-					z.poolMeta.CountItem(idx, version.Size, true)
-					z.poolMetaMutex.Unlock()
-					break // break out on first error
-				}
-				var failure bool
-				// gr.Close() is ensured by decommissionObject().
-				if err = z.decommissionObject(ctx, bName, gr); err != nil {
-					logger.LogIf(ctx, err)
 						failure = true
+						logger.LogIf(ctx, err)
+						continue
+					}
+					if err = z.decommissionObject(ctx, bName, gr); err != nil {
+						failure = true
+						logger.LogIf(ctx, err)
+						continue
+					}
+					failure = false
+					break
 				}
 				z.poolMetaMutex.Lock()
 				z.poolMeta.CountItem(idx, version.Size, failure)