From 41cdb357bb7de2cc832c332b9a9f5174bd924c19 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Tue, 24 May 2022 18:57:14 -0700 Subject: [PATCH] Compensate for different server pool sizes (#14968) When a server pool with a different number of sets is added they are not compensated when choosing a destination pool for new objects. This leads to the unbalanced placement of objects with smaller pools getting a bigger number of objects since we only compare the destination sets directly. This change will compensate for differences in set sizes when choosing the destination pool. Different set sizes are already compensated by fewer disks. --- cmd/erasure-server-pool.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index 03af7844e..c6499c677 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -270,11 +270,13 @@ func (z *erasureServerPools) getAvailablePoolIdx(ctx context.Context, bucket, ob // getServerPoolsAvailableSpace will return the available space of each pool after storing the content. // If there is not enough space the pool will return 0 bytes available. +// The size of each will be multiplied by the number of sets. // Negative sizes are seen as 0 bytes. func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, bucket, object string, size int64) serverPoolsAvailableSpace { serverPools := make(serverPoolsAvailableSpace, len(z.serverPools)) storageInfos := make([][]*DiskInfo, len(z.serverPools)) + nSets := make([]int, len(z.serverPools)) g := errgroup.WithNErrs(len(z.serverPools)) for index := range z.serverPools { index := index @@ -282,9 +284,11 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, b if z.IsSuspended(index) { continue } + pool := z.serverPools[index] + nSets[index] = pool.setCount g.Go(func() error { // Get the set where it would be placed. - storageInfos[index] = getDiskInfos(ctx, z.serverPools[index].getHashedSet(object).getDisks()) + storageInfos[index] = getDiskInfos(ctx, pool.getHashedSet(object).getDisks()) return nil }, index) } @@ -304,6 +308,14 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, b } available += disk.Total - disk.Used } + + // Since we are comparing pools that may have a different number of sets + // we multiply by the number of sets in the pool. + // This will compensate for differences in set sizes + // when choosing destination pool. + // Different set sizes are already compensated by less disks. + available *= uint64(nSets[i]) + serverPools[i] = poolAvailableSpace{ Index: i, Available: available,