From d45a8784fc5a1c75c788e32e8c80423f0e24f9f0 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 30 Nov 2017 12:57:03 -0800 Subject: [PATCH] Fix hash order to generate more even distribution (#5247) The problem in existing code was the following line ``` start := int(keyCrc%uint32(cardinality)) | 1 ``` A given a value of N cardinality the ending result because of the the bitwise '|' would lead to always higher affinity to odd sequences. As can be seen from the test cases that this can lead to many objects being allocated the same set of disks or atleast the first disk is an odd disk always. This introduces a performance problem for majority of the objects under concurrent load. Remove `| 1` to provide a more cleaner distribution and the new code will be. ``` start := int(keyCrc % uint32(cardinality)) ``` Thanks to Krishna Srinivas for pointing out the bitwise situation here. --- cmd/xl-v1-utils.go | 2 +- cmd/xl-v1-utils_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/xl-v1-utils.go b/cmd/xl-v1-utils.go index 92124667c..1a8062716 100644 --- a/cmd/xl-v1-utils.go +++ b/cmd/xl-v1-utils.go @@ -118,7 +118,7 @@ func hashOrder(key string, cardinality int) []int { nums := make([]int, cardinality) keyCrc := crc32.Checksum([]byte(key), crc32.IEEETable) - start := int(keyCrc%uint32(cardinality)) | 1 + start := int(keyCrc % uint32(cardinality)) for i := 1; i <= cardinality; i++ { nums[i-1] = 1 + ((start + i) % cardinality) } diff --git a/cmd/xl-v1-utils_test.go b/cmd/xl-v1-utils_test.go index 5c73f8495..8d9e16210 100644 --- a/cmd/xl-v1-utils_test.go +++ b/cmd/xl-v1-utils_test.go @@ -110,14 +110,14 @@ func TestHashOrder(t *testing.T) { }{ // cases which should pass the test. // passing in valid object name. - {"object", []int{15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}}, - {"The Shining Script .pdf", []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}}, + {"object", []int{14, 15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}}, + {"The Shining Script .pdf", []int{16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}}, {"Cost Benefit Analysis (2009-2010).pptx", []int{15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}}, {"117Gn8rfHL2ACARPAhaFd0AGzic9pUbIA/5OCn5A", []int{3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2}}, {"SHØRT", []int{11, 12, 13, 14, 15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}}, {"There are far too many object names, and far too few bucket names!", []int{15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}}, {"a/b/c/", []int{3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2}}, - {"/a/b/c", []int{7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2, 3, 4, 5, 6}}, + {"/a/b/c", []int{6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2, 3, 4, 5}}, {string([]byte{0xff, 0xfe, 0xfd}), []int{15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}}, }