From 7d290302920fdbd10466a458fdda44c21db73bc6 Mon Sep 17 00:00:00 2001 From: Poorna Date: Wed, 22 May 2024 16:16:34 -0700 Subject: [PATCH] fix list results returned for spark max-keys=2 listing (#19791) This PR continues fix #19725 for some unhandled cases --- cmd/erasure-server-pool.go | 11 +++++++--- cmd/server_test.go | 45 ++++++++++++++++++++++++++++++++------ 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index cc21cee5c..16b380547 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -1647,12 +1647,17 @@ func (z *erasureServerPools) listObjectsGeneric(ctx context.Context, bucket, pre } } } - if o.DeleteMarker { + switch { + case o.DeleteMarker: loi.Objects = append(loi.Objects, ObjectInfo{Bucket: bucket, IsDir: true, Name: prefix}) return loi, nil - } else if len(li.Objects) == 1 { + case len(li.Objects) >= 1: loi.Objects = append(loi.Objects, o) - loi.Prefixes = append(loi.Prefixes, path.Join(opts.Prefix, pfx)) + if pfx != "" { + loi.Prefixes = append(loi.Prefixes, path.Join(opts.Prefix, pfx)) + } + case len(li.Prefixes) > 0: + loi.Prefixes = append(loi.Prefixes, li.Prefixes...) } } return loi, nil diff --git a/cmd/server_test.go b/cmd/server_test.go index 100663702..037a3ea95 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -1671,7 +1671,7 @@ func (s *TestSuiteCommon) TestListObjectsV2HadoopUAHandler(c *check) { c.Assert(err, nil) c.Assert(response.StatusCode, http.StatusOK) - for _, objectName := range []string{"pfx/a/1.txt", "pfx/b/2.txt", "pfx/", "pfx2/c/3.txt", "pfx2/d/3.txt", "pfx1/1.txt", "pfx2/"} { + for _, objectName := range []string{"pfx/a/1.txt", "pfx/b/2.txt", "pfx/", "pfx2/c/3.txt", "pfx2/d/3.txt", "pfx1/1.txt", "pfx2/", "pfx3/", "pfx4/"} { buffer := bytes.NewReader([]byte("")) request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), int64(buffer.Len()), buffer, s.accessKey, s.secretKey, s.signer) @@ -1680,7 +1680,7 @@ func (s *TestSuiteCommon) TestListObjectsV2HadoopUAHandler(c *check) { c.Assert(err, nil) c.Assert(response.StatusCode, http.StatusOK) } - for _, objectName := range []string{"pfx2/c/3.txt", "pfx2/d/3.txt", "pfx2/"} { + for _, objectName := range []string{"pfx2/c/3.txt", "pfx2/d/3.txt", "pfx2/", "pfx3/"} { delRequest, err := newTestSignedRequest(http.MethodDelete, getDeleteObjectURL(s.endPoint, bucketName, objectName), 0, nil, s.accessKey, s.secretKey, s.signer) c.Assert(err, nil) @@ -1697,7 +1697,6 @@ func (s *TestSuiteCommon) TestListObjectsV2HadoopUAHandler(c *check) { getListObjectsV2URL(s.endPoint, bucketName, "pfx/a/", "2", "", "", "/"), []string{ "pfx/a/1.txt", - "pfx/a", }, "Hadoop 3.3.2, aws-sdk-java/1.12.262 Linux/5.14.0-362.24.1.el9_3.x86_64 OpenJDK_64-Bit_Server_VM/11.0.22+7 java/11.0.22 scala/2.12.15 vendor/Eclipse_Adoptium cfg/retry-mode/legacy", }, @@ -1720,21 +1719,24 @@ func (s *TestSuiteCommon) TestListObjectsV2HadoopUAHandler(c *check) { { getListObjectsV2URL(s.endPoint, bucketName, "pfx2/c", "2", "true", "", "/"), []string{ - "pfx2/c02/false", + "pfx2/c12/false", + "pfx2/c/", }, "Hadoop 3.3.2, aws-sdk-java/1.12.262 Linux/5.14.0-362.24.1.el9_3.x86_64 OpenJDK_64-Bit_Server_VM/11.0.22+7 java/11.0.22 scala/2.12.15 vendor/Eclipse_Adoptium cfg/retry-mode/legacy", }, { getListObjectsV2URL(s.endPoint, bucketName, "pfx2/c", "2", "true", "", "/"), []string{ - "pfx2/c02/false", + "pfx2/c12/false", + "pfx2/c/", }, "Hadoop 3.3.2, aws-sdk-java/1.12.262 Linux/5.14.0-362.24.1.el9_3.x86_64 OpenJDK_64-Bit_Server_VM/11.0.22+7 java/11.0.22 scala/2.12.15 vendor/Eclipse_Adoptium cfg/retry-mode/legacy", }, { getListObjectsV2URL(s.endPoint, bucketName, "pfx2/", "2", "false", "", "/"), []string{ - "pfx2/02/false", + "pfx2/22/false", + "pfx2/c/pfx2/d/", }, "Hadoop 3.3.2, aws-sdk-java/1.12.262 Linux/5.14.0-362.24.1.el9_3.x86_64 OpenJDK_64-Bit_Server_VM/11.0.22+7 java/11.0.22 scala/2.12.15 vendor/Eclipse_Adoptium cfg/retry-mode/legacy", }, @@ -1748,10 +1750,39 @@ func (s *TestSuiteCommon) TestListObjectsV2HadoopUAHandler(c *check) { { getListObjectsV2URL(s.endPoint, bucketName, "pfx2/", "2", "false", "", "/"), []string{ - "pfx2/02/false", + "pfx2/22/false", + "pfx2/c/pfx2/d/", }, "Hadoop 3.3.2, aws-sdk-java/1.12.262 Linux/5.14.0-362.24.1.el9_3.x86_64 OpenJDK_64-Bit_Server_VM/11.0.22+7 java/11.0.22 scala/2.12.15 vendor/Eclipse_Adoptium cfg/retry-mode/legacy", }, + { + getListObjectsV2URL(s.endPoint, bucketName, "pfx3/", "2", "false", "", "/"), + []string{ + "pfx3/02/false", + }, + "Hadoop 3.3.2, aws-sdk-java/1.12.262 Linux/5.14.0-362.24.1.el9_3.x86_64 OpenJDK_64-Bit_Server_VM/11.0.22+7 java/11.0.22 scala/2.12.15 vendor/Eclipse_Adoptium cfg/retry-mode/legacy", + }, + { + getListObjectsV2URL(s.endPoint, bucketName, "pfx4/", "2", "false", "", "/"), + []string{ + "pfx4/12/falsepfx4/", + }, + "Hadoop 3.3.2, aws-sdk-java/1.12.262 Linux/5.14.0-362.24.1.el9_3.x86_64 OpenJDK_64-Bit_Server_VM/11.0.22+7 java/11.0.22 scala/2.12.15 vendor/Eclipse_Adoptium cfg/retry-mode/legacy", + }, + { + getListObjectsV2URL(s.endPoint, bucketName, "pfx3/", "2", "false", "", "/"), + []string{ + "pfx3/02/false", + }, + "", + }, + { + getListObjectsV2URL(s.endPoint, bucketName, "pfx4/", "2", "false", "", "/"), + []string{ + "pfx4/12/falsepfx4/", + }, + "", + }, } for _, testCase := range testCases { // create listObjectsV1 request with valid parameters