MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-23 21:55:53 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: iso8601TimeFormat padding issue for certain nanoseconds (#16207)

Browse Source
This commit is contained in:
Harshavardhana 2022-12-12 10:28:30 -08:00 committed by GitHub
parent a2cbeaa9e6
commit 2fc182d8e6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 146 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
cmd/api-response.go
View File

@ -29,6 +29,7 @@ import (
"strings"
"time"
"github.com/minio/minio/internal/amztime"
"github.com/minio/minio/internal/crypto"
"github.com/minio/minio/internal/handlers"
"github.com/minio/minio/internal/hash"
@ -38,12 +39,10 @@ import (
)
const (
// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
maxDeleteList = 1000 // Limit number of objects deleted in a delete call.
maxUploadsList = 10000 // Limit number of uploads in a listUploadsResponse.
maxPartsList = 10000 // Limit number of parts in a listPartsResponse.
maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
maxDeleteList = 1000 // Limit number of objects deleted in a delete call.
maxUploadsList = 10000 // Limit number of uploads in a listUploadsResponse.
maxPartsList = 10000 // Limit number of parts in a listPartsResponse.
)
// LocationResponse - format for location response.
@ -482,7 +481,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
for _, bucket := range buckets {
listbuckets = append(listbuckets, Bucket{
Name: bucket.Name,
CreationDate: bucket.Created.UTC().Format(iso8601TimeFormat),
CreationDate: amztime.ISO8601Format(bucket.Created.UTC()),
})
}
@ -508,7 +507,7 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
}
content := ObjectVersion{}
content.Key = s3EncodeName(object.Name, encodingType)
content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
content.LastModified = amztime.ISO8601Format(object.ModTime.UTC())
if object.ETag != "" {
content.ETag = "\"" + object.ETag + "\""
}
@ -566,7 +565,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
continue
}
content.Key = s3EncodeName(object.Name, encodingType)
content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
content.LastModified = amztime.ISO8601Format(object.ModTime.UTC())
if object.ETag != "" {
content.ETag = "\"" + object.ETag + "\""
}
@ -615,7 +614,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
continue
}
content.Key = s3EncodeName(object.Name, encodingType)
content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
content.LastModified = amztime.ISO8601Format(object.ModTime.UTC())
if object.ETag != "" {
content.ETag = "\"" + object.ETag + "\""
}
@ -678,7 +677,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
return CopyObjectResponse{
ETag: "\"" + etag + "\"",
LastModified: lastModified.UTC().Format(iso8601TimeFormat),
LastModified: amztime.ISO8601Format(lastModified.UTC()),
}
}
@ -686,7 +685,7 @@ func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectR
func generateCopyObjectPartResponse(etag string, lastModified time.Time) CopyObjectPartResponse {
return CopyObjectPartResponse{
ETag: "\"" + etag + "\"",
LastModified: lastModified.UTC().Format(iso8601TimeFormat),
LastModified: amztime.ISO8601Format(lastModified.UTC()),
}
}
@ -746,7 +745,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
newPart.PartNumber = part.PartNumber
newPart.ETag = "\"" + part.ETag + "\""
newPart.Size = part.Size
newPart.LastModified = part.LastModified.UTC().Format(iso8601TimeFormat)
newPart.LastModified = amztime.ISO8601Format(part.LastModified.UTC())
newPart.ChecksumCRC32 = part.ChecksumCRC32
newPart.ChecksumCRC32C = part.ChecksumCRC32C
newPart.ChecksumSHA1 = part.ChecksumSHA1
@ -780,7 +779,7 @@ func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMult
newUpload := Upload{}
newUpload.UploadID = upload.UploadID
newUpload.Key = s3EncodeName(upload.Object, encodingType)
newUpload.Initiated = upload.Initiated.UTC().Format(iso8601TimeFormat)
newUpload.Initiated = amztime.ISO8601Format(upload.Initiated.UTC())
listMultipartUploadsResponse.Uploads[index] = newUpload
}
return listMultipartUploadsResponse

8
cmd/bucket-replication.go
View File

@ -38,6 +38,7 @@ import (
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/encrypt"
"github.com/minio/minio-go/v7/pkg/tags"
"github.com/minio/minio/internal/amztime"
"github.com/minio/minio/internal/bucket/bandwidth"
objectlock "github.com/minio/minio/internal/bucket/object/lock"
"github.com/minio/minio/internal/bucket/replication"
@ -738,12 +739,9 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (put
putOpts.Mode = rmode
}
if retainDateStr, ok := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate); ok {
rdate, err := time.Parse(iso8601TimeFormat, retainDateStr)
rdate, err := amztime.ISO8601Parse(retainDateStr)
if err != nil {
rdate, err = time.Parse(time.RFC3339, retainDateStr)
if err != nil {
return putOpts, err
}
return putOpts, err
}
putOpts.RetainUntilDate = rdate
// set retention timestamp in opts

11
cmd/object-handlers.go
View File

@ -40,6 +40,7 @@ import (
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/encrypt"
"github.com/minio/minio-go/v7/pkg/tags"
"github.com/minio/minio/internal/amztime"
sse "github.com/minio/minio/internal/bucket/encryption"
"github.com/minio/minio/internal/bucket/lifecycle"
objectlock "github.com/minio/minio/internal/bucket/object/lock"
@ -1434,13 +1435,13 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
// update retention metadata only if replica timestamp is newer than what's on disk
if err != nil || (err == nil && ondiskTimestamp.Before(srcTimestamp)) {
srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retentionDate.UTC().Format(iso8601TimeFormat)
srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC())
srcInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = srcTimestamp.UTC().Format(time.RFC3339Nano)
}
}
} else {
srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retentionDate.UTC().Format(iso8601TimeFormat)
srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC())
srcInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = UTCNow().Format(time.RFC3339Nano)
}
}
@ -1846,7 +1847,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms)
if s3Err == ErrNone && retentionMode.Valid() {
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retentionDate.UTC().Format(iso8601TimeFormat)
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC())
}
if s3Err == ErrNone && legalHold.Status.Valid() {
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
@ -2190,7 +2191,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
retentionMode, retentionDate, legalHold, s3err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms)
if s3err == ErrNone && retentionMode.Valid() {
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retentionDate.UTC().Format(iso8601TimeFormat)
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC())
}
if s3err == ErrNone && legalHold.Status.Valid() {
@ -2694,7 +2695,7 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
}
if objRetention.Mode.Valid() {
oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = string(objRetention.Mode)
oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = objRetention.RetainUntilDate.UTC().Format(iso8601TimeFormat)
oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(objRetention.RetainUntilDate.UTC())
} else {
oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = ""
oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = ""

3
cmd/object-multipart-handlers.go
View File

@ -33,6 +33,7 @@ import (
"github.com/gorilla/mux"
"github.com/minio/minio-go/v7/pkg/encrypt"
"github.com/minio/minio-go/v7/pkg/tags"
"github.com/minio/minio/internal/amztime"
sse "github.com/minio/minio/internal/bucket/encryption"
objectlock "github.com/minio/minio/internal/bucket/object/lock"
"github.com/minio/minio/internal/bucket/replication"
@ -150,7 +151,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms)
if s3Err == ErrNone && retentionMode.Valid() {
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retentionDate.UTC().Format(iso8601TimeFormat)
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC())
}
if s3Err == ErrNone && legalHold.Status.Valid() {
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)

2
cmd/test-utils_test.go
View File

@ -126,6 +126,8 @@ func TestMain(m *testing.M) {
// concurrency level for certain parallel tests.
const testConcurrencyLevel = 10
const iso8601TimeFormat = "2006-01-02T15:04:05.000Z"
// Excerpts from @lsegal - https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258
//
// User-Agent:

55
internal/amztime/iso8601_time.go Normal file
View File

@ -0,0 +1,55 @@
// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package amztime
import (
"strings"
"time"
)
// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
const iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
// ISO8601Format converts time 't' into ISO8601 time format expected in AWS S3 spec.
//
// This function is needed to avoid a Go's float64 precision bug, where Go avoids
// padding the extra '0' before the timezone.
func ISO8601Format(t time.Time) string {
value := t.Format(iso8601TimeFormat)
if len(value) < len(iso8601TimeFormat) {
value = t.Format(iso8601TimeFormat[:len(iso8601TimeFormat)-1])
// Pad necessary zeroes to full-fill the iso8601TimeFormat
return value + strings.Repeat("0", (len(iso8601TimeFormat)-1)-len(value)) + "Z"
}
return value
}
// ISO8601Parse parses ISO8601 date string
func ISO8601Parse(iso8601 string) (t time.Time, err error) {
for _, layout := range []string{
iso8601TimeFormat,
time.RFC3339,
} {
t, err = time.Parse(layout, iso8601)
if err == nil {
return t, nil
}
}
return t, err
}

58
internal/amztime/iso8601_time_test.go Normal file
View File

@ -0,0 +1,58 @@
// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package amztime
import (
"testing"
"time"
)
func TestISO8601Format(t *testing.T) {
testCases := []struct {
date time.Time
expectedOutput string
}{
{
date: time.Date(2009, time.November, 13, 4, 51, 1, 940303531, time.UTC),
expectedOutput: "2009-11-13T04:51:01.940Z",
},
{
date: time.Date(2009, time.November, 13, 4, 51, 1, 901303531, time.UTC),
expectedOutput: "2009-11-13T04:51:01.901Z",
},
{
date: time.Date(2009, time.November, 13, 4, 51, 1, 900303531, time.UTC),
expectedOutput: "2009-11-13T04:51:01.900Z",
},
{
date: time.Date(2009, time.November, 13, 4, 51, 1, 941303531, time.UTC),
expectedOutput: "2009-11-13T04:51:01.941Z",
},
}
for _, testCase := range testCases {
testCase := testCase
t.Run(testCase.expectedOutput, func(t *testing.T) {
gotOutput := ISO8601Format(testCase.date)
t.Log("Go", testCase.date.Format(iso8601TimeFormat))
if gotOutput != testCase.expectedOutput {
t.Errorf("Expected %s, got %s", testCase.expectedOutput, gotOutput)
}
})
}
}

26
internal/bucket/object/lock/lock.go
View File

@ -30,6 +30,7 @@ import (
"time"
"github.com/beevik/ntp"
"github.com/minio/minio/internal/amztime"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/logger"
@ -48,10 +49,6 @@ const (
// RetCompliance - compliance mode.
RetCompliance RetMode = "COMPLIANCE"
// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with millisecond precision.
)
// Valid - returns if retention mode is valid
@ -312,12 +309,9 @@ func (rDate *RetentionDate) UnmarshalXML(d *xml.Decoder, startElement xml.StartE
// While AWS documentation mentions that the date specified
// must be present in ISO 8601 format, in reality they allow
// users to provide RFC 3339 compliant dates.
retDate, err := time.Parse(iso8601TimeFormat, dateStr)
retDate, err := amztime.ISO8601Parse(dateStr)
if err != nil {
retDate, err = time.Parse(time.RFC3339, dateStr)
if err != nil {
return ErrInvalidRetentionDate
}
return ErrInvalidRetentionDate
}
*rDate = RetentionDate{retDate}
@ -330,7 +324,7 @@ func (rDate *RetentionDate) MarshalXML(e *xml.Encoder, startElement xml.StartEle
if rDate.IsZero() {
return nil
}
return e.EncodeElement(rDate.Format(iso8601TimeFormat), startElement)
return e.EncodeElement(amztime.ISO8601Format(rDate.Time), startElement)
}
// ObjectRetention specified in
@ -420,12 +414,9 @@ func ParseObjectLockRetentionHeaders(h http.Header) (rmode RetMode, r RetentionD
// While AWS documentation mentions that the date specified
// must be present in ISO 8601 format, in reality they allow
// users to provide RFC 3339 compliant dates.
retDate, err = time.Parse(iso8601TimeFormat, dateStr)
retDate, err = amztime.ISO8601Parse(dateStr)
if err != nil {
retDate, err = time.Parse(time.RFC3339, dateStr)
if err != nil {
return rmode, r, ErrInvalidRetentionDate
}
return rmode, r, ErrInvalidRetentionDate
}
_, replReq := h[textproto.CanonicalMIMEHeaderKey(xhttp.MinIOSourceReplicationRequest)]
@ -465,10 +456,7 @@ func GetObjectRetentionMeta(meta map[string]string) ObjectRetention {
tillStr, ok = meta[AmzObjectLockRetainUntilDate]
}
if ok {
if t, e := time.Parse(iso8601TimeFormat, tillStr); e == nil {
retainTill = RetentionDate{t.UTC()}
}
if t, e := time.Parse(time.RFC3339, tillStr); e == nil {
if t, e := amztime.ISO8601Parse(tillStr); e == nil {
retainTill = RetentionDate{t.UTC()}
}
}