diff --git a/cmd/api-response.go b/cmd/api-response.go index 485c466be..dc819294a 100644 --- a/cmd/api-response.go +++ b/cmd/api-response.go @@ -29,6 +29,7 @@ import ( "strings" "time" + "github.com/minio/minio/internal/amztime" "github.com/minio/minio/internal/crypto" "github.com/minio/minio/internal/handlers" "github.com/minio/minio/internal/hash" @@ -38,12 +39,10 @@ import ( ) const ( - // RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z - iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision. - maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse. - maxDeleteList = 1000 // Limit number of objects deleted in a delete call. - maxUploadsList = 10000 // Limit number of uploads in a listUploadsResponse. - maxPartsList = 10000 // Limit number of parts in a listPartsResponse. + maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse. + maxDeleteList = 1000 // Limit number of objects deleted in a delete call. + maxUploadsList = 10000 // Limit number of uploads in a listUploadsResponse. + maxPartsList = 10000 // Limit number of parts in a listPartsResponse. ) // LocationResponse - format for location response. @@ -482,7 +481,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse { for _, bucket := range buckets { listbuckets = append(listbuckets, Bucket{ Name: bucket.Name, - CreationDate: bucket.Created.UTC().Format(iso8601TimeFormat), + CreationDate: amztime.ISO8601Format(bucket.Created.UTC()), }) } @@ -508,7 +507,7 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim } content := ObjectVersion{} content.Key = s3EncodeName(object.Name, encodingType) - content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat) + content.LastModified = amztime.ISO8601Format(object.ModTime.UTC()) if object.ETag != "" { content.ETag = "\"" + object.ETag + "\"" } @@ -566,7 +565,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy continue } content.Key = s3EncodeName(object.Name, encodingType) - content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat) + content.LastModified = amztime.ISO8601Format(object.ModTime.UTC()) if object.ETag != "" { content.ETag = "\"" + object.ETag + "\"" } @@ -615,7 +614,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, continue } content.Key = s3EncodeName(object.Name, encodingType) - content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat) + content.LastModified = amztime.ISO8601Format(object.ModTime.UTC()) if object.ETag != "" { content.ETag = "\"" + object.ETag + "\"" } @@ -678,7 +677,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse { return CopyObjectResponse{ ETag: "\"" + etag + "\"", - LastModified: lastModified.UTC().Format(iso8601TimeFormat), + LastModified: amztime.ISO8601Format(lastModified.UTC()), } } @@ -686,7 +685,7 @@ func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectR func generateCopyObjectPartResponse(etag string, lastModified time.Time) CopyObjectPartResponse { return CopyObjectPartResponse{ ETag: "\"" + etag + "\"", - LastModified: lastModified.UTC().Format(iso8601TimeFormat), + LastModified: amztime.ISO8601Format(lastModified.UTC()), } } @@ -746,7 +745,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis newPart.PartNumber = part.PartNumber newPart.ETag = "\"" + part.ETag + "\"" newPart.Size = part.Size - newPart.LastModified = part.LastModified.UTC().Format(iso8601TimeFormat) + newPart.LastModified = amztime.ISO8601Format(part.LastModified.UTC()) newPart.ChecksumCRC32 = part.ChecksumCRC32 newPart.ChecksumCRC32C = part.ChecksumCRC32C newPart.ChecksumSHA1 = part.ChecksumSHA1 @@ -780,7 +779,7 @@ func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMult newUpload := Upload{} newUpload.UploadID = upload.UploadID newUpload.Key = s3EncodeName(upload.Object, encodingType) - newUpload.Initiated = upload.Initiated.UTC().Format(iso8601TimeFormat) + newUpload.Initiated = amztime.ISO8601Format(upload.Initiated.UTC()) listMultipartUploadsResponse.Uploads[index] = newUpload } return listMultipartUploadsResponse diff --git a/cmd/bucket-replication.go b/cmd/bucket-replication.go index f1ba7ac6a..cb4e30c1f 100644 --- a/cmd/bucket-replication.go +++ b/cmd/bucket-replication.go @@ -38,6 +38,7 @@ import ( "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/minio/minio-go/v7/pkg/tags" + "github.com/minio/minio/internal/amztime" "github.com/minio/minio/internal/bucket/bandwidth" objectlock "github.com/minio/minio/internal/bucket/object/lock" "github.com/minio/minio/internal/bucket/replication" @@ -738,12 +739,9 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (put putOpts.Mode = rmode } if retainDateStr, ok := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate); ok { - rdate, err := time.Parse(iso8601TimeFormat, retainDateStr) + rdate, err := amztime.ISO8601Parse(retainDateStr) if err != nil { - rdate, err = time.Parse(time.RFC3339, retainDateStr) - if err != nil { - return putOpts, err - } + return putOpts, err } putOpts.RetainUntilDate = rdate // set retention timestamp in opts diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index f7be534b6..13060aafc 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -40,6 +40,7 @@ import ( "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/minio/minio-go/v7/pkg/tags" + "github.com/minio/minio/internal/amztime" sse "github.com/minio/minio/internal/bucket/encryption" "github.com/minio/minio/internal/bucket/lifecycle" objectlock "github.com/minio/minio/internal/bucket/object/lock" @@ -1434,13 +1435,13 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re // update retention metadata only if replica timestamp is newer than what's on disk if err != nil || (err == nil && ondiskTimestamp.Before(srcTimestamp)) { srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode) - srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retentionDate.UTC().Format(iso8601TimeFormat) + srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC()) srcInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = srcTimestamp.UTC().Format(time.RFC3339Nano) } } } else { srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode) - srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retentionDate.UTC().Format(iso8601TimeFormat) + srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC()) srcInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = UTCNow().Format(time.RFC3339Nano) } } @@ -1846,7 +1847,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms) if s3Err == ErrNone && retentionMode.Valid() { metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode) - metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retentionDate.UTC().Format(iso8601TimeFormat) + metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC()) } if s3Err == ErrNone && legalHold.Status.Valid() { metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status) @@ -2190,7 +2191,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h retentionMode, retentionDate, legalHold, s3err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms) if s3err == ErrNone && retentionMode.Valid() { metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode) - metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retentionDate.UTC().Format(iso8601TimeFormat) + metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC()) } if s3err == ErrNone && legalHold.Status.Valid() { @@ -2694,7 +2695,7 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r } if objRetention.Mode.Valid() { oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = string(objRetention.Mode) - oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = objRetention.RetainUntilDate.UTC().Format(iso8601TimeFormat) + oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(objRetention.RetainUntilDate.UTC()) } else { oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = "" oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = "" diff --git a/cmd/object-multipart-handlers.go b/cmd/object-multipart-handlers.go index b5cad8920..66189deeb 100644 --- a/cmd/object-multipart-handlers.go +++ b/cmd/object-multipart-handlers.go @@ -33,6 +33,7 @@ import ( "github.com/gorilla/mux" "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/minio/minio-go/v7/pkg/tags" + "github.com/minio/minio/internal/amztime" sse "github.com/minio/minio/internal/bucket/encryption" objectlock "github.com/minio/minio/internal/bucket/object/lock" "github.com/minio/minio/internal/bucket/replication" @@ -150,7 +151,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms) if s3Err == ErrNone && retentionMode.Valid() { metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode) - metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retentionDate.UTC().Format(iso8601TimeFormat) + metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC()) } if s3Err == ErrNone && legalHold.Status.Valid() { metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status) diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go index c903ec89d..ec7468bbc 100644 --- a/cmd/test-utils_test.go +++ b/cmd/test-utils_test.go @@ -126,6 +126,8 @@ func TestMain(m *testing.M) { // concurrency level for certain parallel tests. const testConcurrencyLevel = 10 +const iso8601TimeFormat = "2006-01-02T15:04:05.000Z" + // Excerpts from @lsegal - https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258 // // User-Agent: diff --git a/internal/amztime/iso8601_time.go b/internal/amztime/iso8601_time.go new file mode 100644 index 000000000..885d0bbda --- /dev/null +++ b/internal/amztime/iso8601_time.go @@ -0,0 +1,55 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package amztime + +import ( + "strings" + "time" +) + +// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z +const iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision. + +// ISO8601Format converts time 't' into ISO8601 time format expected in AWS S3 spec. +// +// This function is needed to avoid a Go's float64 precision bug, where Go avoids +// padding the extra '0' before the timezone. +func ISO8601Format(t time.Time) string { + value := t.Format(iso8601TimeFormat) + if len(value) < len(iso8601TimeFormat) { + value = t.Format(iso8601TimeFormat[:len(iso8601TimeFormat)-1]) + // Pad necessary zeroes to full-fill the iso8601TimeFormat + return value + strings.Repeat("0", (len(iso8601TimeFormat)-1)-len(value)) + "Z" + } + return value +} + +// ISO8601Parse parses ISO8601 date string +func ISO8601Parse(iso8601 string) (t time.Time, err error) { + for _, layout := range []string{ + iso8601TimeFormat, + time.RFC3339, + } { + t, err = time.Parse(layout, iso8601) + if err == nil { + return t, nil + } + } + + return t, err +} diff --git a/internal/amztime/iso8601_time_test.go b/internal/amztime/iso8601_time_test.go new file mode 100644 index 000000000..73270a4e7 --- /dev/null +++ b/internal/amztime/iso8601_time_test.go @@ -0,0 +1,58 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package amztime + +import ( + "testing" + "time" +) + +func TestISO8601Format(t *testing.T) { + testCases := []struct { + date time.Time + expectedOutput string + }{ + { + date: time.Date(2009, time.November, 13, 4, 51, 1, 940303531, time.UTC), + expectedOutput: "2009-11-13T04:51:01.940Z", + }, + { + date: time.Date(2009, time.November, 13, 4, 51, 1, 901303531, time.UTC), + expectedOutput: "2009-11-13T04:51:01.901Z", + }, + { + date: time.Date(2009, time.November, 13, 4, 51, 1, 900303531, time.UTC), + expectedOutput: "2009-11-13T04:51:01.900Z", + }, + { + date: time.Date(2009, time.November, 13, 4, 51, 1, 941303531, time.UTC), + expectedOutput: "2009-11-13T04:51:01.941Z", + }, + } + + for _, testCase := range testCases { + testCase := testCase + t.Run(testCase.expectedOutput, func(t *testing.T) { + gotOutput := ISO8601Format(testCase.date) + t.Log("Go", testCase.date.Format(iso8601TimeFormat)) + if gotOutput != testCase.expectedOutput { + t.Errorf("Expected %s, got %s", testCase.expectedOutput, gotOutput) + } + }) + } +} diff --git a/internal/bucket/object/lock/lock.go b/internal/bucket/object/lock/lock.go index 0eda09aeb..b718f28e8 100644 --- a/internal/bucket/object/lock/lock.go +++ b/internal/bucket/object/lock/lock.go @@ -30,6 +30,7 @@ import ( "time" "github.com/beevik/ntp" + "github.com/minio/minio/internal/amztime" xhttp "github.com/minio/minio/internal/http" "github.com/minio/minio/internal/logger" @@ -48,10 +49,6 @@ const ( // RetCompliance - compliance mode. RetCompliance RetMode = "COMPLIANCE" - - // RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z - iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with millisecond precision. - ) // Valid - returns if retention mode is valid @@ -312,12 +309,9 @@ func (rDate *RetentionDate) UnmarshalXML(d *xml.Decoder, startElement xml.StartE // While AWS documentation mentions that the date specified // must be present in ISO 8601 format, in reality they allow // users to provide RFC 3339 compliant dates. - retDate, err := time.Parse(iso8601TimeFormat, dateStr) + retDate, err := amztime.ISO8601Parse(dateStr) if err != nil { - retDate, err = time.Parse(time.RFC3339, dateStr) - if err != nil { - return ErrInvalidRetentionDate - } + return ErrInvalidRetentionDate } *rDate = RetentionDate{retDate} @@ -330,7 +324,7 @@ func (rDate *RetentionDate) MarshalXML(e *xml.Encoder, startElement xml.StartEle if rDate.IsZero() { return nil } - return e.EncodeElement(rDate.Format(iso8601TimeFormat), startElement) + return e.EncodeElement(amztime.ISO8601Format(rDate.Time), startElement) } // ObjectRetention specified in @@ -420,12 +414,9 @@ func ParseObjectLockRetentionHeaders(h http.Header) (rmode RetMode, r RetentionD // While AWS documentation mentions that the date specified // must be present in ISO 8601 format, in reality they allow // users to provide RFC 3339 compliant dates. - retDate, err = time.Parse(iso8601TimeFormat, dateStr) + retDate, err = amztime.ISO8601Parse(dateStr) if err != nil { - retDate, err = time.Parse(time.RFC3339, dateStr) - if err != nil { - return rmode, r, ErrInvalidRetentionDate - } + return rmode, r, ErrInvalidRetentionDate } _, replReq := h[textproto.CanonicalMIMEHeaderKey(xhttp.MinIOSourceReplicationRequest)] @@ -465,10 +456,7 @@ func GetObjectRetentionMeta(meta map[string]string) ObjectRetention { tillStr, ok = meta[AmzObjectLockRetainUntilDate] } if ok { - if t, e := time.Parse(iso8601TimeFormat, tillStr); e == nil { - retainTill = RetentionDate{t.UTC()} - } - if t, e := time.Parse(time.RFC3339, tillStr); e == nil { + if t, e := amztime.ISO8601Parse(tillStr); e == nil { retainTill = RetentionDate{t.UTC()} } }