2016-01-25 20:29:20 -05:00
/ *
2019-04-09 14:39:42 -04:00
* MinIO Cloud Storage , ( C ) 2016 - 2019 MinIO , Inc .
2016-01-25 20:29:20 -05:00
*
* Licensed under the Apache License , Version 2.0 ( the "License" ) ;
* you may not use this file except in compliance with the License .
* You may obtain a copy of the License at
*
* http : //www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing , software
* distributed under the License is distributed on an "AS IS" BASIS ,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND , either express or implied .
* See the License for the specific language governing permissions and
* limitations under the License .
* /
2016-08-18 19:23:42 -04:00
package cmd
2016-01-23 22:44:32 -05:00
import (
2018-03-15 16:27:16 -04:00
"context"
2020-12-20 13:09:51 -05:00
"crypto/subtle"
2016-08-30 13:04:50 -04:00
"encoding/json"
2019-10-23 01:59:13 -04:00
"errors"
2016-01-23 22:44:32 -05:00
"fmt"
2017-05-10 12:54:24 -04:00
"io"
2016-01-23 22:44:32 -05:00
"net/http"
2018-06-05 13:48:51 -04:00
"net/url"
2016-02-04 01:46:45 -05:00
"os"
2016-02-22 01:38:38 -05:00
"path"
2020-11-21 01:52:17 -05:00
"reflect"
2016-02-04 01:46:45 -05:00
"runtime"
"strconv"
2016-09-23 04:24:49 -04:00
"strings"
2016-01-23 22:44:32 -05:00
"time"
2016-03-31 09:57:29 -04:00
"github.com/gorilla/mux"
2019-09-26 02:08:24 -04:00
"github.com/klauspost/compress/zip"
2020-07-14 12:38:05 -04:00
"github.com/minio/minio-go/v7"
2020-07-17 01:38:58 -04:00
miniogo "github.com/minio/minio-go/v7"
2020-07-14 12:38:05 -04:00
miniogopolicy "github.com/minio/minio-go/v7/pkg/policy"
"github.com/minio/minio-go/v7/pkg/s3utils"
2017-01-23 21:07:22 -05:00
"github.com/minio/minio/browser"
2020-09-10 17:19:32 -04:00
"github.com/minio/minio/cmd/config/dns"
2019-10-23 01:59:13 -04:00
"github.com/minio/minio/cmd/config/identity/openid"
2018-08-17 15:52:14 -04:00
"github.com/minio/minio/cmd/crypto"
2019-07-03 01:34:32 -04:00
xhttp "github.com/minio/minio/cmd/http"
2018-04-05 18:04:40 -04:00
"github.com/minio/minio/cmd/logger"
2017-10-31 14:54:32 -04:00
"github.com/minio/minio/pkg/auth"
2020-11-25 14:24:50 -05:00
"github.com/minio/minio/pkg/bucket/lifecycle"
2020-01-27 17:12:34 -05:00
objectlock "github.com/minio/minio/pkg/bucket/object/lock"
"github.com/minio/minio/pkg/bucket/policy"
2020-08-12 20:32:24 -04:00
"github.com/minio/minio/pkg/bucket/replication"
pkg/etag: add new package for S3 ETag handling (#11577)
This commit adds a new package `etag` for dealing
with S3 ETags.
Even though ETag is often viewed as MD5 checksum of
an object, handling S3 ETags correctly is a surprisingly
complex task. While it is true that the ETag corresponds
to the MD5 for the most basic S3 API operations, there are
many exceptions in case of multipart uploads or encryption.
In worse, some S3 clients expect very specific behavior when
it comes to ETags. For example, some clients expect that the
ETag is a double-quoted string and fail otherwise.
Non-AWS compliant ETag handling has been a source of many bugs
in the past.
Therefore, this commit adds a dedicated `etag` package that provides
functionality for parsing, generating and converting S3 ETags.
Further, this commit removes the ETag computation from the `hash`
package. Instead, the `hash` package (i.e. `hash.Reader`) should
focus only on computing and verifying the content-sha256.
One core feature of this commit is to provide a mechanism to
communicate a computed ETag from a low-level `io.Reader` to
a high-level `io.Reader`.
This problem occurs when an S3 server receives a request and
has to compute the ETag of the content. However, the server
may also wrap the initial body with several other `io.Reader`,
e.g. when encrypting or compressing the content:
```
reader := Encrypt(Compress(ETag(content)))
```
In such a case, the ETag should be accessible by the high-level
`io.Reader`.
The `etag` provides a mechanism to wrap `io.Reader` implementations
such that the `ETag` can be accessed by a type-check.
This technique is applied to the PUT, COPY and Upload handlers.
2021-02-23 15:31:53 -05:00
"github.com/minio/minio/pkg/etag"
2018-03-15 16:03:41 -04:00
"github.com/minio/minio/pkg/event"
2018-10-05 14:20:00 -04:00
"github.com/minio/minio/pkg/handlers"
2017-10-22 01:30:34 -04:00
"github.com/minio/minio/pkg/hash"
2019-02-27 20:46:55 -05:00
iampolicy "github.com/minio/minio/pkg/iam/policy"
2018-08-17 15:52:14 -04:00
"github.com/minio/minio/pkg/ioutil"
2020-12-15 14:18:06 -05:00
"github.com/minio/minio/pkg/rpc/json2"
2016-01-23 22:44:32 -05:00
)
2020-11-21 01:52:17 -05:00
func extractBucketObject ( args reflect . Value ) ( bucketName , objectName string ) {
switch args . Kind ( ) {
case reflect . Ptr :
a := args . Elem ( )
for i := 0 ; i < a . NumField ( ) ; i ++ {
switch a . Type ( ) . Field ( i ) . Name {
case "BucketName" :
bucketName = a . Field ( i ) . String ( )
case "Prefix" :
objectName = a . Field ( i ) . String ( )
case "ObjectName" :
objectName = a . Field ( i ) . String ( )
}
}
}
return bucketName , objectName
}
2016-04-08 02:44:08 -04:00
// WebGenericArgs - empty struct for calls that don't accept arguments
2021-03-02 18:35:50 -05:00
// for ex. ServerInfo
2016-04-08 02:44:08 -04:00
type WebGenericArgs struct { }
2016-03-21 14:15:08 -04:00
2016-04-08 02:44:08 -04:00
// WebGenericRep - reply structure for calls for which reply is success/failure
2016-02-19 03:00:32 -05:00
// for ex. RemoveObject MakeBucket
2016-04-08 02:44:08 -04:00
type WebGenericRep struct {
2016-02-19 03:00:32 -05:00
UIVersion string ` json:"uiVersion" `
}
// ServerInfoRep - server info reply.
type ServerInfoRep struct {
2017-05-25 00:09:23 -04:00
MinioVersion string
MinioMemory string
MinioPlatform string
MinioRuntime string
MinioGlobalInfo map [ string ] interface { }
2019-05-29 16:18:46 -04:00
MinioUserInfo map [ string ] interface { }
2017-05-25 00:09:23 -04:00
UIVersion string ` json:"uiVersion" `
2016-02-19 03:00:32 -05:00
}
2016-02-04 01:46:45 -05:00
// ServerInfo - get server info.
2016-04-12 15:45:15 -04:00
func ( web * webAPIHandlers ) ServerInfo ( r * http . Request , args * WebGenericArgs , reply * ServerInfoRep ) error {
2019-10-23 01:59:13 -04:00
ctx := newWebContext ( r , args , "WebServerInfo" )
claims , owner , authErr := webRequestAuthenticate ( r )
2018-10-17 19:23:09 -04:00
if authErr != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , authErr )
2016-02-04 01:46:45 -05:00
}
host , err := os . Hostname ( )
if err != nil {
host = ""
}
platform := fmt . Sprintf ( "Host: %s | OS: %s | Arch: %s" ,
host ,
runtime . GOOS ,
runtime . GOARCH )
2019-10-23 01:59:13 -04:00
goruntime := fmt . Sprintf ( "Version: %s | CPUs: %d" , runtime . Version ( ) , runtime . NumCPU ( ) )
2016-08-03 16:47:03 -04:00
2016-08-18 19:23:42 -04:00
reply . MinioVersion = Version
2017-05-25 00:09:23 -04:00
reply . MinioGlobalInfo = getGlobalInfo ( )
2019-05-29 16:18:46 -04:00
2019-10-23 01:59:13 -04:00
// Check if the user is IAM user.
2019-05-29 16:18:46 -04:00
reply . MinioUserInfo = map [ string ] interface { } {
"isIAMUser" : ! owner ,
}
2019-10-23 01:59:13 -04:00
if ! owner {
2020-01-30 21:59:22 -05:00
creds , ok := globalIAMSys . GetUser ( claims . AccessKey )
2019-10-23 01:59:13 -04:00
if ok && creds . SessionToken != "" {
reply . MinioUserInfo [ "isTempUser" ] = true
}
}
2016-02-08 15:40:22 -05:00
reply . MinioPlatform = platform
reply . MinioRuntime = goruntime
2017-02-22 20:27:26 -05:00
reply . UIVersion = browser . UIVersion
2016-02-04 01:46:45 -05:00
return nil
}
2016-05-26 17:13:10 -04:00
// StorageInfoRep - contains storage usage statistics.
type StorageInfoRep struct {
2020-06-09 18:05:39 -04:00
Used uint64 ` json:"used" `
UIVersion string ` json:"uiVersion" `
2016-05-26 17:13:10 -04:00
}
// StorageInfo - web call to gather storage usage statistics.
2019-04-18 02:16:27 -04:00
func ( web * webAPIHandlers ) StorageInfo ( r * http . Request , args * WebGenericArgs , reply * StorageInfoRep ) error {
2019-10-23 01:59:13 -04:00
ctx := newWebContext ( r , args , "WebStorageInfo" )
2016-07-31 17:11:14 -04:00
objectAPI := web . ObjectAPI ( )
if objectAPI == nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , errServerNotInitialized )
2016-11-02 17:45:11 -04:00
}
2018-10-17 19:23:09 -04:00
_ , _ , authErr := webRequestAuthenticate ( r )
if authErr != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , authErr )
2016-07-31 17:11:14 -04:00
}
2020-06-09 18:05:39 -04:00
dataUsageInfo , _ := loadDataUsageFromBackend ( ctx , objectAPI )
reply . Used = dataUsageInfo . ObjectsTotalSize
2017-02-22 20:27:26 -05:00
reply . UIVersion = browser . UIVersion
2016-05-26 17:13:10 -04:00
return nil
}
2016-02-19 03:00:32 -05:00
// MakeBucketArgs - make bucket args.
type MakeBucketArgs struct {
BucketName string ` json:"bucketName" `
}
2017-03-03 06:01:42 -05:00
// MakeBucket - creates a new bucket.
2016-04-12 15:45:15 -04:00
func ( web * webAPIHandlers ) MakeBucket ( r * http . Request , args * MakeBucketArgs , reply * WebGenericRep ) error {
2019-10-23 01:59:13 -04:00
ctx := newWebContext ( r , args , "WebMakeBucket" )
2016-07-31 17:11:14 -04:00
objectAPI := web . ObjectAPI ( )
if objectAPI == nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , errServerNotInitialized )
2016-11-02 17:45:11 -04:00
}
2019-04-09 11:17:41 -04:00
claims , owner , authErr := webRequestAuthenticate ( r )
2018-10-17 19:23:09 -04:00
if authErr != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , authErr )
2018-10-17 19:23:09 -04:00
}
2018-12-19 08:13:47 -05:00
2019-04-09 11:17:41 -04:00
// For authenticated users apply IAM policy.
if ! globalIAMSys . IsAllowed ( iampolicy . Args {
2020-01-30 21:59:22 -05:00
AccountName : claims . AccessKey ,
2019-04-09 11:17:41 -04:00
Action : iampolicy . CreateBucketAction ,
BucketName : args . BucketName ,
2020-01-30 21:59:22 -05:00
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
2019-04-09 11:17:41 -04:00
IsOwner : owner ,
2019-10-23 01:59:13 -04:00
Claims : claims . Map ( ) ,
2019-04-09 11:17:41 -04:00
} ) {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , errAccessDenied )
2016-07-31 17:11:14 -04:00
}
2017-03-03 06:01:42 -05:00
2017-09-01 15:16:54 -04:00
// Check if bucket is a reserved bucket name or invalid.
2019-03-05 13:42:32 -05:00
if isReservedOrInvalidBucket ( args . BucketName , true ) {
2020-07-10 15:10:39 -04:00
return toJSONError ( ctx , errInvalidBucketName , args . BucketName )
2017-03-03 06:01:42 -05:00
}
2020-06-12 23:04:01 -04:00
opts := BucketOptions {
Location : globalServerRegion ,
LockEnabled : false ,
}
2018-05-01 13:36:37 -04:00
if globalDNSConfig != nil {
if _ , err := globalDNSConfig . Get ( args . BucketName ) ; err != nil {
2020-09-09 15:20:49 -04:00
if err == dns . ErrNoEntriesFound || err == dns . ErrNotImplemented {
2018-05-01 13:36:37 -04:00
// Proceed to creating a bucket.
2020-06-12 23:04:01 -04:00
if err = objectAPI . MakeBucketWithLocation ( ctx , args . BucketName , opts ) ; err != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err )
2018-05-01 13:36:37 -04:00
}
2020-07-20 15:52:49 -04:00
2018-05-01 13:36:37 -04:00
if err = globalDNSConfig . Put ( args . BucketName ) ; err != nil {
2020-03-28 00:52:59 -04:00
objectAPI . DeleteBucket ( ctx , args . BucketName , false )
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err )
2018-05-01 13:36:37 -04:00
}
reply . UIVersion = browser . UIVersion
return nil
}
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err )
2018-05-01 13:36:37 -04:00
}
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , errBucketAlreadyExists )
2018-05-01 13:36:37 -04:00
}
2020-06-12 23:04:01 -04:00
if err := objectAPI . MakeBucketWithLocation ( ctx , args . BucketName , opts ) ; err != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2016-02-12 21:55:17 -05:00
}
2017-03-03 06:01:42 -05:00
2017-02-22 20:27:26 -05:00
reply . UIVersion = browser . UIVersion
2020-07-20 15:52:49 -04:00
sendEvent ( eventArgs {
EventName : event . BucketCreated ,
BucketName : args . BucketName ,
ReqParams : extractReqParams ( r ) ,
UserAgent : r . UserAgent ( ) ,
Host : handlers . GetSourceIP ( r ) ,
} )
2016-02-12 21:55:17 -05:00
return nil
2016-01-25 01:26:53 -05:00
}
2017-12-29 08:15:44 -05:00
// RemoveBucketArgs - remove bucket args.
type RemoveBucketArgs struct {
BucketName string ` json:"bucketName" `
}
// DeleteBucket - removes a bucket, must be empty.
func ( web * webAPIHandlers ) DeleteBucket ( r * http . Request , args * RemoveBucketArgs , reply * WebGenericRep ) error {
2019-10-23 01:59:13 -04:00
ctx := newWebContext ( r , args , "WebDeleteBucket" )
2017-12-29 08:15:44 -05:00
objectAPI := web . ObjectAPI ( )
if objectAPI == nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , errServerNotInitialized )
2017-12-29 08:15:44 -05:00
}
2019-04-09 11:17:41 -04:00
claims , owner , authErr := webRequestAuthenticate ( r )
2018-10-17 19:23:09 -04:00
if authErr != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , authErr )
2018-10-17 19:23:09 -04:00
}
2018-12-19 08:13:47 -05:00
2019-04-09 11:17:41 -04:00
// For authenticated users apply IAM policy.
if ! globalIAMSys . IsAllowed ( iampolicy . Args {
2020-01-30 21:59:22 -05:00
AccountName : claims . AccessKey ,
2019-04-09 11:17:41 -04:00
Action : iampolicy . DeleteBucketAction ,
BucketName : args . BucketName ,
2020-01-30 21:59:22 -05:00
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
2019-04-09 11:17:41 -04:00
IsOwner : owner ,
2019-10-23 01:59:13 -04:00
Claims : claims . Map ( ) ,
2019-04-09 11:17:41 -04:00
} ) {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , errAccessDenied )
2017-12-29 08:15:44 -05:00
}
2019-04-04 02:10:37 -04:00
// Check if bucket is a reserved bucket name or invalid.
if isReservedOrInvalidBucket ( args . BucketName , false ) {
2020-07-10 15:10:39 -04:00
return toJSONError ( ctx , errInvalidBucketName , args . BucketName )
2019-04-04 02:10:37 -04:00
}
2018-12-19 08:13:47 -05:00
reply . UIVersion = browser . UIVersion
2019-06-03 18:40:04 -04:00
if isRemoteCallRequired ( ctx , args . BucketName , objectAPI ) {
2018-12-19 08:13:47 -05:00
sr , err := globalDNSConfig . Get ( args . BucketName )
if err != nil {
if err == dns . ErrNoEntriesFound {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , BucketNotFound {
2018-12-19 08:13:47 -05:00
Bucket : args . BucketName ,
} , args . BucketName )
}
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
core , err := getRemoteInstanceClient ( r , getHostFromSrv ( sr ) )
if err != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
2020-07-14 12:38:05 -04:00
if err = core . RemoveBucket ( ctx , args . BucketName ) ; err != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
return nil
}
2018-03-28 17:14:06 -04:00
deleteBucket := objectAPI . DeleteBucket
2018-04-24 18:53:30 -04:00
2020-03-28 00:52:59 -04:00
if err := deleteBucket ( ctx , args . BucketName , false ) ; err != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2017-12-29 08:15:44 -05:00
}
2020-06-12 23:04:01 -04:00
globalNotificationSys . DeleteBucketMetadata ( ctx , args . BucketName )
2018-05-01 13:36:37 -04:00
if globalDNSConfig != nil {
if err := globalDNSConfig . Delete ( args . BucketName ) ; err != nil {
2020-09-09 15:20:49 -04:00
logger . LogIf ( ctx , fmt . Errorf ( "Unable to delete bucket DNS entry %w, please delete it manually" , err ) )
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err )
2018-05-01 13:36:37 -04:00
}
}
2020-07-20 15:52:49 -04:00
sendEvent ( eventArgs {
EventName : event . BucketRemoved ,
BucketName : args . BucketName ,
ReqParams : extractReqParams ( r ) ,
UserAgent : r . UserAgent ( ) ,
Host : handlers . GetSourceIP ( r ) ,
} )
2017-12-29 08:15:44 -05:00
return nil
}
2016-02-19 03:00:32 -05:00
// ListBucketsRep - list buckets response
type ListBucketsRep struct {
objectAPI: Fix object API interface, remove unnecessary structs.
ObjectAPI changes.
```
ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, *probe.Error)
ListMultipartUploads(bucket, objectPrefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, *probe.Error)
ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, *probe.Error)
CompleteMultipartUpload(bucket string, object string, uploadID string, parts []completePart) (ObjectInfo, *probe.Error)
```
2016-04-03 04:34:20 -04:00
Buckets [ ] WebBucketInfo ` json:"buckets" `
UIVersion string ` json:"uiVersion" `
2016-02-19 03:00:32 -05:00
}
objectAPI: Fix object API interface, remove unnecessary structs.
ObjectAPI changes.
```
ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, *probe.Error)
ListMultipartUploads(bucket, objectPrefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, *probe.Error)
ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, *probe.Error)
CompleteMultipartUpload(bucket string, object string, uploadID string, parts []completePart) (ObjectInfo, *probe.Error)
```
2016-04-03 04:34:20 -04:00
// WebBucketInfo container for list buckets metadata.
type WebBucketInfo struct {
2016-02-19 03:00:32 -05:00
// The name of the bucket.
Name string ` json:"name" `
// Date the bucket was created.
CreationDate time . Time ` json:"creationDate" `
}
2016-01-23 22:44:32 -05:00
// ListBuckets - list buckets api.
2016-04-12 15:45:15 -04:00
func ( web * webAPIHandlers ) ListBuckets ( r * http . Request , args * WebGenericArgs , reply * ListBucketsRep ) error {
2019-10-23 01:59:13 -04:00
ctx := newWebContext ( r , args , "WebListBuckets" )
2016-07-31 17:11:14 -04:00
objectAPI := web . ObjectAPI ( )
if objectAPI == nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , errServerNotInitialized )
2016-11-02 17:45:11 -04:00
}
2018-03-28 17:14:06 -04:00
listBuckets := objectAPI . ListBuckets
2018-10-09 17:00:01 -04:00
2018-12-10 12:27:22 -05:00
claims , owner , authErr := webRequestAuthenticate ( r )
if authErr != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , authErr )
2016-07-31 17:11:14 -04:00
}
2018-10-09 17:00:01 -04:00
2019-01-20 02:20:01 -05:00
// Set prefix value for "s3:prefix" policy conditionals.
r . Header . Set ( "prefix" , "" )
// Set delimiter value for "s3:delimiter" policy conditionals.
2019-08-06 15:08:58 -04:00
r . Header . Set ( "delimiter" , SlashSeparator )
2019-01-20 02:20:01 -05:00
2018-05-01 13:36:37 -04:00
// If etcd, dns federation configured list buckets from etcd.
2020-01-06 22:33:00 -05:00
if globalDNSConfig != nil && globalBucketFederation {
2018-05-01 13:36:37 -04:00
dnsBuckets , err := globalDNSConfig . List ( )
2021-01-28 14:44:48 -05:00
if err != nil && ! IsErrIgnored ( err ,
dns . ErrNoEntriesFound ,
dns . ErrDomainMissing ) {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err )
2018-05-01 13:36:37 -04:00
}
2020-02-03 03:24:20 -05:00
for _ , dnsRecords := range dnsBuckets {
2018-12-10 12:27:22 -05:00
if globalIAMSys . IsAllowed ( iampolicy . Args {
2020-01-30 21:59:22 -05:00
AccountName : claims . AccessKey ,
2018-12-13 23:15:09 -05:00
Action : iampolicy . ListBucketAction ,
2020-02-03 03:24:20 -05:00
BucketName : dnsRecords [ 0 ] . Key ,
2020-01-30 21:59:22 -05:00
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
2018-12-10 12:27:22 -05:00
IsOwner : owner ,
ObjectName : "" ,
2019-10-23 01:59:13 -04:00
Claims : claims . Map ( ) ,
2018-12-10 12:27:22 -05:00
} ) {
reply . Buckets = append ( reply . Buckets , WebBucketInfo {
2020-02-03 03:24:20 -05:00
Name : dnsRecords [ 0 ] . Key ,
CreationDate : dnsRecords [ 0 ] . CreationDate ,
2018-12-10 12:27:22 -05:00
} )
}
2018-05-01 13:36:37 -04:00
}
} else {
2019-06-03 18:40:04 -04:00
buckets , err := listBuckets ( ctx )
2018-05-01 13:36:37 -04:00
if err != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err )
2018-05-01 13:36:37 -04:00
}
for _ , bucket := range buckets {
2018-12-10 12:27:22 -05:00
if globalIAMSys . IsAllowed ( iampolicy . Args {
2020-01-30 21:59:22 -05:00
AccountName : claims . AccessKey ,
2020-06-24 02:21:11 -04:00
Action : iampolicy . ListBucketAction ,
2018-12-10 12:27:22 -05:00
BucketName : bucket . Name ,
2020-01-30 21:59:22 -05:00
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
2018-12-10 12:27:22 -05:00
IsOwner : owner ,
ObjectName : "" ,
2019-10-23 01:59:13 -04:00
Claims : claims . Map ( ) ,
2018-12-10 12:27:22 -05:00
} ) {
reply . Buckets = append ( reply . Buckets , WebBucketInfo {
Name : bucket . Name ,
CreationDate : bucket . Created ,
} )
}
2018-05-01 13:36:37 -04:00
}
2016-01-23 22:44:32 -05:00
}
2018-05-01 13:36:37 -04:00
2017-02-22 20:27:26 -05:00
reply . UIVersion = browser . UIVersion
2016-01-23 22:44:32 -05:00
return nil
}
2016-02-19 03:00:32 -05:00
// ListObjectsArgs - list object args.
type ListObjectsArgs struct {
BucketName string ` json:"bucketName" `
Prefix string ` json:"prefix" `
2017-02-11 01:54:42 -05:00
Marker string ` json:"marker" `
2016-02-19 03:00:32 -05:00
}
// ListObjectsRep - list objects response.
type ListObjectsRep struct {
2019-06-25 19:31:50 -04:00
Objects [ ] WebObjectInfo ` json:"objects" `
Writable bool ` json:"writable" ` // Used by client to show "upload file" button.
UIVersion string ` json:"uiVersion" `
2016-02-19 03:00:32 -05:00
}
objectAPI: Fix object API interface, remove unnecessary structs.
ObjectAPI changes.
```
ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, *probe.Error)
ListMultipartUploads(bucket, objectPrefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, *probe.Error)
ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, *probe.Error)
CompleteMultipartUpload(bucket string, object string, uploadID string, parts []completePart) (ObjectInfo, *probe.Error)
```
2016-04-03 04:34:20 -04:00
// WebObjectInfo container for list objects metadata.
type WebObjectInfo struct {
2016-02-19 03:00:32 -05:00
// Name of the object
Key string ` json:"name" `
// Date and time the object was last modified.
LastModified time . Time ` json:"lastModified" `
// Size in bytes of the object.
Size int64 ` json:"size" `
// ContentType is mime type of the object.
ContentType string ` json:"contentType" `
}
2016-01-23 22:44:32 -05:00
// ListObjects - list objects api.
2016-04-12 15:45:15 -04:00
func ( web * webAPIHandlers ) ListObjects ( r * http . Request , args * ListObjectsArgs , reply * ListObjectsRep ) error {
2019-10-23 01:59:13 -04:00
ctx := newWebContext ( r , args , "WebListObjects" )
2017-02-22 20:27:26 -05:00
reply . UIVersion = browser . UIVersion
2016-11-02 17:45:11 -04:00
objectAPI := web . ObjectAPI ( )
if objectAPI == nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , errServerNotInitialized )
2016-11-02 17:45:11 -04:00
}
2018-10-09 17:00:01 -04:00
2018-03-28 17:14:06 -04:00
listObjects := objectAPI . ListObjects
2018-04-24 18:53:30 -04:00
2019-06-03 18:40:04 -04:00
if isRemoteCallRequired ( ctx , args . BucketName , objectAPI ) {
2018-12-19 08:13:47 -05:00
sr , err := globalDNSConfig . Get ( args . BucketName )
if err != nil {
if err == dns . ErrNoEntriesFound {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , BucketNotFound {
2018-12-19 08:13:47 -05:00
Bucket : args . BucketName ,
} , args . BucketName )
}
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
2020-09-12 02:03:08 -04:00
core , err := getRemoteInstanceClient ( r , getHostFromSrv ( sr ) )
2018-12-19 08:13:47 -05:00
if err != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
2019-06-25 19:31:50 -04:00
nextMarker := ""
// Fetch all the objects
for {
2020-05-25 19:51:32 -04:00
// Let listObjects reply back the maximum from server implementation
2020-12-23 22:58:15 -05:00
result , err := core . ListObjects ( args . BucketName , args . Prefix , nextMarker , SlashSeparator , 1000 )
2019-06-25 19:31:50 -04:00
if err != nil {
return toJSONError ( ctx , err , args . BucketName )
}
for _ , obj := range result . Contents {
reply . Objects = append ( reply . Objects , WebObjectInfo {
Key : obj . Key ,
LastModified : obj . LastModified ,
Size : obj . Size ,
ContentType : obj . ContentType ,
} )
}
for _ , p := range result . CommonPrefixes {
reply . Objects = append ( reply . Objects , WebObjectInfo {
Key : p . Prefix ,
} )
}
nextMarker = result . NextMarker
// Return when there are no more objects
if ! result . IsTruncated {
return nil
}
2018-12-19 08:13:47 -05:00
}
}
2018-10-09 17:00:01 -04:00
claims , owner , authErr := webRequestAuthenticate ( r )
if authErr != nil {
if authErr == errNoAuthToken {
2019-01-20 02:20:01 -05:00
// Set prefix value for "s3:prefix" policy conditionals.
r . Header . Set ( "prefix" , args . Prefix )
// Set delimiter value for "s3:delimiter" policy conditionals.
2019-08-06 15:08:58 -04:00
r . Header . Set ( "delimiter" , SlashSeparator )
2018-12-13 23:15:09 -05:00
2018-10-09 17:00:01 -04:00
// Check if anonymous (non-owner) has access to download objects.
readable := globalPolicySys . IsAllowed ( policy . Args {
2018-12-13 23:15:09 -05:00
Action : policy . ListBucketAction ,
2018-10-09 17:00:01 -04:00
BucketName : args . BucketName ,
2019-10-16 11:59:59 -04:00
ConditionValues : getConditionValues ( r , "" , "" , nil ) ,
2018-10-09 17:00:01 -04:00
IsOwner : false ,
} )
// Check if anonymous (non-owner) has access to upload objects.
writable := globalPolicySys . IsAllowed ( policy . Args {
Action : policy . PutObjectAction ,
BucketName : args . BucketName ,
2019-10-16 11:59:59 -04:00
ConditionValues : getConditionValues ( r , "" , "" , nil ) ,
2018-10-09 17:00:01 -04:00
IsOwner : false ,
2019-08-06 15:08:58 -04:00
ObjectName : args . Prefix + SlashSeparator ,
2018-10-09 17:00:01 -04:00
} )
2018-04-24 18:53:30 -04:00
2018-10-09 17:00:01 -04:00
reply . Writable = writable
if ! readable {
// Error out if anonymous user (non-owner) has no access to download or upload objects
if ! writable {
2018-10-17 19:23:09 -04:00
return errAccessDenied
2018-10-09 17:00:01 -04:00
}
// return empty object list if access is write only
return nil
}
} else {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , authErr )
2018-04-24 18:53:30 -04:00
}
2018-10-09 17:00:01 -04:00
}
// For authenticated users apply IAM policy.
if authErr == nil {
2019-01-20 02:20:01 -05:00
// Set prefix value for "s3:prefix" policy conditionals.
r . Header . Set ( "prefix" , args . Prefix )
// Set delimiter value for "s3:delimiter" policy conditionals.
2019-08-06 15:08:58 -04:00
r . Header . Set ( "delimiter" , SlashSeparator )
2018-12-13 23:15:09 -05:00
2018-10-09 17:00:01 -04:00
readable := globalIAMSys . IsAllowed ( iampolicy . Args {
2020-01-30 21:59:22 -05:00
AccountName : claims . AccessKey ,
2018-12-13 23:15:09 -05:00
Action : iampolicy . ListBucketAction ,
2018-10-09 17:00:01 -04:00
BucketName : args . BucketName ,
2020-01-30 21:59:22 -05:00
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
2018-10-09 17:00:01 -04:00
IsOwner : owner ,
2019-10-23 01:59:13 -04:00
Claims : claims . Map ( ) ,
2018-10-09 17:00:01 -04:00
} )
writable := globalIAMSys . IsAllowed ( iampolicy . Args {
2020-01-30 21:59:22 -05:00
AccountName : claims . AccessKey ,
2018-12-13 23:15:09 -05:00
Action : iampolicy . PutObjectAction ,
2018-10-09 17:00:01 -04:00
BucketName : args . BucketName ,
2020-01-30 21:59:22 -05:00
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
2018-10-09 17:00:01 -04:00
IsOwner : owner ,
2019-08-06 15:08:58 -04:00
ObjectName : args . Prefix + SlashSeparator ,
2019-10-23 01:59:13 -04:00
Claims : claims . Map ( ) ,
2018-10-09 17:00:01 -04:00
} )
2018-04-24 18:53:30 -04:00
2018-08-31 16:20:27 -04:00
reply . Writable = writable
if ! readable {
// Error out if anonymous user (non-owner) has no access to download or upload objects
if ! writable {
2018-10-17 19:23:09 -04:00
return errAccessDenied
2018-08-31 16:20:27 -04:00
}
// return empty object list if access is write only
return nil
2018-04-24 18:53:30 -04:00
}
2016-01-23 22:44:32 -05:00
}
2018-04-24 18:53:30 -04:00
2019-04-04 02:10:37 -04:00
// Check if bucket is a reserved bucket name or invalid.
if isReservedOrInvalidBucket ( args . BucketName , false ) {
2020-07-10 15:10:39 -04:00
return toJSONError ( ctx , errInvalidBucketName , args . BucketName )
2019-04-04 02:10:37 -04:00
}
2019-06-25 19:31:50 -04:00
nextMarker := ""
// Fetch all the objects
for {
2020-12-23 22:58:15 -05:00
// Limit browser to '1000' batches to be more responsive, scrolling friendly.
// Also don't change the maxKeys value silly GCS SDKs do not honor maxKeys
// values to be '-1'
lo , err := listObjects ( ctx , args . BucketName , args . Prefix , nextMarker , SlashSeparator , 1000 )
2019-06-25 19:31:50 -04:00
if err != nil {
return & json2 . Error { Message : err . Error ( ) }
}
2020-12-19 12:36:04 -05:00
nextMarker = lo . NextMarker
2019-06-25 19:31:50 -04:00
for i := range lo . Objects {
2020-05-24 14:19:17 -04:00
lo . Objects [ i ] . Size , err = lo . Objects [ i ] . GetActualSize ( )
if err != nil {
return toJSONError ( ctx , err )
2018-08-17 15:52:14 -04:00
}
}
2017-02-11 01:54:42 -05:00
2019-06-25 19:31:50 -04:00
for _ , obj := range lo . Objects {
reply . Objects = append ( reply . Objects , WebObjectInfo {
Key : obj . Name ,
LastModified : obj . ModTime ,
Size : obj . Size ,
ContentType : obj . ContentType ,
} )
}
for _ , prefix := range lo . Prefixes {
reply . Objects = append ( reply . Objects , WebObjectInfo {
Key : prefix ,
} )
}
// Return when there are no more objects
if ! lo . IsTruncated {
return nil
}
}
2016-01-23 22:44:32 -05:00
}
2017-04-27 02:27:48 -04:00
// RemoveObjectArgs - args to remove an object, JSON will look like.
//
// {
// "bucketname": "testbucket",
// "objects": [
// "photos/hawaii/",
// "photos/maldives/",
// "photos/sanjose.jpg"
// ]
// }
2016-02-19 03:00:32 -05:00
type RemoveObjectArgs struct {
2017-04-27 02:27:48 -04:00
Objects [ ] string ` json:"objects" ` // Contains objects, prefixes.
BucketName string ` json:"bucketname" ` // Contains bucket name.
2016-02-19 03:00:32 -05:00
}
2017-04-27 02:27:48 -04:00
// RemoveObject - removes an object, or all the objects at a given prefix.
2016-04-12 15:45:15 -04:00
func ( web * webAPIHandlers ) RemoveObject ( r * http . Request , args * RemoveObjectArgs , reply * WebGenericRep ) error {
2019-10-23 01:59:13 -04:00
ctx := newWebContext ( r , args , "WebRemoveObject" )
2016-07-31 17:11:14 -04:00
objectAPI := web . ObjectAPI ( )
if objectAPI == nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , errServerNotInitialized )
2016-11-02 17:45:11 -04:00
}
2020-04-06 16:44:16 -04:00
deleteObjects := objectAPI . DeleteObjects
if web . CacheAPI ( ) != nil {
deleteObjects = web . CacheAPI ( ) . DeleteObjects
}
2020-11-19 21:43:58 -05:00
getObjectInfoFn := objectAPI . GetObjectInfo
if web . CacheAPI ( ) != nil {
getObjectInfoFn = web . CacheAPI ( ) . GetObjectInfo
}
2020-04-06 16:44:16 -04:00
2018-10-17 19:23:09 -04:00
claims , owner , authErr := webRequestAuthenticate ( r )
if authErr != nil {
2019-04-22 10:54:43 -04:00
if authErr == errNoAuthToken {
// Check if all objects are allowed to be deleted anonymously
for _ , object := range args . Objects {
if ! globalPolicySys . IsAllowed ( policy . Args {
Action : policy . DeleteObjectAction ,
BucketName : args . BucketName ,
2019-10-16 11:59:59 -04:00
ConditionValues : getConditionValues ( r , "" , "" , nil ) ,
2019-04-22 10:54:43 -04:00
IsOwner : false ,
ObjectName : object ,
} ) {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , errAuthentication )
2019-04-22 10:54:43 -04:00
}
}
} else {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , authErr )
2019-04-22 10:54:43 -04:00
}
2016-07-31 17:11:14 -04:00
}
2017-04-27 02:27:48 -04:00
2017-02-28 22:07:28 -05:00
if args . BucketName == "" || len ( args . Objects ) == 0 {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , errInvalidArgument )
2017-02-28 22:07:28 -05:00
}
2017-04-27 02:27:48 -04:00
2019-04-04 02:10:37 -04:00
// Check if bucket is a reserved bucket name or invalid.
if isReservedOrInvalidBucket ( args . BucketName , false ) {
2020-07-10 15:10:39 -04:00
return toJSONError ( ctx , errInvalidBucketName , args . BucketName )
2019-04-04 02:10:37 -04:00
}
2018-12-19 08:13:47 -05:00
reply . UIVersion = browser . UIVersion
2019-06-03 18:40:04 -04:00
if isRemoteCallRequired ( ctx , args . BucketName , objectAPI ) {
2018-12-19 08:13:47 -05:00
sr , err := globalDNSConfig . Get ( args . BucketName )
if err != nil {
if err == dns . ErrNoEntriesFound {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , BucketNotFound {
2018-12-19 08:13:47 -05:00
Bucket : args . BucketName ,
} , args . BucketName )
}
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
2020-09-12 02:03:08 -04:00
core , err := getRemoteInstanceClient ( r , getHostFromSrv ( sr ) )
2018-12-19 08:13:47 -05:00
if err != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
2020-07-17 01:38:58 -04:00
objectsCh := make ( chan miniogo . ObjectInfo )
2018-12-19 08:13:47 -05:00
// Send object names that are needed to be removed to objectsCh
go func ( ) {
defer close ( objectsCh )
for _ , objectName := range args . Objects {
2020-07-17 01:38:58 -04:00
objectsCh <- miniogo . ObjectInfo {
Key : objectName ,
}
2018-12-19 08:13:47 -05:00
}
} ( )
2020-07-14 12:38:05 -04:00
for resp := range core . RemoveObjects ( ctx , args . BucketName , objectsCh , minio . RemoveObjectsOptions { } ) {
2018-12-19 08:13:47 -05:00
if resp . Err != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , resp . Err , args . BucketName , resp . ObjectName )
2018-12-19 08:13:47 -05:00
}
}
return nil
}
2020-07-11 01:21:04 -04:00
opts := ObjectOptions {
2020-09-02 03:19:03 -04:00
Versioned : globalBucketVersioningSys . Enabled ( args . BucketName ) ,
VersionSuspended : globalBucketVersioningSys . Suspended ( args . BucketName ) ,
2020-07-11 01:21:04 -04:00
}
2021-01-12 01:36:51 -05:00
var (
err error
replicateSync bool
)
2017-04-27 02:27:48 -04:00
next :
for _ , objectName := range args . Objects {
// If not a directory, remove the object.
2019-12-06 02:16:06 -05:00
if ! HasSuffix ( objectName , SlashSeparator ) && objectName != "" {
2020-05-03 17:01:28 -04:00
// Check permissions for non-anonymous user.
2019-04-22 10:54:43 -04:00
if authErr != errNoAuthToken {
if ! globalIAMSys . IsAllowed ( iampolicy . Args {
2020-01-30 21:59:22 -05:00
AccountName : claims . AccessKey ,
2019-04-22 10:54:43 -04:00
Action : iampolicy . DeleteObjectAction ,
BucketName : args . BucketName ,
2020-01-30 21:59:22 -05:00
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
2019-04-22 10:54:43 -04:00
IsOwner : owner ,
ObjectName : objectName ,
2019-10-23 01:59:13 -04:00
Claims : claims . Map ( ) ,
2019-04-22 10:54:43 -04:00
} ) {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , errAccessDenied )
2019-04-22 10:54:43 -04:00
}
2019-11-20 16:18:09 -05:00
}
2020-05-03 17:01:28 -04:00
2019-11-20 16:18:09 -05:00
if authErr == errNoAuthToken {
2020-05-03 17:01:28 -04:00
// Check if object is allowed to be deleted anonymously.
2020-03-10 00:21:35 -04:00
if ! globalPolicySys . IsAllowed ( policy . Args {
Action : policy . DeleteObjectAction ,
BucketName : args . BucketName ,
ConditionValues : getConditionValues ( r , "" , "" , nil ) ,
IsOwner : false ,
ObjectName : objectName ,
} ) {
return toJSONError ( ctx , errAccessDenied )
}
2018-10-17 19:23:09 -04:00
}
2020-11-25 14:24:50 -05:00
var (
replicateDel , hasLifecycleConfig bool
goi ObjectInfo
gerr error
)
if _ , err := globalBucketMetadataSys . GetLifecycleConfig ( args . BucketName ) ; err == nil {
hasLifecycleConfig = true
}
if hasReplicationRules ( ctx , args . BucketName , [ ] ObjectToDelete { { ObjectName : objectName } } ) || hasLifecycleConfig {
goi , gerr = getObjectInfoFn ( ctx , args . BucketName , objectName , opts )
2021-01-12 01:36:51 -05:00
if _ , replicateDel , replicateSync = checkReplicateDelete ( ctx , args . BucketName , ObjectToDelete { ObjectName : objectName , VersionID : goi . VersionID } , goi , gerr ) ; replicateDel {
2020-11-25 14:24:50 -05:00
opts . DeleteMarkerReplicationStatus = string ( replication . Pending )
opts . DeleteMarker = true
}
2020-11-19 21:43:58 -05:00
}
2021-02-11 01:00:42 -05:00
deleteObject := objectAPI . DeleteObject
if web . CacheAPI ( ) != nil {
deleteObject = web . CacheAPI ( ) . DeleteObject
}
oi , err := deleteObject ( ctx , args . BucketName , objectName , opts )
if err != nil {
switch err . ( type ) {
case BucketNotFound :
return toJSONError ( ctx , err )
}
}
if oi . Name == "" {
logger . LogIf ( ctx , err )
continue
}
eventName := event . ObjectRemovedDelete
if oi . DeleteMarker {
eventName = event . ObjectRemovedDeleteMarkerCreated
}
// Notify object deleted event.
sendEvent ( eventArgs {
EventName : eventName ,
BucketName : args . BucketName ,
Object : oi ,
ReqParams : extractReqParams ( r ) ,
UserAgent : r . UserAgent ( ) ,
Host : handlers . GetSourceIP ( r ) ,
} )
if replicateDel {
2021-01-12 01:36:51 -05:00
dobj := DeletedObjectVersionInfo {
2020-11-19 21:43:58 -05:00
DeletedObject : DeletedObject {
ObjectName : objectName ,
DeleteMarkerVersionID : oi . VersionID ,
DeleteMarkerReplicationStatus : string ( oi . ReplicationStatus ) ,
2020-11-29 00:15:45 -05:00
DeleteMarkerMTime : DeleteMarkerMTime { oi . ModTime } ,
2020-11-19 21:43:58 -05:00
DeleteMarker : oi . DeleteMarker ,
VersionPurgeStatus : oi . VersionPurgeStatus ,
} ,
Bucket : args . BucketName ,
2021-01-12 01:36:51 -05:00
}
scheduleReplicationDelete ( ctx , dobj , objectAPI , replicateSync )
2020-11-19 21:43:58 -05:00
}
2021-02-11 01:00:42 -05:00
if goi . TransitionStatus == lifecycle . TransitionComplete {
deleteTransitionedObject ( ctx , objectAPI , args . BucketName , objectName , lifecycle . ObjectOpts {
Name : objectName ,
UserTags : goi . UserTags ,
VersionID : goi . VersionID ,
DeleteMarker : goi . DeleteMarker ,
TransitionStatus : goi . TransitionStatus ,
IsLatest : goi . IsLatest ,
2021-02-01 12:52:11 -05:00
} , false , true )
2020-11-25 14:24:50 -05:00
}
2020-04-06 16:44:16 -04:00
2020-06-12 23:04:01 -04:00
logger . LogIf ( ctx , err )
2020-07-24 16:16:11 -04:00
continue
2017-02-28 22:07:28 -05:00
}
2017-04-27 02:27:48 -04:00
2020-03-10 00:21:35 -04:00
if authErr == errNoAuthToken {
// Check if object is allowed to be deleted anonymously
if ! globalPolicySys . IsAllowed ( policy . Args {
Action : iampolicy . DeleteObjectAction ,
BucketName : args . BucketName ,
ConditionValues : getConditionValues ( r , "" , "" , nil ) ,
IsOwner : false ,
ObjectName : objectName ,
} ) {
return toJSONError ( ctx , errAccessDenied )
}
} else {
if ! globalIAMSys . IsAllowed ( iampolicy . Args {
AccountName : claims . AccessKey ,
Action : iampolicy . DeleteObjectAction ,
BucketName : args . BucketName ,
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
IsOwner : owner ,
ObjectName : objectName ,
Claims : claims . Map ( ) ,
} ) {
return toJSONError ( ctx , errAccessDenied )
}
2018-10-17 19:23:09 -04:00
}
2020-04-06 16:44:16 -04:00
// Allocate new results channel to receive ObjectInfo.
objInfoCh := make ( chan ObjectInfo )
// Walk through all objects
2020-07-11 01:21:04 -04:00
if err = objectAPI . Walk ( ctx , args . BucketName , objectName , objInfoCh , ObjectOptions { } ) ; err != nil {
2020-04-06 16:44:16 -04:00
break next
}
2017-02-28 22:07:28 -05:00
for {
2020-06-12 23:04:01 -04:00
var objects [ ] ObjectToDelete
2020-04-06 16:44:16 -04:00
for obj := range objInfoCh {
2020-05-25 19:51:32 -04:00
if len ( objects ) == maxDeleteList {
2020-04-06 16:44:16 -04:00
// Reached maximum delete requests, attempt a delete for now.
break
2017-02-28 22:07:28 -05:00
}
2020-11-19 21:43:58 -05:00
if obj . ReplicationStatus == replication . Replica {
if authErr == errNoAuthToken {
// Check if object is allowed to be deleted anonymously
if ! globalPolicySys . IsAllowed ( policy . Args {
Action : iampolicy . ReplicateDeleteAction ,
BucketName : args . BucketName ,
ConditionValues : getConditionValues ( r , "" , "" , nil ) ,
IsOwner : false ,
ObjectName : objectName ,
} ) {
return toJSONError ( ctx , errAccessDenied )
}
} else {
if ! globalIAMSys . IsAllowed ( iampolicy . Args {
AccountName : claims . AccessKey ,
Action : iampolicy . ReplicateDeleteAction ,
BucketName : args . BucketName ,
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
IsOwner : owner ,
ObjectName : objectName ,
Claims : claims . Map ( ) ,
} ) {
return toJSONError ( ctx , errAccessDenied )
}
}
}
2021-01-12 01:36:51 -05:00
_ , replicateDel , _ := checkReplicateDelete ( ctx , args . BucketName , ObjectToDelete { ObjectName : obj . Name , VersionID : obj . VersionID } , obj , nil )
2020-11-19 21:43:58 -05:00
// since versioned delete is not available on web browser, yet - this is a simple DeleteMarker replication
objToDel := ObjectToDelete { ObjectName : obj . Name }
if replicateDel {
objToDel . DeleteMarkerReplicationStatus = string ( replication . Pending )
}
objects = append ( objects , objToDel )
2020-04-06 16:44:16 -04:00
}
// Nothing to do.
if len ( objects ) == 0 {
break next
2017-02-28 22:07:28 -05:00
}
2020-04-06 16:44:16 -04:00
// Deletes a list of objects.
2020-10-02 16:36:13 -04:00
deletedObjects , errs := deleteObjects ( ctx , args . BucketName , objects , opts )
2020-11-19 21:43:58 -05:00
for i , err := range errs {
if err != nil && ! isErrObjectNotFound ( err ) {
deletedObjects [ i ] . DeleteMarkerReplicationStatus = objects [ i ] . DeleteMarkerReplicationStatus
deletedObjects [ i ] . VersionPurgeStatus = objects [ i ] . VersionPurgeStatus
}
2020-06-12 23:04:01 -04:00
if err != nil {
logger . LogIf ( ctx , err )
break next
}
2017-02-28 22:07:28 -05:00
}
2020-10-02 16:36:13 -04:00
// Notify deleted event for objects.
for _ , dobj := range deletedObjects {
objInfo := ObjectInfo {
Name : dobj . ObjectName ,
VersionID : dobj . VersionID ,
}
if dobj . DeleteMarker {
objInfo = ObjectInfo {
Name : dobj . ObjectName ,
DeleteMarker : dobj . DeleteMarker ,
VersionID : dobj . DeleteMarkerVersionID ,
}
}
sendEvent ( eventArgs {
EventName : event . ObjectRemovedDelete ,
BucketName : args . BucketName ,
Object : objInfo ,
ReqParams : extractReqParams ( r ) ,
UserAgent : r . UserAgent ( ) ,
Host : handlers . GetSourceIP ( r ) ,
} )
2020-11-19 21:43:58 -05:00
if dobj . DeleteMarkerReplicationStatus == string ( replication . Pending ) || dobj . VersionPurgeStatus == Pending {
2021-01-12 01:36:51 -05:00
dv := DeletedObjectVersionInfo {
2020-11-19 21:43:58 -05:00
DeletedObject : dobj ,
Bucket : args . BucketName ,
2021-01-12 01:36:51 -05:00
}
scheduleReplicationDelete ( ctx , dv , objectAPI , replicateSync )
2020-11-19 21:43:58 -05:00
}
2020-10-02 16:36:13 -04:00
}
2016-11-10 18:02:03 -05:00
}
2016-02-12 21:55:17 -05:00
}
2016-11-10 10:42:55 -05:00
2020-07-02 19:17:27 -04:00
if err != nil && ! isErrObjectNotFound ( err ) && ! isErrVersionNotFound ( err ) {
2017-02-28 22:07:28 -05:00
// Ignore object not found error.
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName , "" )
2017-02-28 22:07:28 -05:00
}
2016-11-10 10:42:55 -05:00
2016-02-12 21:55:17 -05:00
return nil
2016-02-05 09:16:36 -05:00
}
2016-02-19 03:00:32 -05:00
// LoginArgs - login arguments.
type LoginArgs struct {
Username string ` json:"username" form:"username" `
Password string ` json:"password" form:"password" `
}
// LoginRep - login reply.
type LoginRep struct {
Token string ` json:"token" `
UIVersion string ` json:"uiVersion" `
}
2016-01-23 22:44:32 -05:00
// Login - user login handler.
2016-04-12 15:45:15 -04:00
func ( web * webAPIHandlers ) Login ( r * http . Request , args * LoginArgs , reply * LoginRep ) error {
2019-10-23 01:59:13 -04:00
ctx := newWebContext ( r , args , "WebLogin" )
2016-12-27 11:28:10 -05:00
token , err := authenticateWeb ( args . Username , args . Password )
2016-07-12 00:57:40 -04:00
if err != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err )
2016-07-12 00:57:40 -04:00
}
reply . Token = token
2017-02-22 20:27:26 -05:00
reply . UIVersion = browser . UIVersion
2016-07-12 00:57:40 -04:00
return nil
2016-01-23 22:44:32 -05:00
}
2016-03-21 14:15:08 -04:00
// SetAuthArgs - argument for SetAuth
type SetAuthArgs struct {
2019-05-29 16:18:46 -04:00
CurrentAccessKey string ` json:"currentAccessKey" `
CurrentSecretKey string ` json:"currentSecretKey" `
NewAccessKey string ` json:"newAccessKey" `
NewSecretKey string ` json:"newSecretKey" `
2016-03-21 14:15:08 -04:00
}
// SetAuthReply - reply for SetAuth
type SetAuthReply struct {
2016-10-17 23:18:08 -04:00
Token string ` json:"token" `
UIVersion string ` json:"uiVersion" `
PeerErrMsgs map [ string ] string ` json:"peerErrMsgs" `
2016-03-21 14:15:08 -04:00
}
// SetAuth - Set accessKey and secretKey credentials.
2016-04-12 15:45:15 -04:00
func ( web * webAPIHandlers ) SetAuth ( r * http . Request , args * SetAuthArgs , reply * SetAuthReply ) error {
2019-10-23 01:59:13 -04:00
ctx := newWebContext ( r , args , "WebSetAuth" )
2019-05-29 16:18:46 -04:00
claims , owner , authErr := webRequestAuthenticate ( r )
2018-10-17 19:23:09 -04:00
if authErr != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , authErr )
2016-03-21 14:15:08 -04:00
}
2016-11-22 14:12:38 -05:00
2019-05-29 16:18:46 -04:00
if owner {
2019-10-23 01:59:13 -04:00
// Owner is not allowed to change credentials through browser.
return toJSONError ( ctx , errChangeCredNotAllowed )
}
2016-03-31 09:57:29 -04:00
2021-03-02 18:35:50 -05:00
if ! globalIAMSys . IsAllowed ( iampolicy . Args {
AccountName : claims . AccessKey ,
Action : iampolicy . CreateUserAdminAction ,
IsOwner : false ,
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
Claims : claims . Map ( ) ,
DenyOnly : true ,
} ) {
return toJSONError ( ctx , errChangeCredNotAllowed )
}
2019-10-23 01:59:13 -04:00
// for IAM users, access key cannot be updated
2020-01-30 21:59:22 -05:00
// claims.AccessKey is used instead of accesskey from args
prevCred , ok := globalIAMSys . GetUser ( claims . AccessKey )
2019-10-23 01:59:13 -04:00
if ! ok {
return errInvalidAccessKeyID
}
2019-05-29 16:18:46 -04:00
2019-10-23 01:59:13 -04:00
// Throw error when wrong secret key is provided
2020-12-20 13:09:51 -05:00
if subtle . ConstantTimeCompare ( [ ] byte ( prevCred . SecretKey ) , [ ] byte ( args . CurrentSecretKey ) ) != 1 {
2019-10-23 01:59:13 -04:00
return errIncorrectCreds
}
2019-05-29 16:18:46 -04:00
2020-01-30 21:59:22 -05:00
creds , err := auth . CreateCredentials ( claims . AccessKey , args . NewSecretKey )
2019-10-23 01:59:13 -04:00
if err != nil {
return toJSONError ( ctx , err )
}
2019-05-30 08:14:35 -04:00
2019-10-23 01:59:13 -04:00
err = globalIAMSys . SetUserSecretKey ( creds . AccessKey , creds . SecretKey )
if err != nil {
return toJSONError ( ctx , err )
}
2019-06-03 18:40:04 -04:00
2019-10-23 01:59:13 -04:00
reply . Token , err = authenticateWeb ( creds . AccessKey , creds . SecretKey )
if err != nil {
return toJSONError ( ctx , err )
2016-03-31 09:57:29 -04:00
}
2019-05-29 16:18:46 -04:00
2017-02-22 20:27:26 -05:00
reply . UIVersion = browser . UIVersion
2019-05-29 16:18:46 -04:00
2016-03-31 09:57:29 -04:00
return nil
}
2017-07-24 15:46:37 -04:00
// URLTokenReply contains the reply for CreateURLToken.
type URLTokenReply struct {
Token string ` json:"token" `
UIVersion string ` json:"uiVersion" `
}
// CreateURLToken creates a URL token (short-lived) for GET requests.
func ( web * webAPIHandlers ) CreateURLToken ( r * http . Request , args * WebGenericArgs , reply * URLTokenReply ) error {
2019-10-23 01:59:13 -04:00
ctx := newWebContext ( r , args , "WebCreateURLToken" )
2018-10-17 19:23:09 -04:00
claims , owner , authErr := webRequestAuthenticate ( r )
if authErr != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , authErr )
2017-07-24 15:46:37 -04:00
}
2019-10-23 01:59:13 -04:00
creds := globalActiveCred
2018-10-17 19:23:09 -04:00
if ! owner {
var ok bool
2020-01-30 21:59:22 -05:00
creds , ok = globalIAMSys . GetUser ( claims . AccessKey )
2018-10-17 19:23:09 -04:00
if ! ok {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , errInvalidAccessKeyID )
2018-10-17 19:23:09 -04:00
}
}
2017-07-24 15:46:37 -04:00
2019-10-23 01:59:13 -04:00
if creds . SessionToken != "" {
// Use the same session token for URL token.
reply . Token = creds . SessionToken
} else {
token , err := authenticateURL ( creds . AccessKey , creds . SecretKey )
if err != nil {
return toJSONError ( ctx , err )
}
reply . Token = token
2017-07-24 15:46:37 -04:00
}
reply . UIVersion = browser . UIVersion
return nil
}
2016-03-31 09:57:29 -04:00
// Upload - file upload handler.
2016-04-12 15:45:15 -04:00
func ( web * webAPIHandlers ) Upload ( w http . ResponseWriter , r * http . Request ) {
2018-10-12 15:25:59 -04:00
ctx := newContext ( r , w , "WebUpload" )
2020-05-27 15:38:44 -04:00
// obtain the claims here if possible, for audit logging.
claims , owner , authErr := webRequestAuthenticate ( r )
2021-01-26 16:21:51 -05:00
defer logger . AuditLog ( ctx , w , r , claims . Map ( ) )
2018-10-12 15:25:59 -04:00
2016-11-02 17:45:11 -04:00
objectAPI := web . ObjectAPI ( )
if objectAPI == nil {
writeWebErrorResponse ( w , errServerNotInitialized )
return
}
2020-05-27 15:38:44 -04:00
2016-03-31 09:57:29 -04:00
vars := mux . Vars ( r )
bucket := vars [ "bucket" ]
2020-02-11 22:38:02 -05:00
object , err := url . PathUnescape ( vars [ "object" ] )
if err != nil {
writeWebErrorResponse ( w , err )
return
}
2016-07-28 00:11:15 -04:00
2019-11-20 16:18:09 -05:00
retPerms := ErrAccessDenied
2020-01-16 18:41:56 -05:00
holdPerms := ErrAccessDenied
2020-08-12 20:32:24 -04:00
replPerms := ErrAccessDenied
2018-10-09 17:00:01 -04:00
if authErr != nil {
if authErr == errNoAuthToken {
// Check if anonymous (non-owner) has access to upload objects.
if ! globalPolicySys . IsAllowed ( policy . Args {
Action : policy . PutObjectAction ,
BucketName : bucket ,
2019-10-16 11:59:59 -04:00
ConditionValues : getConditionValues ( r , "" , "" , nil ) ,
2018-10-09 17:00:01 -04:00
IsOwner : false ,
ObjectName : object ,
} ) {
writeWebErrorResponse ( w , errAuthentication )
return
}
} else {
writeWebErrorResponse ( w , authErr )
2018-04-24 18:53:30 -04:00
return
}
2018-10-09 17:00:01 -04:00
}
2018-04-24 18:53:30 -04:00
2018-10-09 17:00:01 -04:00
// For authenticated users apply IAM policy.
if authErr == nil {
if ! globalIAMSys . IsAllowed ( iampolicy . Args {
2020-01-30 21:59:22 -05:00
AccountName : claims . AccessKey ,
2018-12-13 23:15:09 -05:00
Action : iampolicy . PutObjectAction ,
2018-04-24 18:53:30 -04:00
BucketName : bucket ,
2020-01-30 21:59:22 -05:00
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
2018-10-09 17:00:01 -04:00
IsOwner : owner ,
2018-04-24 18:53:30 -04:00
ObjectName : object ,
2019-10-23 01:59:13 -04:00
Claims : claims . Map ( ) ,
2018-04-24 18:53:30 -04:00
} ) {
writeWebErrorResponse ( w , errAuthentication )
return
}
2019-11-20 16:18:09 -05:00
if globalIAMSys . IsAllowed ( iampolicy . Args {
2020-01-30 21:59:22 -05:00
AccountName : claims . AccessKey ,
2019-11-20 16:18:09 -05:00
Action : iampolicy . PutObjectRetentionAction ,
BucketName : bucket ,
2020-01-30 21:59:22 -05:00
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
2019-11-20 16:18:09 -05:00
IsOwner : owner ,
ObjectName : object ,
Claims : claims . Map ( ) ,
} ) {
retPerms = ErrNone
}
2020-01-16 18:41:56 -05:00
if globalIAMSys . IsAllowed ( iampolicy . Args {
2020-01-30 21:59:22 -05:00
AccountName : claims . AccessKey ,
2020-01-16 18:41:56 -05:00
Action : iampolicy . PutObjectLegalHoldAction ,
BucketName : bucket ,
2020-01-30 21:59:22 -05:00
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
2020-01-16 18:41:56 -05:00
IsOwner : owner ,
ObjectName : object ,
Claims : claims . Map ( ) ,
} ) {
holdPerms = ErrNone
}
2020-08-12 20:32:24 -04:00
if globalIAMSys . IsAllowed ( iampolicy . Args {
AccountName : claims . AccessKey ,
Action : iampolicy . GetReplicationConfigurationAction ,
BucketName : bucket ,
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
IsOwner : owner ,
2020-09-24 15:25:41 -04:00
ObjectName : "" ,
2020-08-12 20:32:24 -04:00
Claims : claims . Map ( ) ,
} ) {
replPerms = ErrNone
}
2017-01-11 16:26:42 -05:00
}
2019-04-04 02:10:37 -04:00
// Check if bucket is a reserved bucket name or invalid.
if isReservedOrInvalidBucket ( bucket , false ) {
writeWebErrorResponse ( w , errInvalidBucketName )
return
}
2020-02-05 04:42:34 -05:00
// Check if bucket encryption is enabled
2020-05-19 16:53:54 -04:00
_ , err = globalBucketSSEConfigSys . Get ( bucket )
if ( globalAutoEncryption || err == nil ) && ! crypto . SSEC . IsRequested ( r . Header ) {
2020-12-22 12:19:32 -05:00
r . Header . Set ( xhttp . AmzServerSideEncryption , xhttp . AmzEncryptionAES )
2018-12-14 16:35:48 -05:00
}
2017-01-11 16:26:42 -05:00
2017-02-02 13:45:00 -05:00
// Require Content-Length to be set in the request
size := r . ContentLength
if size < 0 {
writeWebErrorResponse ( w , errSizeUnspecified )
return
}
2020-07-25 00:16:54 -04:00
if err := enforceBucketQuota ( ctx , bucket , size ) ; err != nil {
writeWebErrorResponse ( w , err )
return
}
2016-07-28 00:11:15 -04:00
// Extract incoming metadata if any.
2019-06-03 18:40:04 -04:00
metadata , err := extractMetadata ( ctx , r )
2017-07-05 19:56:10 -04:00
if err != nil {
2019-02-13 19:07:21 -05:00
writeErrorResponse ( ctx , w , toAPIError ( ctx , err ) , r . URL , guessIsBrowserReq ( r ) )
2017-07-05 19:56:10 -04:00
return
}
2016-07-28 00:11:15 -04:00
2018-12-14 16:35:48 -05:00
var pReader * PutObjReader
var reader io . Reader = r . Body
2018-09-27 23:36:17 -04:00
actualSize := size
pkg/etag: add new package for S3 ETag handling (#11577)
This commit adds a new package `etag` for dealing
with S3 ETags.
Even though ETag is often viewed as MD5 checksum of
an object, handling S3 ETags correctly is a surprisingly
complex task. While it is true that the ETag corresponds
to the MD5 for the most basic S3 API operations, there are
many exceptions in case of multipart uploads or encryption.
In worse, some S3 clients expect very specific behavior when
it comes to ETags. For example, some clients expect that the
ETag is a double-quoted string and fail otherwise.
Non-AWS compliant ETag handling has been a source of many bugs
in the past.
Therefore, this commit adds a dedicated `etag` package that provides
functionality for parsing, generating and converting S3 ETags.
Further, this commit removes the ETag computation from the `hash`
package. Instead, the `hash` package (i.e. `hash.Reader`) should
focus only on computing and verifying the content-sha256.
One core feature of this commit is to provide a mechanism to
communicate a computed ETag from a low-level `io.Reader` to
a high-level `io.Reader`.
This problem occurs when an S3 server receives a request and
has to compute the ETag of the content. However, the server
may also wrap the initial body with several other `io.Reader`,
e.g. when encrypting or compressing the content:
```
reader := Encrypt(Compress(ETag(content)))
```
In such a case, the ETag should be accessible by the high-level
`io.Reader`.
The `etag` provides a mechanism to wrap `io.Reader` implementations
such that the `ETag` can be accessed by a type-check.
This technique is applied to the PUT, COPY and Upload handlers.
2021-02-23 15:31:53 -05:00
hashReader , err := hash . NewReader ( reader , size , "" , "" , actualSize )
2018-12-14 16:35:48 -05:00
if err != nil {
writeWebErrorResponse ( w , err )
return
}
2020-07-11 01:21:04 -04:00
2018-09-27 23:36:17 -04:00
if objectAPI . IsCompressionSupported ( ) && isCompressible ( r . Header , object ) && size > 0 {
// Storing the compression metadata.
2019-09-26 02:08:24 -04:00
metadata [ ReservedMetadataPrefix + "compression" ] = compressionAlgorithmV2
2021-01-22 15:09:24 -05:00
metadata [ ReservedMetadataPrefix + "actual-size" ] = strconv . FormatInt ( actualSize , 10 )
2018-09-27 23:36:17 -04:00
pkg/etag: add new package for S3 ETag handling (#11577)
This commit adds a new package `etag` for dealing
with S3 ETags.
Even though ETag is often viewed as MD5 checksum of
an object, handling S3 ETags correctly is a surprisingly
complex task. While it is true that the ETag corresponds
to the MD5 for the most basic S3 API operations, there are
many exceptions in case of multipart uploads or encryption.
In worse, some S3 clients expect very specific behavior when
it comes to ETags. For example, some clients expect that the
ETag is a double-quoted string and fail otherwise.
Non-AWS compliant ETag handling has been a source of many bugs
in the past.
Therefore, this commit adds a dedicated `etag` package that provides
functionality for parsing, generating and converting S3 ETags.
Further, this commit removes the ETag computation from the `hash`
package. Instead, the `hash` package (i.e. `hash.Reader`) should
focus only on computing and verifying the content-sha256.
One core feature of this commit is to provide a mechanism to
communicate a computed ETag from a low-level `io.Reader` to
a high-level `io.Reader`.
This problem occurs when an S3 server receives a request and
has to compute the ETag of the content. However, the server
may also wrap the initial body with several other `io.Reader`,
e.g. when encrypting or compressing the content:
```
reader := Encrypt(Compress(ETag(content)))
```
In such a case, the ETag should be accessible by the high-level
`io.Reader`.
The `etag` provides a mechanism to wrap `io.Reader` implementations
such that the `ETag` can be accessed by a type-check.
This technique is applied to the PUT, COPY and Upload handlers.
2021-02-23 15:31:53 -05:00
actualReader , err := hash . NewReader ( reader , actualSize , "" , "" , actualSize )
2018-09-27 23:36:17 -04:00
if err != nil {
writeWebErrorResponse ( w , err )
return
}
2018-09-28 03:44:59 -04:00
2018-09-27 23:36:17 -04:00
// Set compression metrics.
size = - 1 // Since compressed size is un-predictable.
2021-01-22 15:09:24 -05:00
s2c := newS2CompressReader ( actualReader , actualSize )
2019-09-26 02:08:24 -04:00
defer s2c . Close ( )
pkg/etag: add new package for S3 ETag handling (#11577)
This commit adds a new package `etag` for dealing
with S3 ETags.
Even though ETag is often viewed as MD5 checksum of
an object, handling S3 ETags correctly is a surprisingly
complex task. While it is true that the ETag corresponds
to the MD5 for the most basic S3 API operations, there are
many exceptions in case of multipart uploads or encryption.
In worse, some S3 clients expect very specific behavior when
it comes to ETags. For example, some clients expect that the
ETag is a double-quoted string and fail otherwise.
Non-AWS compliant ETag handling has been a source of many bugs
in the past.
Therefore, this commit adds a dedicated `etag` package that provides
functionality for parsing, generating and converting S3 ETags.
Further, this commit removes the ETag computation from the `hash`
package. Instead, the `hash` package (i.e. `hash.Reader`) should
focus only on computing and verifying the content-sha256.
One core feature of this commit is to provide a mechanism to
communicate a computed ETag from a low-level `io.Reader` to
a high-level `io.Reader`.
This problem occurs when an S3 server receives a request and
has to compute the ETag of the content. However, the server
may also wrap the initial body with several other `io.Reader`,
e.g. when encrypting or compressing the content:
```
reader := Encrypt(Compress(ETag(content)))
```
In such a case, the ETag should be accessible by the high-level
`io.Reader`.
The `etag` provides a mechanism to wrap `io.Reader` implementations
such that the `ETag` can be accessed by a type-check.
This technique is applied to the PUT, COPY and Upload handlers.
2021-02-23 15:31:53 -05:00
reader = etag . Wrap ( s2c , actualReader )
hashReader , err = hash . NewReader ( reader , size , "" , "" , actualSize )
2018-12-14 16:35:48 -05:00
if err != nil {
writeWebErrorResponse ( w , err )
return
}
2018-09-27 23:36:17 -04:00
}
2020-07-11 01:21:04 -04:00
2021-01-12 01:36:51 -05:00
mustReplicate , sync := mustReplicateWeb ( ctx , r , bucket , object , metadata , "" , replPerms )
2020-08-12 20:32:24 -04:00
if mustReplicate {
metadata [ xhttp . AmzBucketReplicationStatus ] = string ( replication . Pending )
}
2021-02-10 11:52:50 -05:00
pReader = NewPutObjReader ( hashReader )
2019-01-05 17:16:43 -05:00
// get gateway encryption options
2020-07-11 01:21:04 -04:00
opts , err := putOpts ( ctx , r , bucket , object , metadata )
2019-01-05 17:16:43 -05:00
if err != nil {
2019-02-12 04:25:52 -05:00
writeErrorResponseHeadersOnly ( w , toAPIError ( ctx , err ) )
2019-01-05 17:16:43 -05:00
return
}
2020-07-11 01:21:04 -04:00
2018-12-14 16:35:48 -05:00
if objectAPI . IsEncryptionSupported ( ) {
2020-12-22 12:19:32 -05:00
if _ , ok := crypto . IsRequested ( r . Header ) ; ok && ! HasSuffix ( object , SlashSeparator ) { // handle SSE requests
pkg/etag: add new package for S3 ETag handling (#11577)
This commit adds a new package `etag` for dealing
with S3 ETags.
Even though ETag is often viewed as MD5 checksum of
an object, handling S3 ETags correctly is a surprisingly
complex task. While it is true that the ETag corresponds
to the MD5 for the most basic S3 API operations, there are
many exceptions in case of multipart uploads or encryption.
In worse, some S3 clients expect very specific behavior when
it comes to ETags. For example, some clients expect that the
ETag is a double-quoted string and fail otherwise.
Non-AWS compliant ETag handling has been a source of many bugs
in the past.
Therefore, this commit adds a dedicated `etag` package that provides
functionality for parsing, generating and converting S3 ETags.
Further, this commit removes the ETag computation from the `hash`
package. Instead, the `hash` package (i.e. `hash.Reader`) should
focus only on computing and verifying the content-sha256.
One core feature of this commit is to provide a mechanism to
communicate a computed ETag from a low-level `io.Reader` to
a high-level `io.Reader`.
This problem occurs when an S3 server receives a request and
has to compute the ETag of the content. However, the server
may also wrap the initial body with several other `io.Reader`,
e.g. when encrypting or compressing the content:
```
reader := Encrypt(Compress(ETag(content)))
```
In such a case, the ETag should be accessible by the high-level
`io.Reader`.
The `etag` provides a mechanism to wrap `io.Reader` implementations
such that the `ETag` can be accessed by a type-check.
This technique is applied to the PUT, COPY and Upload handlers.
2021-02-23 15:31:53 -05:00
var (
objectEncryptionKey crypto . ObjectKey
encReader io . Reader
)
encReader , objectEncryptionKey , err = EncryptRequest ( hashReader , r , bucket , object , metadata )
2018-12-14 16:35:48 -05:00
if err != nil {
2019-02-13 19:07:21 -05:00
writeErrorResponse ( ctx , w , toAPIError ( ctx , err ) , r . URL , guessIsBrowserReq ( r ) )
2018-12-14 16:35:48 -05:00
return
}
info := ObjectInfo { Size : size }
2019-05-08 21:35:40 -04:00
// do not try to verify encrypted content
pkg/etag: add new package for S3 ETag handling (#11577)
This commit adds a new package `etag` for dealing
with S3 ETags.
Even though ETag is often viewed as MD5 checksum of
an object, handling S3 ETags correctly is a surprisingly
complex task. While it is true that the ETag corresponds
to the MD5 for the most basic S3 API operations, there are
many exceptions in case of multipart uploads or encryption.
In worse, some S3 clients expect very specific behavior when
it comes to ETags. For example, some clients expect that the
ETag is a double-quoted string and fail otherwise.
Non-AWS compliant ETag handling has been a source of many bugs
in the past.
Therefore, this commit adds a dedicated `etag` package that provides
functionality for parsing, generating and converting S3 ETags.
Further, this commit removes the ETag computation from the `hash`
package. Instead, the `hash` package (i.e. `hash.Reader`) should
focus only on computing and verifying the content-sha256.
One core feature of this commit is to provide a mechanism to
communicate a computed ETag from a low-level `io.Reader` to
a high-level `io.Reader`.
This problem occurs when an S3 server receives a request and
has to compute the ETag of the content. However, the server
may also wrap the initial body with several other `io.Reader`,
e.g. when encrypting or compressing the content:
```
reader := Encrypt(Compress(ETag(content)))
```
In such a case, the ETag should be accessible by the high-level
`io.Reader`.
The `etag` provides a mechanism to wrap `io.Reader` implementations
such that the `ETag` can be accessed by a type-check.
This technique is applied to the PUT, COPY and Upload handlers.
2021-02-23 15:31:53 -05:00
hashReader , err = hash . NewReader ( etag . Wrap ( encReader , hashReader ) , info . EncryptedSize ( ) , "" , "" , size )
2018-12-14 16:35:48 -05:00
if err != nil {
2019-02-13 19:07:21 -05:00
writeErrorResponse ( ctx , w , toAPIError ( ctx , err ) , r . URL , guessIsBrowserReq ( r ) )
2018-12-14 16:35:48 -05:00
return
}
2021-02-10 11:52:50 -05:00
pReader , err = pReader . WithEncryption ( hashReader , & objectEncryptionKey )
if err != nil {
writeErrorResponse ( ctx , w , toAPIError ( ctx , err ) , r . URL , guessIsBrowserReq ( r ) )
return
}
2018-12-14 16:35:48 -05:00
}
2017-10-22 01:30:34 -04:00
}
2018-12-19 08:13:47 -05:00
2018-12-14 16:35:48 -05:00
// Ensure that metadata does not contain sensitive information
crypto . RemoveSensitiveEntries ( metadata )
2020-04-06 16:44:16 -04:00
putObject := objectAPI . PutObject
2019-11-20 16:18:09 -05:00
getObjectInfo := objectAPI . GetObjectInfo
if web . CacheAPI ( ) != nil {
2020-04-06 16:44:16 -04:00
putObject = web . CacheAPI ( ) . PutObject
2019-11-20 16:18:09 -05:00
getObjectInfo = web . CacheAPI ( ) . GetObjectInfo
}
2018-06-06 21:10:51 -04:00
2020-08-18 15:09:42 -04:00
// enforce object retention rules
retentionMode , retentionDate , _ , s3Err := checkPutObjectLockAllowed ( ctx , r , bucket , object , getObjectInfo , retPerms , holdPerms )
if s3Err != ErrNone {
writeErrorResponse ( ctx , w , errorCodes . ToAPIErr ( s3Err ) , r . URL , guessIsBrowserReq ( r ) )
return
}
if retentionMode != "" {
2021-02-08 21:12:28 -05:00
opts . UserDefined [ strings . ToLower ( xhttp . AmzObjectLockMode ) ] = string ( retentionMode )
opts . UserDefined [ strings . ToLower ( xhttp . AmzObjectLockRetainUntilDate ) ] = retentionDate . UTC ( ) . Format ( iso8601TimeFormat )
2019-09-05 10:20:16 -04:00
}
2019-08-09 20:09:08 -04:00
2020-04-09 12:30:02 -04:00
objInfo , err := putObject ( GlobalContext , bucket , object , pReader , opts )
2016-07-28 00:11:15 -04:00
if err != nil {
2016-12-10 19:15:12 -05:00
writeWebErrorResponse ( w , err )
2016-07-28 00:11:15 -04:00
return
}
2018-12-14 16:35:48 -05:00
if objectAPI . IsEncryptionSupported ( ) {
2021-02-03 18:19:08 -05:00
switch kind , _ := crypto . IsEncrypted ( objInfo . UserDefined ) ; kind {
case crypto . S3 :
w . Header ( ) . Set ( xhttp . AmzServerSideEncryption , xhttp . AmzEncryptionAES )
case crypto . SSEC :
w . Header ( ) . Set ( xhttp . AmzServerSideEncryptionCustomerAlgorithm , r . Header . Get ( xhttp . AmzServerSideEncryptionCustomerAlgorithm ) )
w . Header ( ) . Set ( xhttp . AmzServerSideEncryptionCustomerKeyMD5 , r . Header . Get ( xhttp . AmzServerSideEncryptionCustomerKeyMD5 ) )
2018-12-14 16:35:48 -05:00
}
}
2020-08-12 20:32:24 -04:00
if mustReplicate {
2021-02-03 23:41:33 -05:00
scheduleReplication ( ctx , objInfo . Clone ( ) , objectAPI , sync )
2020-08-12 20:32:24 -04:00
}
2020-09-16 19:04:55 -04:00
2016-09-29 01:46:19 -04:00
// Notify object created event.
2018-03-15 16:03:41 -04:00
sendEvent ( eventArgs {
2018-11-02 21:40:08 -04:00
EventName : event . ObjectCreatedPut ,
BucketName : bucket ,
Object : objInfo ,
ReqParams : extractReqParams ( r ) ,
RespElements : extractRespElements ( w ) ,
UserAgent : r . UserAgent ( ) ,
2019-03-25 14:45:42 -04:00
Host : handlers . GetSourceIP ( r ) ,
2016-09-29 01:46:19 -04:00
} )
2016-03-31 09:57:29 -04:00
}
// Download - file download handler.
2016-04-12 15:45:15 -04:00
func ( web * webAPIHandlers ) Download ( w http . ResponseWriter , r * http . Request ) {
2018-10-12 15:25:59 -04:00
ctx := newContext ( r , w , "WebDownload" )
2020-05-27 15:38:44 -04:00
claims , owner , authErr := webTokenAuthenticate ( r . URL . Query ( ) . Get ( "token" ) )
2021-01-26 16:21:51 -05:00
defer logger . AuditLog ( ctx , w , r , claims . Map ( ) )
2018-10-12 15:25:59 -04:00
2016-11-02 17:45:11 -04:00
objectAPI := web . ObjectAPI ( )
if objectAPI == nil {
writeWebErrorResponse ( w , errServerNotInitialized )
return
}
2021-01-26 16:21:51 -05:00
vars := mux . Vars ( r )
2016-03-31 09:57:29 -04:00
bucket := vars [ "bucket" ]
2020-02-11 22:38:02 -05:00
object , err := url . PathUnescape ( vars [ "object" ] )
if err != nil {
writeWebErrorResponse ( w , err )
return
}
2016-07-12 00:57:40 -04:00
2019-11-20 16:18:09 -05:00
getRetPerms := ErrAccessDenied
2020-01-16 18:41:56 -05:00
legalHoldPerms := ErrAccessDenied
2019-11-20 16:18:09 -05:00
2018-10-09 17:00:01 -04:00
if authErr != nil {
if authErr == errNoAuthToken {
// Check if anonymous (non-owner) has access to download objects.
if ! globalPolicySys . IsAllowed ( policy . Args {
Action : policy . GetObjectAction ,
BucketName : bucket ,
2019-10-16 11:59:59 -04:00
ConditionValues : getConditionValues ( r , "" , "" , nil ) ,
2018-10-09 17:00:01 -04:00
IsOwner : false ,
ObjectName : object ,
} ) {
writeWebErrorResponse ( w , errAuthentication )
return
}
2019-11-20 16:18:09 -05:00
if globalPolicySys . IsAllowed ( policy . Args {
Action : policy . GetObjectRetentionAction ,
BucketName : bucket ,
ConditionValues : getConditionValues ( r , "" , "" , nil ) ,
IsOwner : false ,
ObjectName : object ,
} ) {
getRetPerms = ErrNone
}
2020-01-16 18:41:56 -05:00
if globalPolicySys . IsAllowed ( policy . Args {
Action : policy . GetObjectLegalHoldAction ,
BucketName : bucket ,
ConditionValues : getConditionValues ( r , "" , "" , nil ) ,
IsOwner : false ,
ObjectName : object ,
} ) {
legalHoldPerms = ErrNone
}
2018-10-09 17:00:01 -04:00
} else {
writeWebErrorResponse ( w , authErr )
return
}
}
// For authenticated users apply IAM policy.
if authErr == nil {
if ! globalIAMSys . IsAllowed ( iampolicy . Args {
2020-01-30 21:59:22 -05:00
AccountName : claims . AccessKey ,
2018-12-13 23:15:09 -05:00
Action : iampolicy . GetObjectAction ,
2018-04-24 18:53:30 -04:00
BucketName : bucket ,
2020-01-30 21:59:22 -05:00
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
2018-10-09 17:00:01 -04:00
IsOwner : owner ,
2018-04-24 18:53:30 -04:00
ObjectName : object ,
2019-10-23 01:59:13 -04:00
Claims : claims . Map ( ) ,
2018-04-24 18:53:30 -04:00
} ) {
writeWebErrorResponse ( w , errAuthentication )
return
}
2019-11-20 16:18:09 -05:00
if globalIAMSys . IsAllowed ( iampolicy . Args {
2020-01-30 21:59:22 -05:00
AccountName : claims . AccessKey ,
2019-11-20 16:18:09 -05:00
Action : iampolicy . GetObjectRetentionAction ,
BucketName : bucket ,
2020-01-30 21:59:22 -05:00
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
2019-11-20 16:18:09 -05:00
IsOwner : owner ,
ObjectName : object ,
Claims : claims . Map ( ) ,
} ) {
getRetPerms = ErrNone
}
2020-01-16 18:41:56 -05:00
if globalIAMSys . IsAllowed ( iampolicy . Args {
2020-01-30 21:59:22 -05:00
AccountName : claims . AccessKey ,
2020-01-16 18:41:56 -05:00
Action : iampolicy . GetObjectLegalHoldAction ,
BucketName : bucket ,
2020-01-30 21:59:22 -05:00
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
2020-01-16 18:41:56 -05:00
IsOwner : owner ,
ObjectName : object ,
Claims : claims . Map ( ) ,
} ) {
legalHoldPerms = ErrNone
}
2016-03-31 09:57:29 -04:00
}
2018-09-27 23:36:17 -04:00
2019-04-04 02:10:37 -04:00
// Check if bucket is a reserved bucket name or invalid.
if isReservedOrInvalidBucket ( bucket , false ) {
writeWebErrorResponse ( w , errInvalidBucketName )
return
}
2018-12-13 23:15:09 -05:00
getObjectNInfo := objectAPI . GetObjectNInfo
2018-08-17 15:52:14 -04:00
if web . CacheAPI ( ) != nil {
2018-12-13 23:15:09 -05:00
getObjectNInfo = web . CacheAPI ( ) . GetObjectNInfo
2018-08-17 15:52:14 -04:00
}
2018-12-13 23:15:09 -05:00
var opts ObjectOptions
gr , err := getObjectNInfo ( ctx , bucket , object , nil , r . Header , readLock , opts )
2018-08-17 15:52:14 -04:00
if err != nil {
writeWebErrorResponse ( w , err )
return
}
2018-12-13 23:15:09 -05:00
defer gr . Close ( )
objInfo := gr . ObjInfo
2019-11-20 16:18:09 -05:00
// filter object lock metadata if permission does not permit
2020-01-16 18:41:56 -05:00
objInfo . UserDefined = objectlock . FilterObjectLockMetadata ( objInfo . UserDefined , getRetPerms != ErrNone , legalHoldPerms != ErrNone )
2019-11-20 16:18:09 -05:00
2018-08-17 15:52:14 -04:00
if objectAPI . IsEncryptionSupported ( ) {
2020-07-17 16:01:22 -04:00
if _ , err = DecryptObjectInfo ( & objInfo , r ) ; err != nil {
2018-08-24 10:56:24 -04:00
writeWebErrorResponse ( w , err )
2018-08-17 15:52:14 -04:00
return
}
2018-12-13 23:15:09 -05:00
}
// Set encryption response headers
if objectAPI . IsEncryptionSupported ( ) {
2021-02-03 18:19:08 -05:00
switch kind , _ := crypto . IsEncrypted ( objInfo . UserDefined ) ; kind {
case crypto . S3 :
w . Header ( ) . Set ( xhttp . AmzServerSideEncryption , xhttp . AmzEncryptionAES )
case crypto . SSEC :
w . Header ( ) . Set ( xhttp . AmzServerSideEncryptionCustomerAlgorithm , r . Header . Get ( xhttp . AmzServerSideEncryptionCustomerAlgorithm ) )
w . Header ( ) . Set ( xhttp . AmzServerSideEncryptionCustomerKeyMD5 , r . Header . Get ( xhttp . AmzServerSideEncryptionCustomerKeyMD5 ) )
2018-09-20 22:22:09 -04:00
}
2018-08-17 15:52:14 -04:00
}
2018-09-28 03:44:59 -04:00
2020-06-10 12:22:15 -04:00
// Set Parts Count Header
if opts . PartNumber > 0 && len ( objInfo . Parts ) > 0 {
setPartsCountHeaders ( w , objInfo )
}
2020-10-01 18:41:12 -04:00
if err = setObjectHeaders ( w , objInfo , nil , opts ) ; err != nil {
2018-12-13 23:15:09 -05:00
writeWebErrorResponse ( w , err )
return
2018-09-27 23:36:17 -04:00
}
2018-08-17 15:52:14 -04:00
2018-12-13 23:15:09 -05:00
// Add content disposition.
2019-07-03 01:34:32 -04:00
w . Header ( ) . Set ( xhttp . ContentDisposition , fmt . Sprintf ( "attachment; filename=\"%s\"" , path . Base ( objInfo . Name ) ) )
2018-09-27 23:36:17 -04:00
2018-12-13 23:15:09 -05:00
setHeadGetRespHeaders ( w , r . URL . Query ( ) )
2018-08-17 15:52:14 -04:00
2018-12-13 23:15:09 -05:00
httpWriter := ioutil . WriteOnClose ( w )
2016-03-31 09:57:29 -04:00
2018-12-13 23:15:09 -05:00
// Write object content to response body
if _ , err = io . Copy ( httpWriter , gr ) ; err != nil {
if ! httpWriter . HasWritten ( ) { // write error response only if no data or headers has been written to client yet
writeWebErrorResponse ( w , err )
2018-09-27 23:36:17 -04:00
}
fs: Break fs package to top-level and introduce ObjectAPI interface.
ObjectAPI interface brings in changes needed for XL ObjectAPI layer.
The new interface for any ObjectAPI layer is as below
```
// ObjectAPI interface.
type ObjectAPI interface {
// Bucket resource API.
DeleteBucket(bucket string) *probe.Error
ListBuckets() ([]BucketInfo, *probe.Error)
MakeBucket(bucket string) *probe.Error
GetBucketInfo(bucket string) (BucketInfo, *probe.Error)
// Bucket query API.
ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsResult, *probe.Error)
ListMultipartUploads(bucket string, resources BucketMultipartResourcesMetadata) (BucketMultipartResourcesMetadata, *probe.Error)
// Object resource API.
GetObject(bucket, object string, startOffset int64) (io.ReadCloser, *probe.Error)
GetObjectInfo(bucket, object string) (ObjectInfo, *probe.Error)
PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (ObjectInfo, *probe.Error)
DeleteObject(bucket, object string) *probe.Error
// Object query API.
NewMultipartUpload(bucket, object string) (string, *probe.Error)
PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, *probe.Error)
ListObjectParts(bucket, object string, resources ObjectResourcesMetadata) (ObjectResourcesMetadata, *probe.Error)
CompleteMultipartUpload(bucket string, object string, uploadID string, parts []CompletePart) (ObjectInfo, *probe.Error)
AbortMultipartUpload(bucket, object, uploadID string) *probe.Error
}
```
2016-03-30 19:15:28 -04:00
return
2016-03-31 09:57:29 -04:00
}
2018-12-13 23:15:09 -05:00
2018-08-17 15:52:14 -04:00
if err = httpWriter . Close ( ) ; err != nil {
2018-12-13 23:15:09 -05:00
if ! httpWriter . HasWritten ( ) { // write error response only if no data or headers has been written to client yet
2018-08-17 15:52:14 -04:00
writeWebErrorResponse ( w , err )
return
}
}
2018-10-05 14:20:00 -04:00
// Notify object accessed via a GET request.
sendEvent ( eventArgs {
EventName : event . ObjectAccessedGet ,
BucketName : bucket ,
Object : objInfo ,
ReqParams : extractReqParams ( r ) ,
RespElements : extractRespElements ( w ) ,
UserAgent : r . UserAgent ( ) ,
2019-03-25 14:45:42 -04:00
Host : handlers . GetSourceIP ( r ) ,
2018-10-05 14:20:00 -04:00
} )
2016-03-31 09:57:29 -04:00
}
2017-02-09 02:39:08 -05:00
// DownloadZipArgs - Argument for downloading a bunch of files as a zip file.
// JSON will look like:
// '{"bucketname":"testbucket","prefix":"john/pics/","objects":["hawaii/","maldives/","sanjose.jpg"]}'
type DownloadZipArgs struct {
Objects [ ] string ` json:"objects" ` // can be files or sub-directories
Prefix string ` json:"prefix" ` // current directory in the browser-ui
BucketName string ` json:"bucketname" ` // bucket name.
}
// Takes a list of objects and creates a zip file that sent as the response body.
func ( web * webAPIHandlers ) DownloadZip ( w http . ResponseWriter , r * http . Request ) {
2019-03-25 14:45:42 -04:00
host := handlers . GetSourceIP ( r )
2018-10-05 14:20:00 -04:00
2020-05-27 15:38:44 -04:00
claims , owner , authErr := webTokenAuthenticate ( r . URL . Query ( ) . Get ( "token" ) )
2018-10-12 15:25:59 -04:00
ctx := newContext ( r , w , "WebDownloadZip" )
2021-01-26 16:21:51 -05:00
defer logger . AuditLog ( ctx , w , r , claims . Map ( ) )
2018-10-12 15:25:59 -04:00
2017-02-09 02:39:08 -05:00
objectAPI := web . ObjectAPI ( )
if objectAPI == nil {
writeWebErrorResponse ( w , errServerNotInitialized )
return
}
2018-10-09 17:00:01 -04:00
2017-05-10 12:54:24 -04:00
// Auth is done after reading the body to accommodate for anonymous requests
// when bucket policy is enabled.
2017-02-09 02:39:08 -05:00
var args DownloadZipArgs
2017-05-10 12:54:24 -04:00
tenKB := 10 * 1024 // To limit r.Body to take care of misbehaving anonymous client.
decodeErr := json . NewDecoder ( io . LimitReader ( r . Body , int64 ( tenKB ) ) ) . Decode ( & args )
2017-02-09 02:39:08 -05:00
if decodeErr != nil {
writeWebErrorResponse ( w , decodeErr )
return
}
2020-05-27 15:38:44 -04:00
2019-11-20 16:18:09 -05:00
var getRetPerms [ ] APIErrorCode
2020-01-16 18:41:56 -05:00
var legalHoldPerms [ ] APIErrorCode
2018-10-09 17:00:01 -04:00
if authErr != nil {
if authErr == errNoAuthToken {
for _ , object := range args . Objects {
// Check if anonymous (non-owner) has access to download objects.
if ! globalPolicySys . IsAllowed ( policy . Args {
Action : policy . GetObjectAction ,
BucketName : args . BucketName ,
2019-10-16 11:59:59 -04:00
ConditionValues : getConditionValues ( r , "" , "" , nil ) ,
2018-10-09 17:00:01 -04:00
IsOwner : false ,
ObjectName : pathJoin ( args . Prefix , object ) ,
} ) {
writeWebErrorResponse ( w , errAuthentication )
return
}
2019-11-20 16:18:09 -05:00
retentionPerm := ErrAccessDenied
if globalPolicySys . IsAllowed ( policy . Args {
Action : policy . GetObjectRetentionAction ,
BucketName : args . BucketName ,
ConditionValues : getConditionValues ( r , "" , "" , nil ) ,
IsOwner : false ,
ObjectName : pathJoin ( args . Prefix , object ) ,
} ) {
retentionPerm = ErrNone
}
getRetPerms = append ( getRetPerms , retentionPerm )
2020-01-16 18:41:56 -05:00
legalHoldPerm := ErrAccessDenied
if globalPolicySys . IsAllowed ( policy . Args {
Action : policy . GetObjectLegalHoldAction ,
BucketName : args . BucketName ,
ConditionValues : getConditionValues ( r , "" , "" , nil ) ,
IsOwner : false ,
ObjectName : pathJoin ( args . Prefix , object ) ,
} ) {
legalHoldPerm = ErrNone
}
legalHoldPerms = append ( legalHoldPerms , legalHoldPerm )
2018-10-09 17:00:01 -04:00
}
} else {
writeWebErrorResponse ( w , authErr )
return
}
}
// For authenticated users apply IAM policy.
if authErr == nil {
2017-05-10 12:54:24 -04:00
for _ , object := range args . Objects {
2018-10-09 17:00:01 -04:00
if ! globalIAMSys . IsAllowed ( iampolicy . Args {
2020-01-30 21:59:22 -05:00
AccountName : claims . AccessKey ,
2018-12-13 23:15:09 -05:00
Action : iampolicy . GetObjectAction ,
2018-04-24 18:53:30 -04:00
BucketName : args . BucketName ,
2020-01-30 21:59:22 -05:00
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
2018-10-09 17:00:01 -04:00
IsOwner : owner ,
2018-04-24 18:53:30 -04:00
ObjectName : pathJoin ( args . Prefix , object ) ,
2019-10-23 01:59:13 -04:00
Claims : claims . Map ( ) ,
2018-04-24 18:53:30 -04:00
} ) {
2017-05-10 12:54:24 -04:00
writeWebErrorResponse ( w , errAuthentication )
return
}
2019-11-20 16:18:09 -05:00
retentionPerm := ErrAccessDenied
if globalIAMSys . IsAllowed ( iampolicy . Args {
2020-01-30 21:59:22 -05:00
AccountName : claims . AccessKey ,
2020-01-16 18:41:56 -05:00
Action : iampolicy . GetObjectRetentionAction ,
2019-11-20 16:18:09 -05:00
BucketName : args . BucketName ,
2020-01-30 21:59:22 -05:00
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
2019-11-20 16:18:09 -05:00
IsOwner : owner ,
ObjectName : pathJoin ( args . Prefix , object ) ,
Claims : claims . Map ( ) ,
} ) {
retentionPerm = ErrNone
}
getRetPerms = append ( getRetPerms , retentionPerm )
2020-01-16 18:41:56 -05:00
legalHoldPerm := ErrAccessDenied
if globalIAMSys . IsAllowed ( iampolicy . Args {
2020-01-30 21:59:22 -05:00
AccountName : claims . AccessKey ,
2020-01-16 18:41:56 -05:00
Action : iampolicy . GetObjectLegalHoldAction ,
BucketName : args . BucketName ,
2020-01-30 21:59:22 -05:00
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
2020-01-16 18:41:56 -05:00
IsOwner : owner ,
ObjectName : pathJoin ( args . Prefix , object ) ,
Claims : claims . Map ( ) ,
} ) {
legalHoldPerm = ErrNone
}
legalHoldPerms = append ( legalHoldPerms , legalHoldPerm )
2017-05-10 12:54:24 -04:00
}
}
2019-04-04 02:10:37 -04:00
// Check if bucket is a reserved bucket name or invalid.
if isReservedOrInvalidBucket ( args . BucketName , false ) {
writeWebErrorResponse ( w , errInvalidBucketName )
return
}
2020-04-06 16:44:16 -04:00
2019-08-09 20:09:08 -04:00
getObjectNInfo := objectAPI . GetObjectNInfo
2018-10-09 17:00:01 -04:00
if web . CacheAPI ( ) != nil {
2019-08-09 20:09:08 -04:00
getObjectNInfo = web . CacheAPI ( ) . GetObjectNInfo
2018-10-09 17:00:01 -04:00
}
2019-08-09 20:09:08 -04:00
2017-02-09 02:39:08 -05:00
archive := zip . NewWriter ( w )
defer archive . Close ( )
2018-10-09 17:00:01 -04:00
2019-11-20 16:18:09 -05:00
for i , object := range args . Objects {
2017-02-09 02:39:08 -05:00
// Writes compressed object file to the response.
zipit := func ( objectName string ) error {
2019-08-09 20:09:08 -04:00
var opts ObjectOptions
gr , err := getObjectNInfo ( ctx , args . BucketName , objectName , nil , r . Header , readLock , opts )
2017-02-09 02:39:08 -05:00
if err != nil {
return err
}
2019-08-09 20:09:08 -04:00
defer gr . Close ( )
info := gr . ObjInfo
2019-11-20 16:18:09 -05:00
// filter object lock metadata if permission does not permit
2020-01-16 18:41:56 -05:00
info . UserDefined = objectlock . FilterObjectLockMetadata ( info . UserDefined , getRetPerms [ i ] != ErrNone , legalHoldPerms [ i ] != ErrNone )
2020-05-24 14:19:17 -04:00
// For reporting, set the file size to the uncompressed size.
info . Size , err = info . GetActualSize ( )
if err != nil {
return err
2018-09-27 23:36:17 -04:00
}
2017-02-09 02:39:08 -05:00
header := & zip . FileHeader {
2020-01-06 15:43:00 -05:00
Name : strings . TrimPrefix ( objectName , args . Prefix ) ,
Method : zip . Deflate ,
Flags : 1 << 11 ,
Modified : info . ModTime ,
2019-09-26 02:08:24 -04:00
}
if hasStringSuffixInSlice ( info . Name , standardExcludeCompressExtensions ) || hasPattern ( standardExcludeCompressContentTypes , info . ContentType ) {
// We strictly disable compression for standard extensions/content-types.
header . Method = zip . Store
2017-02-09 02:39:08 -05:00
}
2019-09-26 02:08:24 -04:00
writer , err := archive . CreateHeader ( header )
2017-02-09 02:39:08 -05:00
if err != nil {
writeWebErrorResponse ( w , errUnexpected )
return err
}
2018-08-17 15:52:14 -04:00
httpWriter := ioutil . WriteOnClose ( writer )
2019-08-09 20:09:08 -04:00
// Write object content to response body
if _ , err = io . Copy ( httpWriter , gr ) ; err != nil {
2018-09-27 23:36:17 -04:00
httpWriter . Close ( )
2019-08-09 20:09:08 -04:00
if ! httpWriter . HasWritten ( ) { // write error response only if no data or headers has been written to client yet
writeWebErrorResponse ( w , err )
}
2018-08-17 15:52:14 -04:00
return err
}
2019-08-09 20:09:08 -04:00
2018-08-17 15:52:14 -04:00
if err = httpWriter . Close ( ) ; err != nil {
if ! httpWriter . HasWritten ( ) { // write error response only if no data has been written to client yet
writeWebErrorResponse ( w , err )
return err
}
}
2018-10-05 14:20:00 -04:00
// Notify object accessed via a GET request.
sendEvent ( eventArgs {
EventName : event . ObjectAccessedGet ,
BucketName : args . BucketName ,
Object : info ,
ReqParams : extractReqParams ( r ) ,
RespElements : extractRespElements ( w ) ,
UserAgent : r . UserAgent ( ) ,
Host : host ,
} )
2018-08-17 15:52:14 -04:00
return nil
2017-02-09 02:39:08 -05:00
}
2019-12-06 02:16:06 -05:00
if ! HasSuffix ( object , SlashSeparator ) {
2017-02-09 02:39:08 -05:00
// If not a directory, compress the file and write it to response.
err := zipit ( pathJoin ( args . Prefix , object ) )
if err != nil {
2020-04-06 16:44:16 -04:00
logger . LogIf ( ctx , err )
2017-02-09 02:39:08 -05:00
return
}
continue
}
2020-04-06 16:44:16 -04:00
objInfoCh := make ( chan ObjectInfo )
// Walk through all objects
2020-07-11 01:21:04 -04:00
if err := objectAPI . Walk ( ctx , args . BucketName , pathJoin ( args . Prefix , object ) , objInfoCh , ObjectOptions { } ) ; err != nil {
2020-04-06 16:44:16 -04:00
logger . LogIf ( ctx , err )
continue
}
for obj := range objInfoCh {
if err := zipit ( obj . Name ) ; err != nil {
logger . LogIf ( ctx , err )
continue
2017-02-09 02:39:08 -05:00
}
}
}
}
2016-08-30 13:04:50 -04:00
// GetBucketPolicyArgs - get bucket policy args.
type GetBucketPolicyArgs struct {
BucketName string ` json:"bucketName" `
Prefix string ` json:"prefix" `
}
// GetBucketPolicyRep - get bucket policy reply.
type GetBucketPolicyRep struct {
2018-04-24 18:53:30 -04:00
UIVersion string ` json:"uiVersion" `
Policy miniogopolicy . BucketPolicy ` json:"policy" `
2016-08-30 13:04:50 -04:00
}
2017-06-19 22:45:13 -04:00
// GetBucketPolicy - get bucket policy for the requested prefix.
2016-08-30 13:04:50 -04:00
func ( web * webAPIHandlers ) GetBucketPolicy ( r * http . Request , args * GetBucketPolicyArgs , reply * GetBucketPolicyRep ) error {
2019-10-23 01:59:13 -04:00
ctx := newWebContext ( r , args , "WebGetBucketPolicy" )
2020-05-27 15:38:44 -04:00
2016-08-30 22:22:27 -04:00
objectAPI := web . ObjectAPI ( )
if objectAPI == nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , errServerNotInitialized )
2016-08-30 22:22:27 -04:00
}
2016-11-02 17:45:11 -04:00
2019-04-09 11:17:41 -04:00
claims , owner , authErr := webRequestAuthenticate ( r )
2018-10-17 19:23:09 -04:00
if authErr != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , authErr )
2018-10-17 19:23:09 -04:00
}
2019-07-15 15:00:41 -04:00
2019-04-09 11:17:41 -04:00
// For authenticated users apply IAM policy.
if ! globalIAMSys . IsAllowed ( iampolicy . Args {
2020-01-30 21:59:22 -05:00
AccountName : claims . AccessKey ,
2019-04-09 11:17:41 -04:00
Action : iampolicy . GetBucketPolicyAction ,
BucketName : args . BucketName ,
2020-01-30 21:59:22 -05:00
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
2019-04-09 11:17:41 -04:00
IsOwner : owner ,
2019-10-23 01:59:13 -04:00
Claims : claims . Map ( ) ,
2019-04-09 11:17:41 -04:00
} ) {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , errAccessDenied )
2016-11-02 17:45:11 -04:00
}
2019-04-04 02:10:37 -04:00
// Check if bucket is a reserved bucket name or invalid.
if isReservedOrInvalidBucket ( args . BucketName , false ) {
2020-07-10 15:10:39 -04:00
return toJSONError ( ctx , errInvalidBucketName , args . BucketName )
2019-04-04 02:10:37 -04:00
}
2018-12-19 08:13:47 -05:00
var policyInfo = & miniogopolicy . BucketAccessPolicy { Version : "2012-10-17" }
2019-06-03 18:40:04 -04:00
if isRemoteCallRequired ( ctx , args . BucketName , objectAPI ) {
2018-12-19 08:13:47 -05:00
sr , err := globalDNSConfig . Get ( args . BucketName )
if err != nil {
if err == dns . ErrNoEntriesFound {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , BucketNotFound {
2018-12-19 08:13:47 -05:00
Bucket : args . BucketName ,
} , args . BucketName )
}
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2017-06-19 22:45:13 -04:00
}
2018-12-19 08:13:47 -05:00
client , rerr := getRemoteInstanceClient ( r , getHostFromSrv ( sr ) )
if rerr != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , rerr , args . BucketName )
2018-12-19 08:13:47 -05:00
}
2020-07-14 12:38:05 -04:00
policyStr , err := client . GetBucketPolicy ( ctx , args . BucketName )
2018-12-19 08:13:47 -05:00
if err != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , rerr , args . BucketName )
2018-12-19 08:13:47 -05:00
}
bucketPolicy , err := policy . ParseConfig ( strings . NewReader ( policyStr ) , args . BucketName )
if err != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , rerr , args . BucketName )
2018-12-19 08:13:47 -05:00
}
policyInfo , err = PolicyToBucketAccessPolicy ( bucketPolicy )
if err != nil {
// This should not happen.
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
} else {
2020-05-19 16:53:54 -04:00
bucketPolicy , err := globalPolicySys . Get ( args . BucketName )
2018-12-19 08:13:47 -05:00
if err != nil {
if _ , ok := err . ( BucketPolicyNotFound ) ; ! ok {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
}
2016-08-30 13:04:50 -04:00
2018-12-19 08:13:47 -05:00
policyInfo , err = PolicyToBucketAccessPolicy ( bucketPolicy )
if err != nil {
// This should not happen.
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
2018-04-24 18:53:30 -04:00
}
2017-02-22 20:27:26 -05:00
reply . UIVersion = browser . UIVersion
2018-04-24 18:53:30 -04:00
reply . Policy = miniogopolicy . GetPolicy ( policyInfo . Statements , args . BucketName , args . Prefix )
2016-08-30 13:04:50 -04:00
return nil
}
2016-09-26 00:53:19 -04:00
// ListAllBucketPoliciesArgs - get all bucket policies.
type ListAllBucketPoliciesArgs struct {
2016-09-23 02:06:45 -04:00
BucketName string ` json:"bucketName" `
}
2017-03-16 15:21:58 -04:00
// BucketAccessPolicy - Collection of canned bucket policy at a given prefix.
type BucketAccessPolicy struct {
2018-04-24 18:53:30 -04:00
Bucket string ` json:"bucket" `
Prefix string ` json:"prefix" `
Policy miniogopolicy . BucketPolicy ` json:"policy" `
2016-09-26 00:53:19 -04:00
}
// ListAllBucketPoliciesRep - get all bucket policy reply.
type ListAllBucketPoliciesRep struct {
UIVersion string ` json:"uiVersion" `
2017-03-16 15:21:58 -04:00
Policies [ ] BucketAccessPolicy ` json:"policies" `
2016-09-23 02:06:45 -04:00
}
2018-04-24 18:53:30 -04:00
// ListAllBucketPolicies - get all bucket policy.
2016-09-26 00:53:19 -04:00
func ( web * webAPIHandlers ) ListAllBucketPolicies ( r * http . Request , args * ListAllBucketPoliciesArgs , reply * ListAllBucketPoliciesRep ) error {
2019-07-11 17:37:13 -04:00
ctx := newWebContext ( r , args , "WebListAllBucketPolicies" )
2016-09-23 02:06:45 -04:00
objectAPI := web . ObjectAPI ( )
if objectAPI == nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , errServerNotInitialized )
2016-11-02 17:45:11 -04:00
}
2019-07-15 15:00:41 -04:00
claims , owner , authErr := webRequestAuthenticate ( r )
2018-10-17 19:23:09 -04:00
if authErr != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , authErr )
2018-10-17 19:23:09 -04:00
}
2019-04-09 11:17:41 -04:00
2019-07-15 15:00:41 -04:00
// For authenticated users apply IAM policy.
if ! globalIAMSys . IsAllowed ( iampolicy . Args {
2020-01-30 21:59:22 -05:00
AccountName : claims . AccessKey ,
2019-07-15 15:00:41 -04:00
Action : iampolicy . GetBucketPolicyAction ,
BucketName : args . BucketName ,
2020-01-30 21:59:22 -05:00
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
2019-07-15 15:00:41 -04:00
IsOwner : owner ,
2019-10-23 01:59:13 -04:00
Claims : claims . Map ( ) ,
2019-07-15 15:00:41 -04:00
} ) {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , errAccessDenied )
2016-09-23 02:06:45 -04:00
}
2018-04-24 18:53:30 -04:00
2019-04-04 02:10:37 -04:00
// Check if bucket is a reserved bucket name or invalid.
if isReservedOrInvalidBucket ( args . BucketName , false ) {
2020-07-10 15:10:39 -04:00
return toJSONError ( ctx , errInvalidBucketName , args . BucketName )
2019-04-04 02:10:37 -04:00
}
2018-12-19 08:13:47 -05:00
var policyInfo = new ( miniogopolicy . BucketAccessPolicy )
2019-06-03 18:40:04 -04:00
if isRemoteCallRequired ( ctx , args . BucketName , objectAPI ) {
2018-12-19 08:13:47 -05:00
sr , err := globalDNSConfig . Get ( args . BucketName )
if err != nil {
if err == dns . ErrNoEntriesFound {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , BucketNotFound {
2018-12-19 08:13:47 -05:00
Bucket : args . BucketName ,
} , args . BucketName )
}
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
core , rerr := getRemoteInstanceClient ( r , getHostFromSrv ( sr ) )
if rerr != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , rerr , args . BucketName )
2018-12-19 08:13:47 -05:00
}
var policyStr string
2020-07-14 12:38:05 -04:00
policyStr , err = core . Client . GetBucketPolicy ( ctx , args . BucketName )
2018-12-19 08:13:47 -05:00
if err != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
if policyStr != "" {
if err = json . Unmarshal ( [ ] byte ( policyStr ) , policyInfo ) ; err != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
}
} else {
2020-05-19 16:53:54 -04:00
bucketPolicy , err := globalPolicySys . Get ( args . BucketName )
2018-12-19 08:13:47 -05:00
if err != nil {
if _ , ok := err . ( BucketPolicyNotFound ) ; ! ok {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
}
policyInfo , err = PolicyToBucketAccessPolicy ( bucketPolicy )
if err != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2017-06-01 12:43:20 -04:00
}
2018-04-24 18:53:30 -04:00
}
2017-02-22 20:27:26 -05:00
reply . UIVersion = browser . UIVersion
2018-04-24 18:53:30 -04:00
for prefix , policy := range miniogopolicy . GetPolicies ( policyInfo . Statements , args . BucketName , "" ) {
2020-01-21 17:07:49 -05:00
bucketName , objectPrefix := path2BucketObject ( prefix )
2018-02-27 22:14:49 -05:00
objectPrefix = strings . TrimSuffix ( objectPrefix , "*" )
2017-03-16 15:21:58 -04:00
reply . Policies = append ( reply . Policies , BucketAccessPolicy {
2018-02-27 22:14:49 -05:00
Bucket : bucketName ,
Prefix : objectPrefix ,
2016-09-26 00:53:19 -04:00
Policy : policy ,
} )
}
2018-04-24 18:53:30 -04:00
2016-09-23 02:06:45 -04:00
return nil
}
2018-04-24 18:53:30 -04:00
// SetBucketPolicyWebArgs - set bucket policy args.
type SetBucketPolicyWebArgs struct {
2016-08-30 13:04:50 -04:00
BucketName string ` json:"bucketName" `
Prefix string ` json:"prefix" `
Policy string ` json:"policy" `
}
// SetBucketPolicy - set bucket policy.
2018-04-24 18:53:30 -04:00
func ( web * webAPIHandlers ) SetBucketPolicy ( r * http . Request , args * SetBucketPolicyWebArgs , reply * WebGenericRep ) error {
2019-10-23 01:59:13 -04:00
ctx := newWebContext ( r , args , "WebSetBucketPolicy" )
2016-09-22 19:35:12 -04:00
objectAPI := web . ObjectAPI ( )
2017-06-05 11:11:54 -04:00
reply . UIVersion = browser . UIVersion
2016-09-22 19:35:12 -04:00
if objectAPI == nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , errServerNotInitialized )
2016-11-02 17:45:11 -04:00
}
2019-04-09 11:17:41 -04:00
claims , owner , authErr := webRequestAuthenticate ( r )
2018-10-17 19:23:09 -04:00
if authErr != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , authErr )
2018-10-17 19:23:09 -04:00
}
2019-04-09 11:17:41 -04:00
// For authenticated users apply IAM policy.
if ! globalIAMSys . IsAllowed ( iampolicy . Args {
2020-01-30 21:59:22 -05:00
AccountName : claims . AccessKey ,
2019-04-09 11:17:41 -04:00
Action : iampolicy . PutBucketPolicyAction ,
BucketName : args . BucketName ,
2020-01-30 21:59:22 -05:00
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
2019-04-09 11:17:41 -04:00
IsOwner : owner ,
2019-10-23 01:59:13 -04:00
Claims : claims . Map ( ) ,
2019-04-09 11:17:41 -04:00
} ) {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , errAccessDenied )
2016-09-22 19:35:12 -04:00
}
2016-08-30 13:04:50 -04:00
2019-04-04 02:10:37 -04:00
// Check if bucket is a reserved bucket name or invalid.
if isReservedOrInvalidBucket ( args . BucketName , false ) {
2020-07-10 15:10:39 -04:00
return toJSONError ( ctx , errInvalidBucketName , args . BucketName )
2019-04-04 02:10:37 -04:00
}
2018-04-24 18:53:30 -04:00
policyType := miniogopolicy . BucketPolicy ( args . Policy )
if ! policyType . IsValidBucketPolicy ( ) {
2016-11-22 14:12:38 -05:00
return & json2 . Error {
Message : "Invalid policy type " + args . Policy ,
}
2016-08-30 13:04:50 -04:00
}
2019-06-03 18:40:04 -04:00
if isRemoteCallRequired ( ctx , args . BucketName , objectAPI ) {
2018-12-19 08:13:47 -05:00
sr , err := globalDNSConfig . Get ( args . BucketName )
if err != nil {
if err == dns . ErrNoEntriesFound {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , BucketNotFound {
2018-12-19 08:13:47 -05:00
Bucket : args . BucketName ,
} , args . BucketName )
}
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2017-06-01 12:43:20 -04:00
}
2018-12-19 08:13:47 -05:00
core , rerr := getRemoteInstanceClient ( r , getHostFromSrv ( sr ) )
if rerr != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , rerr , args . BucketName )
2018-12-19 08:13:47 -05:00
}
var policyStr string
// Use the abstracted API instead of core, such that
// NoSuchBucketPolicy errors are automatically handled.
2020-07-14 12:38:05 -04:00
policyStr , err = core . Client . GetBucketPolicy ( ctx , args . BucketName )
2018-12-19 08:13:47 -05:00
if err != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
var policyInfo = & miniogopolicy . BucketAccessPolicy { Version : "2012-10-17" }
if policyStr != "" {
if err = json . Unmarshal ( [ ] byte ( policyStr ) , policyInfo ) ; err != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
}
2017-06-01 12:43:20 -04:00
2018-12-19 08:13:47 -05:00
policyInfo . Statements = miniogopolicy . SetPolicy ( policyInfo . Statements , policyType , args . BucketName , args . Prefix )
if len ( policyInfo . Statements ) == 0 {
2020-07-14 12:38:05 -04:00
if err = core . SetBucketPolicy ( ctx , args . BucketName , "" ) ; err != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
return nil
}
2018-04-24 18:53:30 -04:00
2018-12-19 08:13:47 -05:00
bucketPolicy , err := BucketAccessPolicyToPolicy ( policyInfo )
if err != nil {
// This should not happen.
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
2017-06-01 12:43:20 -04:00
2018-12-19 08:13:47 -05:00
policyData , err := json . Marshal ( bucketPolicy )
if err != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2016-09-26 06:11:22 -04:00
}
2018-04-24 18:53:30 -04:00
2020-07-14 12:38:05 -04:00
if err = core . SetBucketPolicy ( ctx , args . BucketName , string ( policyData ) ) ; err != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
2017-10-27 19:14:06 -04:00
2018-12-19 08:13:47 -05:00
} else {
2020-05-19 16:53:54 -04:00
bucketPolicy , err := globalPolicySys . Get ( args . BucketName )
2018-12-19 08:13:47 -05:00
if err != nil {
if _ , ok := err . ( BucketPolicyNotFound ) ; ! ok {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
}
policyInfo , err := PolicyToBucketAccessPolicy ( bucketPolicy )
if err != nil {
// This should not happen.
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
2018-02-27 22:14:49 -05:00
2018-12-19 08:13:47 -05:00
policyInfo . Statements = miniogopolicy . SetPolicy ( policyInfo . Statements , policyType , args . BucketName , args . Prefix )
if len ( policyInfo . Statements ) == 0 {
2020-05-19 16:53:54 -04:00
if err = globalBucketMetadataSys . Update ( args . BucketName , bucketPolicyConfig , nil ) ; err != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
return nil
}
bucketPolicy , err = BucketAccessPolicyToPolicy ( policyInfo )
if err != nil {
// This should not happen.
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
2018-02-09 18:19:30 -05:00
2020-05-19 16:53:54 -04:00
configData , err := json . Marshal ( bucketPolicy )
if err != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , err , args . BucketName )
2018-12-19 08:13:47 -05:00
}
2020-05-19 16:53:54 -04:00
// Parse validate and save bucket policy.
if err = globalBucketMetadataSys . Update ( args . BucketName , bucketPolicyConfig , configData ) ; err != nil {
return toJSONError ( ctx , err , args . BucketName )
}
2018-12-19 08:13:47 -05:00
}
2018-04-24 18:53:30 -04:00
2016-08-30 13:04:50 -04:00
return nil
}
2016-09-23 04:24:49 -04:00
// PresignedGetArgs - presigned-get API args.
type PresignedGetArgs struct {
// Host header required for signed headers.
HostName string ` json:"host" `
// Bucket name of the object to be presigned.
BucketName string ` json:"bucket" `
// Object name to be presigned.
ObjectName string ` json:"object" `
2016-11-22 14:12:38 -05:00
// Expiry in seconds.
Expiry int64 ` json:"expiry" `
2016-09-23 04:24:49 -04:00
}
// PresignedGetRep - presigned-get URL reply.
type PresignedGetRep struct {
2016-10-23 15:32:35 -04:00
UIVersion string ` json:"uiVersion" `
2016-09-23 04:24:49 -04:00
// Presigned URL of the object.
URL string ` json:"url" `
}
// PresignedGET - returns presigned-Get url.
func ( web * webAPIHandlers ) PresignedGet ( r * http . Request , args * PresignedGetArgs , reply * PresignedGetRep ) error {
2019-10-23 01:59:13 -04:00
ctx := newWebContext ( r , args , "WebPresignedGet" )
2018-10-17 19:23:09 -04:00
claims , owner , authErr := webRequestAuthenticate ( r )
if authErr != nil {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , authErr )
2018-10-17 19:23:09 -04:00
}
var creds auth . Credentials
if ! owner {
var ok bool
2020-01-30 21:59:22 -05:00
creds , ok = globalIAMSys . GetUser ( claims . AccessKey )
2018-10-17 19:23:09 -04:00
if ! ok {
2019-06-03 18:40:04 -04:00
return toJSONError ( ctx , errInvalidAccessKeyID )
2018-10-17 19:23:09 -04:00
}
} else {
2019-10-23 01:59:13 -04:00
creds = globalActiveCred
2016-09-23 04:24:49 -04:00
}
2016-11-22 14:12:38 -05:00
2019-10-23 01:59:13 -04:00
region := globalServerRegion
2016-09-23 04:24:49 -04:00
if args . BucketName == "" || args . ObjectName == "" {
2016-11-22 14:12:38 -05:00
return & json2 . Error {
Message : "Bucket and Object are mandatory arguments." ,
}
2016-09-23 04:24:49 -04:00
}
2018-10-17 19:23:09 -04:00
2019-04-04 02:10:37 -04:00
// Check if bucket is a reserved bucket name or invalid.
if isReservedOrInvalidBucket ( args . BucketName , false ) {
2020-07-10 15:10:39 -04:00
return toJSONError ( ctx , errInvalidBucketName , args . BucketName )
2019-04-04 02:10:37 -04:00
}
2020-05-18 02:38:52 -04:00
// Check if the user indeed has GetObject access,
// if not we do not need to generate presigned URLs
if ! globalIAMSys . IsAllowed ( iampolicy . Args {
AccountName : claims . AccessKey ,
Action : iampolicy . GetObjectAction ,
BucketName : args . BucketName ,
ConditionValues : getConditionValues ( r , "" , claims . AccessKey , claims . Map ( ) ) ,
IsOwner : owner ,
ObjectName : args . ObjectName ,
Claims : claims . Map ( ) ,
} ) {
return toJSONError ( ctx , errPresignedNotAllowed )
}
2017-02-22 20:27:26 -05:00
reply . UIVersion = browser . UIVersion
2018-10-17 19:23:09 -04:00
reply . URL = presignedGet ( args . HostName , args . BucketName , args . ObjectName , args . Expiry , creds , region )
2016-09-23 04:24:49 -04:00
return nil
}
// Returns presigned url for GET method.
2018-10-17 19:23:09 -04:00
func presignedGet ( host , bucket , object string , expiry int64 , creds auth . Credentials , region string ) string {
accessKey := creds . AccessKey
secretKey := creds . SecretKey
2020-07-27 12:02:53 -04:00
sessionToken := creds . SessionToken
2016-09-23 04:24:49 -04:00
2017-03-18 14:28:41 -04:00
date := UTCNow ( )
2016-11-11 00:57:15 -05:00
dateStr := date . Format ( iso8601Format )
2016-09-23 04:24:49 -04:00
credential := fmt . Sprintf ( "%s/%s" , accessKey , getScope ( date , region ) )
2016-11-22 14:12:38 -05:00
var expiryStr = "604800" // Default set to be expire in 7days.
if expiry < 604800 && expiry > 0 {
expiryStr = strconv . FormatInt ( expiry , 10 )
}
2018-06-05 13:48:51 -04:00
query := url . Values { }
2019-07-03 01:34:32 -04:00
query . Set ( xhttp . AmzAlgorithm , signV4Algorithm )
query . Set ( xhttp . AmzCredential , credential )
query . Set ( xhttp . AmzDate , dateStr )
query . Set ( xhttp . AmzExpires , expiryStr )
2020-07-27 12:02:53 -04:00
// Set session token if available.
if sessionToken != "" {
query . Set ( xhttp . AmzSecurityToken , sessionToken )
}
2019-07-03 01:34:32 -04:00
query . Set ( xhttp . AmzSignedHeaders , "host" )
2018-06-05 13:48:51 -04:00
queryStr := s3utils . QueryEncode ( query )
2016-09-23 04:24:49 -04:00
2019-08-06 15:08:58 -04:00
path := SlashSeparator + path . Join ( bucket , object )
2016-09-23 04:24:49 -04:00
2017-04-05 20:00:24 -04:00
// "host" is the only header required to be signed for Presigned URLs.
extractedSignedHeaders := make ( http . Header )
extractedSignedHeaders . Set ( "host" , host )
2019-10-23 01:59:13 -04:00
canonicalRequest := getCanonicalRequest ( extractedSignedHeaders , unsignedPayload , queryStr , path , http . MethodGet )
2017-02-06 16:09:09 -05:00
stringToSign := getStringToSign ( canonicalRequest , date , getScope ( date , region ) )
2019-02-27 20:46:55 -05:00
signingKey := getSigningKey ( secretKey , date , region , serviceS3 )
2016-09-23 04:24:49 -04:00
signature := getSignature ( signingKey , stringToSign )
2019-07-03 01:34:32 -04:00
return host + s3utils . EncodePath ( path ) + "?" + queryStr + "&" + xhttp . AmzSignature + "=" + signature
2016-09-23 04:24:49 -04:00
}
2016-11-22 14:12:38 -05:00
2019-10-23 01:59:13 -04:00
// DiscoveryDocResp - OpenID discovery document reply.
type DiscoveryDocResp struct {
DiscoveryDoc openid . DiscoveryDoc
UIVersion string ` json:"uiVersion" `
2019-11-30 00:37:42 -05:00
ClientID string ` json:"clientId" `
2019-10-23 01:59:13 -04:00
}
// GetDiscoveryDoc - returns parsed value of OpenID discovery document
func ( web * webAPIHandlers ) GetDiscoveryDoc ( r * http . Request , args * WebGenericArgs , reply * DiscoveryDocResp ) error {
if globalOpenIDConfig . DiscoveryDoc . AuthEndpoint != "" {
reply . DiscoveryDoc = globalOpenIDConfig . DiscoveryDoc
2019-11-30 00:37:42 -05:00
reply . ClientID = globalOpenIDConfig . ClientID
2019-10-23 01:59:13 -04:00
}
reply . UIVersion = browser . UIVersion
return nil
}
// LoginSTSArgs - login arguments.
type LoginSTSArgs struct {
Token string ` json:"token" form:"token" `
}
2021-01-25 13:15:03 -05:00
var errSTSNotInitialized = errors . New ( "STS API not initialized, please configure STS support" )
2019-10-23 01:59:13 -04:00
// LoginSTS - STS user login handler.
func ( web * webAPIHandlers ) LoginSTS ( r * http . Request , args * LoginSTSArgs , reply * LoginRep ) error {
ctx := newWebContext ( r , args , "WebLoginSTS" )
2021-01-25 13:15:03 -05:00
if globalOpenIDValidators == nil {
return toJSONError ( ctx , errSTSNotInitialized )
2019-10-23 01:59:13 -04:00
}
2021-01-25 13:15:03 -05:00
v , err := globalOpenIDValidators . Get ( "jwt" )
if err != nil {
logger . LogIf ( ctx , err )
return toJSONError ( ctx , errSTSNotInitialized )
2019-10-23 01:59:13 -04:00
}
2021-01-25 13:15:03 -05:00
m , err := v . Validate ( args . Token , "" )
2019-10-23 01:59:13 -04:00
if err != nil {
return toJSONError ( ctx , err )
}
2021-01-25 13:15:03 -05:00
// JWT has requested a custom claim with policy value set.
// This is a MinIO STS API specific value, this value should
// be set and configured on your identity provider as part of
// JWT custom claims.
var policyName string
policySet , ok := iampolicy . GetPoliciesFromClaims ( m , iamPolicyClaimNameOpenID ( ) )
if ok {
policyName = globalIAMSys . CurrentPolicies ( strings . Join ( policySet . ToSlice ( ) , "," ) )
2019-10-23 01:59:13 -04:00
}
2021-01-25 13:15:03 -05:00
if policyName == "" && globalPolicyOPA == nil {
return toJSONError ( ctx , fmt . Errorf ( "%s claim missing from the JWT token, credentials will not be generated" , iamPolicyClaimNameOpenID ( ) ) )
}
m [ iamPolicyClaimNameOpenID ( ) ] = policyName
secret := globalActiveCred . SecretKey
cred , err := auth . GetNewCredentialsWithMetadata ( m , secret )
2019-10-23 01:59:13 -04:00
if err != nil {
return toJSONError ( ctx , err )
}
2020-03-22 01:10:13 -04:00
2021-01-25 13:15:03 -05:00
// Set the newly generated credentials.
if err = globalIAMSys . SetTempUser ( cred . AccessKey , cred , policyName ) ; err != nil {
return toJSONError ( ctx , err )
2019-10-23 01:59:13 -04:00
}
2021-01-25 13:15:03 -05:00
// Notify all other MinIO peers to reload temp users
for _ , nerr := range globalNotificationSys . LoadUser ( cred . AccessKey , true ) {
if nerr . Err != nil {
logger . GetReqInfo ( ctx ) . SetTags ( "peerAddress" , nerr . Host . String ( ) )
logger . LogIf ( ctx , nerr . Err )
}
2019-10-23 01:59:13 -04:00
}
2021-01-25 13:15:03 -05:00
reply . Token = cred . SessionToken
2019-10-23 01:59:13 -04:00
reply . UIVersion = browser . UIVersion
return nil
}
2016-11-22 14:12:38 -05:00
// toJSONError converts regular errors into more user friendly
// and consumable error message for the browser UI.
2019-06-03 18:40:04 -04:00
func toJSONError ( ctx context . Context , err error , params ... string ) ( jerr * json2 . Error ) {
apiErr := toWebAPIError ( ctx , err )
2016-11-22 14:12:38 -05:00
jerr = & json2 . Error {
Message : apiErr . Description ,
}
switch apiErr . Code {
2017-03-03 06:01:42 -05:00
// Reserved bucket name provided.
case "AllAccessDisabled" :
if len ( params ) > 0 {
jerr = & json2 . Error {
Message : fmt . Sprintf ( "All access to this bucket %s has been disabled." , params [ 0 ] ) ,
}
}
2016-11-22 14:12:38 -05:00
// Bucket name invalid with custom error message.
case "InvalidBucketName" :
if len ( params ) > 0 {
jerr = & json2 . Error {
2017-05-19 10:30:00 -04:00
Message : fmt . Sprintf ( "Bucket Name %s is invalid. Lowercase letters, period, hyphen, numerals are the only allowed characters and should be minimum 3 characters in length." , params [ 0 ] ) ,
2016-11-22 14:12:38 -05:00
}
}
// Bucket not found custom error message.
case "NoSuchBucket" :
if len ( params ) > 0 {
jerr = & json2 . Error {
Message : fmt . Sprintf ( "The specified bucket %s does not exist." , params [ 0 ] ) ,
}
}
// Object not found custom error message.
case "NoSuchKey" :
if len ( params ) > 1 {
jerr = & json2 . Error {
Message : fmt . Sprintf ( "The specified key %s does not exist" , params [ 1 ] ) ,
}
}
// Add more custom error messages here with more context.
}
return jerr
}
// toWebAPIError - convert into error into APIError.
2019-06-03 18:40:04 -04:00
func toWebAPIError ( ctx context . Context , err error ) APIError {
2019-03-18 10:46:20 -04:00
switch err {
2020-01-06 19:15:22 -05:00
case errNoAuthToken :
return APIError {
Code : "WebTokenMissing" ,
HTTPStatusCode : http . StatusBadRequest ,
Description : err . Error ( ) ,
}
2021-01-25 13:15:03 -05:00
case errSTSNotInitialized :
return APIError ( stsErrCodes . ToSTSErr ( ErrSTSNotInitialized ) )
2019-03-18 10:46:20 -04:00
case errServerNotInitialized :
2016-11-22 14:12:38 -05:00
return APIError {
Code : "XMinioServerNotInitialized" ,
HTTPStatusCode : http . StatusServiceUnavailable ,
Description : err . Error ( ) ,
}
2020-01-06 19:15:22 -05:00
case errAuthentication , auth . ErrInvalidAccessKeyLength ,
2020-05-07 00:31:44 -04:00
auth . ErrInvalidSecretKeyLength , errInvalidAccessKeyID , errAccessDenied , errLockedObject :
2017-01-03 04:33:00 -05:00
return APIError {
Code : "AccessDenied" ,
HTTPStatusCode : http . StatusForbidden ,
Description : err . Error ( ) ,
}
2019-03-18 10:46:20 -04:00
case errSizeUnspecified :
2017-02-02 13:45:00 -05:00
return APIError {
Code : "InvalidRequest" ,
HTTPStatusCode : http . StatusBadRequest ,
Description : err . Error ( ) ,
}
2019-03-18 10:46:20 -04:00
case errChangeCredNotAllowed :
2017-02-07 15:51:43 -05:00
return APIError {
Code : "MethodNotAllowed" ,
HTTPStatusCode : http . StatusMethodNotAllowed ,
Description : err . Error ( ) ,
}
2019-03-18 10:46:20 -04:00
case errInvalidBucketName :
2017-03-03 06:01:42 -05:00
return APIError {
2017-09-01 15:16:54 -04:00
Code : "InvalidBucketName" ,
HTTPStatusCode : http . StatusBadRequest ,
2017-03-03 06:01:42 -05:00
Description : err . Error ( ) ,
}
2019-03-18 10:46:20 -04:00
case errInvalidArgument :
2017-04-27 02:27:48 -04:00
return APIError {
Code : "InvalidArgument" ,
HTTPStatusCode : http . StatusBadRequest ,
Description : err . Error ( ) ,
}
2019-03-18 10:46:20 -04:00
case errEncryptedObject :
2018-08-24 10:56:24 -04:00
return getAPIError ( ErrSSEEncryptedObject )
2019-03-18 10:46:20 -04:00
case errInvalidEncryptionParameters :
2018-08-24 10:56:24 -04:00
return getAPIError ( ErrInvalidEncryptionParameters )
2019-03-18 10:46:20 -04:00
case errObjectTampered :
2018-08-24 10:56:24 -04:00
return getAPIError ( ErrObjectTampered )
2019-03-18 10:46:20 -04:00
case errMethodNotAllowed :
2018-06-06 21:10:51 -04:00
return getAPIError ( ErrMethodNotAllowed )
2016-11-22 14:12:38 -05:00
}
2018-06-06 21:10:51 -04:00
2016-11-22 14:12:38 -05:00
// Convert error type to api error code.
switch err . ( type ) {
case StorageFull :
2017-06-07 22:31:23 -04:00
return getAPIError ( ErrStorageFull )
2020-07-25 00:16:54 -04:00
case BucketQuotaExceeded :
return getAPIError ( ErrAdminBucketQuotaExceeded )
2016-11-22 14:12:38 -05:00
case BucketNotFound :
2017-06-07 22:31:23 -04:00
return getAPIError ( ErrNoSuchBucket )
2019-08-21 16:01:46 -04:00
case BucketNotEmpty :
return getAPIError ( ErrBucketNotEmpty )
2016-11-23 20:31:11 -05:00
case BucketExists :
2017-06-07 22:31:23 -04:00
return getAPIError ( ErrBucketAlreadyOwnedByYou )
2016-11-22 14:12:38 -05:00
case BucketNameInvalid :
2017-06-07 22:31:23 -04:00
return getAPIError ( ErrInvalidBucketName )
2017-10-22 01:30:34 -04:00
case hash . BadDigest :
2017-06-07 22:31:23 -04:00
return getAPIError ( ErrBadDigest )
2016-11-22 14:12:38 -05:00
case IncompleteBody :
2017-06-07 22:31:23 -04:00
return getAPIError ( ErrIncompleteBody )
2016-11-22 14:12:38 -05:00
case ObjectExistsAsDirectory :
2017-06-07 22:31:23 -04:00
return getAPIError ( ErrObjectExistsAsDirectory )
2016-11-22 14:12:38 -05:00
case ObjectNotFound :
2017-06-07 22:31:23 -04:00
return getAPIError ( ErrNoSuchKey )
2016-11-22 14:12:38 -05:00
case ObjectNameInvalid :
2017-06-07 22:31:23 -04:00
return getAPIError ( ErrNoSuchKey )
2016-11-22 14:12:38 -05:00
case InsufficientWriteQuorum :
2017-06-07 22:31:23 -04:00
return getAPIError ( ErrWriteQuorum )
2016-11-22 14:12:38 -05:00
case InsufficientReadQuorum :
2017-06-07 22:31:23 -04:00
return getAPIError ( ErrReadQuorum )
case NotImplemented :
return APIError {
Code : "NotImplemented" ,
HTTPStatusCode : http . StatusBadRequest ,
Description : "Functionality not implemented" ,
}
}
// Log unexpected and unhandled errors.
2019-06-03 18:40:04 -04:00
logger . LogIf ( ctx , err )
2020-01-03 14:28:52 -05:00
return toAPIError ( ctx , err )
2016-11-22 14:12:38 -05:00
}
// writeWebErrorResponse - set HTTP status code and write error description to the body.
func writeWebErrorResponse ( w http . ResponseWriter , err error ) {
2019-06-03 18:40:04 -04:00
reqInfo := & logger . ReqInfo {
DeploymentID : globalDeploymentID ,
}
2020-04-09 12:30:02 -04:00
ctx := logger . SetReqInfo ( GlobalContext , reqInfo )
2019-06-03 18:40:04 -04:00
apiErr := toWebAPIError ( ctx , err )
2016-11-22 14:12:38 -05:00
w . WriteHeader ( apiErr . HTTPStatusCode )
w . Write ( [ ] byte ( apiErr . Description ) )
}