mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
event: Enhance event message struct to provide origin server. (#3557)
`principalId` i.e user identity is kept as AccessKey in accordance with S3 spec. Additionally responseElements{} are added starting with `x-amz-request-id` is a hexadecimal of the event time itself in nanosecs. `x-minio-origin-server` - points to the server generating the event. Fixes #3556
This commit is contained in:
parent
0563a9235a
commit
b0cfceb211
@ -20,6 +20,11 @@ import (
|
||||
"encoding/xml"
|
||||
)
|
||||
|
||||
const (
|
||||
// Response request id.
|
||||
responseRequestIDKey = "x-amz-request-id"
|
||||
)
|
||||
|
||||
// ObjectIdentifier carries key name for the object to delete.
|
||||
type ObjectIdentifier struct {
|
||||
ObjectName string `xml:"Key"`
|
||||
|
@ -18,33 +18,24 @@ package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
const requestIDLen = 16
|
||||
|
||||
// mustGetRequestID generates and returns request ID string.
|
||||
func mustGetRequestID() string {
|
||||
reqBytes := make([]byte, requestIDLen)
|
||||
if _, err := rand.Read(reqBytes); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
for i := 0; i < requestIDLen; i++ {
|
||||
reqBytes[i] = alphaNumericTable[reqBytes[i]%alphaNumericTableLen]
|
||||
}
|
||||
|
||||
return string(reqBytes)
|
||||
// Returns a hexadecimal representation of time at the
|
||||
// time response is sent to the client.
|
||||
func mustGetRequestID(t time.Time) string {
|
||||
return fmt.Sprintf("%X", t.UnixNano())
|
||||
}
|
||||
|
||||
// Write http common headers
|
||||
func setCommonHeaders(w http.ResponseWriter) {
|
||||
// Set unique request ID for each reply.
|
||||
w.Header().Set("X-Amz-Request-Id", mustGetRequestID())
|
||||
w.Header().Set(responseRequestIDKey, mustGetRequestID(time.Now().UTC()))
|
||||
w.Header().Set("Server", ("Minio/" + ReleaseTag + " (" + runtime.GOOS + "; " + runtime.GOARCH + ")"))
|
||||
w.Header().Set("Accept-Ranges", "bytes")
|
||||
}
|
||||
|
@ -18,11 +18,12 @@ package cmd
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNewRequestID(t *testing.T) {
|
||||
// Ensure that it returns an alphanumeric result of length 16.
|
||||
var id = mustGetRequestID()
|
||||
var id = mustGetRequestID(time.Now().UTC())
|
||||
|
||||
if len(id) != 16 {
|
||||
t.Fail()
|
||||
|
@ -56,7 +56,7 @@ func enforceBucketPolicy(bucket string, action string, reqURL *url.URL) (s3Error
|
||||
}
|
||||
|
||||
// Construct resource in 'arn:aws:s3:::examplebucket/object' format.
|
||||
resource := AWSResourcePrefix + strings.TrimSuffix(strings.TrimPrefix(reqURL.Path, "/"), "/")
|
||||
resource := bucketARNPrefix + strings.TrimSuffix(strings.TrimPrefix(reqURL.Path, "/"), "/")
|
||||
|
||||
// Get conditions for policy verification.
|
||||
conditionKeyMap := make(map[string]set.StringSet)
|
||||
|
@ -114,15 +114,11 @@ func (eventName EventName) String() string {
|
||||
}
|
||||
}
|
||||
|
||||
// Indentity represents the user id, this is a compliance field.
|
||||
// Indentity represents the accessKey who caused the event.
|
||||
type identity struct {
|
||||
PrincipalID string `json:"principalId"`
|
||||
}
|
||||
|
||||
func defaultIdentity() identity {
|
||||
return identity{"minio"}
|
||||
}
|
||||
|
||||
// Notification event bucket metadata.
|
||||
type bucketMeta struct {
|
||||
Name string `json:"name"`
|
||||
@ -139,6 +135,21 @@ type objectMeta struct {
|
||||
Sequencer string `json:"sequencer"`
|
||||
}
|
||||
|
||||
const (
|
||||
// Event schema version number defaulting to the value in S3 spec.
|
||||
// ref: http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
|
||||
eventSchemaVersion = "1.0"
|
||||
|
||||
// Default ID found in bucket notification configuration.
|
||||
// ref: http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
|
||||
eventConfigID = "Config"
|
||||
)
|
||||
|
||||
const (
|
||||
// Response element origin endpoint key.
|
||||
responseOriginEndpointKey = "x-minio-origin-endpoint"
|
||||
)
|
||||
|
||||
// Notification event server specific metadata.
|
||||
type eventMeta struct {
|
||||
SchemaVersion string `json:"s3SchemaVersion"`
|
||||
@ -147,6 +158,16 @@ type eventMeta struct {
|
||||
Object objectMeta `json:"object"`
|
||||
}
|
||||
|
||||
const (
|
||||
// Event source static value defaulting to the value in S3 spec.
|
||||
// ref: http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
|
||||
eventSource = "aws:s3"
|
||||
|
||||
// Event version number defaulting to the value in S3 spec.
|
||||
// ref: http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
|
||||
eventVersion = "2.0"
|
||||
)
|
||||
|
||||
// NotificationEvent represents an Amazon an S3 bucket notification event.
|
||||
type NotificationEvent struct {
|
||||
EventVersion string `json:"eventVersion"`
|
||||
|
@ -40,7 +40,7 @@ func TestBucketPolicyResourceMatch(t *testing.T) {
|
||||
|
||||
// generates resource prefix.
|
||||
generateResource := func(bucketName, objectName string) string {
|
||||
return AWSResourcePrefix + bucketName + "/" + objectName
|
||||
return bucketARNPrefix + bucketName + "/" + objectName
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
@ -50,30 +50,30 @@ func TestBucketPolicyResourceMatch(t *testing.T) {
|
||||
}{
|
||||
// Test case 1-4.
|
||||
// Policy with resource ending with bucket/* allows access to all objects inside the given bucket.
|
||||
{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/*")), true},
|
||||
{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/*")), true},
|
||||
{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/*")), true},
|
||||
{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/*")), true},
|
||||
{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/*")), true},
|
||||
{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/*")), true},
|
||||
{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/*")), true},
|
||||
{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/*")), true},
|
||||
// Test case - 5.
|
||||
// Policy with resource ending with bucket/oo* should not allow access to bucket/output.txt.
|
||||
{generateResource("minio-bucket", "output.txt"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/oo*")), false},
|
||||
{generateResource("minio-bucket", "output.txt"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/oo*")), false},
|
||||
// Test case - 6.
|
||||
// Policy with resource ending with bucket/oo* should allow access to bucket/ootput.txt.
|
||||
{generateResource("minio-bucket", "ootput.txt"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/oo*")), true},
|
||||
{generateResource("minio-bucket", "ootput.txt"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/oo*")), true},
|
||||
// Test case - 7.
|
||||
// Policy with resource ending with bucket/oo* allows access to all sub-dirs starting with "oo" inside given bucket.
|
||||
{generateResource("minio-bucket", "oop-bucket/my-file"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/oo*")), true},
|
||||
{generateResource("minio-bucket", "oop-bucket/my-file"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/oo*")), true},
|
||||
// Test case - 8.
|
||||
{generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/Asia/Japan/*")), false},
|
||||
{generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/Asia/Japan/*")), false},
|
||||
// Test case - 9.
|
||||
{generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/Asia/Japan/*")), false},
|
||||
{generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/Asia/Japan/*")), false},
|
||||
// Test case - 10.
|
||||
// Proves that the name space is flat.
|
||||
{generateResource("minio-bucket", "Africa/Bihar/India/design_info.doc/Bihar"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix,
|
||||
{generateResource("minio-bucket", "Africa/Bihar/India/design_info.doc/Bihar"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix,
|
||||
"minio-bucket"+"/*/India/*/Bihar")), true},
|
||||
// Test case - 11.
|
||||
// Proves that the name space is flat.
|
||||
{generateResource("minio-bucket", "Asia/China/India/States/Bihar/output.txt"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix,
|
||||
{generateResource("minio-bucket", "Asia/China/India/States/Bihar/output.txt"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix,
|
||||
"minio-bucket"+"/*/India/*/Bihar/*")), true},
|
||||
}
|
||||
for i, testCase := range testCases {
|
||||
|
@ -29,11 +29,6 @@ import (
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
)
|
||||
|
||||
const (
|
||||
// AWSResourcePrefix - bucket policy resource prefix.
|
||||
AWSResourcePrefix = "arn:aws:s3:::"
|
||||
)
|
||||
|
||||
// supportedActionMap - lists all the actions supported by minio.
|
||||
var supportedActionMap = set.CreateStringSet("*", "s3:*", "s3:GetObject",
|
||||
"s3:ListBucket", "s3:PutObject", "s3:GetBucketLocation", "s3:DeleteObject",
|
||||
@ -111,11 +106,11 @@ func isValidResources(resources set.StringSet) (err error) {
|
||||
return err
|
||||
}
|
||||
for resource := range resources {
|
||||
if !strings.HasPrefix(resource, AWSResourcePrefix) {
|
||||
if !strings.HasPrefix(resource, bucketARNPrefix) {
|
||||
err = errors.New("Unsupported resource style found: ‘" + resource + "’, please validate your policy document")
|
||||
return err
|
||||
}
|
||||
resourceSuffix := strings.SplitAfter(resource, AWSResourcePrefix)[1]
|
||||
resourceSuffix := strings.SplitAfter(resource, bucketARNPrefix)[1]
|
||||
if len(resourceSuffix) == 0 || strings.HasPrefix(resourceSuffix, "/") {
|
||||
err = errors.New("Invalid resource style found: ‘" + resource + "’, please validate your policy document")
|
||||
return err
|
||||
@ -236,7 +231,7 @@ func checkBucketPolicyResources(bucket string, bucketPolicy *bucketPolicy) APIEr
|
||||
for _, statement := range bucketPolicy.Statements {
|
||||
for action := range statement.Actions {
|
||||
for resource := range statement.Resources {
|
||||
resourcePrefix := strings.SplitAfter(resource, AWSResourcePrefix)[1]
|
||||
resourcePrefix := strings.SplitAfter(resource, bucketARNPrefix)[1]
|
||||
if _, ok := invalidPrefixActions[action]; ok {
|
||||
// Resource prefix is not equal to bucket for
|
||||
// prefix invalid actions, reject them.
|
||||
|
@ -79,7 +79,7 @@ func getReadWriteObjectStatement(bucketName, objectPrefix string) policyStatemen
|
||||
objectResourceStatement.Principal = map[string]interface{}{
|
||||
"AWS": "*",
|
||||
}
|
||||
objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}...)
|
||||
objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
|
||||
objectResourceStatement.Actions = set.CreateStringSet(readWriteObjectActions...)
|
||||
return objectResourceStatement
|
||||
}
|
||||
@ -91,7 +91,7 @@ func getReadWriteBucketStatement(bucketName, objectPrefix string) policyStatemen
|
||||
bucketResourceStatement.Principal = map[string]interface{}{
|
||||
"AWS": "*",
|
||||
}
|
||||
bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}...)
|
||||
bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
|
||||
bucketResourceStatement.Actions = set.CreateStringSet(readWriteBucketActions...)
|
||||
return bucketResourceStatement
|
||||
}
|
||||
@ -111,7 +111,7 @@ func getReadOnlyBucketStatement(bucketName, objectPrefix string) policyStatement
|
||||
bucketResourceStatement.Principal = map[string]interface{}{
|
||||
"AWS": "*",
|
||||
}
|
||||
bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}...)
|
||||
bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
|
||||
bucketResourceStatement.Actions = set.CreateStringSet(readOnlyBucketActions...)
|
||||
return bucketResourceStatement
|
||||
}
|
||||
@ -123,7 +123,7 @@ func getReadOnlyObjectStatement(bucketName, objectPrefix string) policyStatement
|
||||
objectResourceStatement.Principal = map[string]interface{}{
|
||||
"AWS": "*",
|
||||
}
|
||||
objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}...)
|
||||
objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
|
||||
objectResourceStatement.Actions = set.CreateStringSet(readOnlyObjectActions...)
|
||||
return objectResourceStatement
|
||||
}
|
||||
@ -144,7 +144,7 @@ func getWriteOnlyBucketStatement(bucketName, objectPrefix string) policyStatemen
|
||||
bucketResourceStatement.Principal = map[string]interface{}{
|
||||
"AWS": "*",
|
||||
}
|
||||
bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}...)
|
||||
bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
|
||||
bucketResourceStatement.Actions = set.CreateStringSet(writeOnlyBucketActions...)
|
||||
return bucketResourceStatement
|
||||
}
|
||||
@ -156,7 +156,7 @@ func getWriteOnlyObjectStatement(bucketName, objectPrefix string) policyStatemen
|
||||
objectResourceStatement.Principal = map[string]interface{}{
|
||||
"AWS": "*",
|
||||
}
|
||||
objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}...)
|
||||
objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
|
||||
objectResourceStatement.Actions = set.CreateStringSet(writeOnlyObjectActions...)
|
||||
return objectResourceStatement
|
||||
}
|
||||
@ -269,19 +269,19 @@ func TestIsValidResources(t *testing.T) {
|
||||
// Empty Resources.
|
||||
{[]string{}, errors.New("Resource list cannot be empty"), false},
|
||||
// Test case - 2.
|
||||
// A valid resource should have prefix "arn:aws:s3:::".
|
||||
// A valid resource should have prefix bucketARNPrefix.
|
||||
{[]string{"my-resource"}, errors.New("Unsupported resource style found: ‘my-resource’, please validate your policy document"), false},
|
||||
// Test case - 3.
|
||||
// A Valid resource should have bucket name followed by "arn:aws:s3:::".
|
||||
{[]string{"arn:aws:s3:::"}, errors.New("Invalid resource style found: ‘arn:aws:s3:::’, please validate your policy document"), false},
|
||||
// A Valid resource should have bucket name followed by bucketARNPrefix.
|
||||
{[]string{bucketARNPrefix}, errors.New("Invalid resource style found: ‘arn:aws:s3:::’, please validate your policy document"), false},
|
||||
// Test Case - 4.
|
||||
// Valid resource shouldn't have slash('/') followed by "arn:aws:s3:::".
|
||||
{[]string{"arn:aws:s3:::/"}, errors.New("Invalid resource style found: ‘arn:aws:s3:::/’, please validate your policy document"), false},
|
||||
// Valid resource shouldn't have slash('/') followed by bucketARNPrefix.
|
||||
{[]string{bucketARNPrefix + "/"}, errors.New("Invalid resource style found: ‘arn:aws:s3:::/’, please validate your policy document"), false},
|
||||
|
||||
// Test cases with valid Resources.
|
||||
{[]string{"arn:aws:s3:::my-bucket"}, nil, true},
|
||||
{[]string{"arn:aws:s3:::my-bucket/Asia/*"}, nil, true},
|
||||
{[]string{"arn:aws:s3:::my-bucket/Asia/India/*"}, nil, true},
|
||||
{[]string{bucketARNPrefix + "my-bucket"}, nil, true},
|
||||
{[]string{bucketARNPrefix + "my-bucket/Asia/*"}, nil, true},
|
||||
{[]string{bucketARNPrefix + "my-bucket/Asia/India/*"}, nil, true},
|
||||
}
|
||||
for i, testCase := range testCases {
|
||||
err := isValidResources(set.CreateStringSet(testCase.resources...))
|
||||
|
@ -24,6 +24,12 @@ import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
const (
|
||||
// Static prefix to be used while constructing bucket ARN.
|
||||
// refer to S3 docs for more info.
|
||||
bucketARNPrefix = "arn:" + eventSource + ":::"
|
||||
)
|
||||
|
||||
// Variable represents bucket policies in memory.
|
||||
var globalBucketPolicies *bucketPolicies
|
||||
|
||||
|
19
cmd/certs.go
19
cmd/certs.go
@ -145,3 +145,22 @@ func parseCertificateChain(bytes []byte) ([]*x509.Certificate, error) {
|
||||
}
|
||||
return certs, nil
|
||||
}
|
||||
|
||||
// loadRootCAs fetches CA files provided in minio config and adds them to globalRootCAs
|
||||
// Currently under Windows, there is no way to load system + user CAs at the same time
|
||||
func loadRootCAs() {
|
||||
caFiles := mustGetCAFiles()
|
||||
if len(caFiles) == 0 {
|
||||
return
|
||||
}
|
||||
// Get system cert pool, and empty cert pool under Windows because it is not supported
|
||||
globalRootCAs = mustGetSystemCertPool()
|
||||
// Load custom root CAs for client requests
|
||||
for _, caFile := range caFiles {
|
||||
caCert, err := ioutil.ReadFile(caFile)
|
||||
if err != nil {
|
||||
fatalIf(err, "Unable to load a CA file")
|
||||
}
|
||||
globalRootCAs.AppendCertsFromPEM(caCert)
|
||||
}
|
||||
}
|
||||
|
@ -88,49 +88,76 @@ type eventData struct {
|
||||
// New notification event constructs a new notification event message from
|
||||
// input request metadata which completed successfully.
|
||||
func newNotificationEvent(event eventData) NotificationEvent {
|
||||
/// Construct a new object created event.
|
||||
// Fetch the region.
|
||||
region := serverConfig.GetRegion()
|
||||
tnow := time.Now().UTC()
|
||||
sequencer := fmt.Sprintf("%X", tnow.UnixNano())
|
||||
|
||||
// Fetch the credentials.
|
||||
creds := serverConfig.GetCredential()
|
||||
|
||||
// Time when Minio finished processing the request.
|
||||
eventTime := time.Now().UTC()
|
||||
|
||||
// API endpoint is captured here to be returned back
|
||||
// to the client for it to differentiate from which
|
||||
// server the request came from.
|
||||
var apiEndpoint string
|
||||
if len(globalAPIEndpoints) >= 1 {
|
||||
apiEndpoint = globalAPIEndpoints[0]
|
||||
}
|
||||
|
||||
// Fetch a hexadecimal representation of event time in nano seconds.
|
||||
uniqueID := mustGetRequestID(eventTime)
|
||||
|
||||
/// Construct a new object created event.
|
||||
|
||||
// Following blocks fills in all the necessary details of s3
|
||||
// event message structure.
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
|
||||
nEvent := NotificationEvent{
|
||||
EventVersion: "2.0",
|
||||
EventSource: "aws:s3",
|
||||
EventVersion: eventVersion,
|
||||
EventSource: eventSource,
|
||||
AwsRegion: region,
|
||||
EventTime: tnow.Format(timeFormatAMZ),
|
||||
EventTime: eventTime.Format(timeFormatAMZ),
|
||||
EventName: event.Type.String(),
|
||||
UserIdentity: defaultIdentity(),
|
||||
UserIdentity: identity{creds.AccessKey},
|
||||
RequestParameters: event.ReqParams,
|
||||
ResponseElements: map[string]string{},
|
||||
ResponseElements: map[string]string{
|
||||
responseRequestIDKey: uniqueID,
|
||||
// Following is a custom response element to indicate
|
||||
// event origin server endpoint.
|
||||
responseOriginEndpointKey: apiEndpoint,
|
||||
},
|
||||
S3: eventMeta{
|
||||
SchemaVersion: "1.0",
|
||||
ConfigurationID: "Config",
|
||||
SchemaVersion: eventSchemaVersion,
|
||||
ConfigurationID: eventConfigID,
|
||||
Bucket: bucketMeta{
|
||||
Name: event.Bucket,
|
||||
OwnerIdentity: defaultIdentity(),
|
||||
ARN: "arn:aws:s3:::" + event.Bucket,
|
||||
OwnerIdentity: identity{creds.AccessKey},
|
||||
ARN: bucketARNPrefix + event.Bucket,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// Escape the object name. For example "red flower.jpg" becomes "red+flower.jpg".
|
||||
escapedObj := url.QueryEscape(event.ObjInfo.Name)
|
||||
|
||||
// For delete object event type, we do not need to set ETag and Size.
|
||||
if event.Type == ObjectRemovedDelete {
|
||||
nEvent.S3.Object = objectMeta{
|
||||
Key: escapedObj,
|
||||
Sequencer: sequencer,
|
||||
Sequencer: uniqueID,
|
||||
}
|
||||
return nEvent
|
||||
}
|
||||
|
||||
// For all other events we should set ETag and Size.
|
||||
nEvent.S3.Object = objectMeta{
|
||||
Key: escapedObj,
|
||||
ETag: event.ObjInfo.MD5Sum,
|
||||
Size: event.ObjInfo.Size,
|
||||
Sequencer: sequencer,
|
||||
Sequencer: uniqueID,
|
||||
}
|
||||
|
||||
// Success.
|
||||
return nEvent
|
||||
}
|
||||
|
@ -73,6 +73,7 @@ var (
|
||||
|
||||
// Cache expiry.
|
||||
globalCacheExpiry = objcache.DefaultExpiry
|
||||
|
||||
// Minio local server address (in `host:port` format)
|
||||
globalMinioAddr = ""
|
||||
// Minio default port, can be changed through command line.
|
||||
@ -80,6 +81,9 @@ var (
|
||||
// Holds the host that was passed using --address
|
||||
globalMinioHost = ""
|
||||
|
||||
// Holds the list of API endpoints for a given server.
|
||||
globalAPIEndpoints = []string{}
|
||||
|
||||
// Peer communication struct
|
||||
globalS3Peers = s3Peers{}
|
||||
|
||||
|
@ -17,6 +17,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
@ -225,5 +226,13 @@ func (n *nsLockMap) deleteLockInfoEntryForOps(param nsParam, opsID string) error
|
||||
|
||||
// Return randomly generated string ID
|
||||
func getOpsID() string {
|
||||
return mustGetRequestID()
|
||||
const opsIDLen = 16
|
||||
opsIDBytes := make([]byte, opsIDLen)
|
||||
if _, err := rand.Read(opsIDBytes); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
for i := 0; i < opsIDLen; i++ {
|
||||
opsIDBytes[i] = alphaNumericTable[opsIDBytes[i]%alphaNumericTableLen]
|
||||
}
|
||||
return string(opsIDBytes)
|
||||
}
|
||||
|
@ -228,6 +228,25 @@ func verifyLockState(l lockStateCase, t *testing.T, testNum int) {
|
||||
verifyLockStats(l, t, testNum)
|
||||
}
|
||||
|
||||
func TestGetOpsID(t *testing.T) {
|
||||
// Ensure that it returns an alphanumeric result of length 16.
|
||||
var id = getOpsID()
|
||||
|
||||
if len(id) != 16 {
|
||||
t.Fail()
|
||||
}
|
||||
|
||||
var e rune
|
||||
for _, char := range id {
|
||||
e = rune(char)
|
||||
|
||||
// Ensure that it is alphanumeric, in this case, between 0-9 and A-Z.
|
||||
if !(('0' <= e && e <= '9') || ('A' <= e && e <= 'Z')) {
|
||||
t.Fail()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewDebugLockInfoPerVolumePath - Validates the values initialized by newDebugLockInfoPerVolumePath().
|
||||
func TestNewDebugLockInfoPerVolumePath(t *testing.T) {
|
||||
lockInfo := &debugLockInfoPerVolumePath{
|
||||
|
@ -18,9 +18,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
@ -129,69 +127,6 @@ func parseStorageEndpoints(eps []string) (endpoints []*url.URL, err error) {
|
||||
return endpoints, nil
|
||||
}
|
||||
|
||||
// getListenIPs - gets all the ips to listen on.
|
||||
func getListenIPs(serverAddr string) (hosts []string, port string, err error) {
|
||||
var host string
|
||||
host, port, err = net.SplitHostPort(serverAddr)
|
||||
if err != nil {
|
||||
return nil, port, fmt.Errorf("Unable to parse host address %s", err)
|
||||
}
|
||||
if host == "" {
|
||||
var ipv4s []net.IP
|
||||
ipv4s, err = getInterfaceIPv4s()
|
||||
if err != nil {
|
||||
return nil, port, fmt.Errorf("Unable reverse sorted ips from hosts %s", err)
|
||||
}
|
||||
for _, ip := range ipv4s {
|
||||
hosts = append(hosts, ip.String())
|
||||
}
|
||||
return hosts, port, nil
|
||||
} // if host != "" {
|
||||
// Proceed to append itself, since user requested a specific endpoint.
|
||||
hosts = append(hosts, host)
|
||||
return hosts, port, nil
|
||||
}
|
||||
|
||||
// Finalizes the endpoints based on the host list and port.
|
||||
func finalizeEndpoints(tls bool, apiServer *http.Server) (endPoints []string) {
|
||||
// Verify current scheme.
|
||||
scheme := "http"
|
||||
if tls {
|
||||
scheme = "https"
|
||||
}
|
||||
|
||||
// Get list of listen ips and port.
|
||||
hosts, port, err := getListenIPs(apiServer.Addr)
|
||||
fatalIf(err, "Unable to get list of ips to listen on")
|
||||
|
||||
// Construct proper endpoints.
|
||||
for _, host := range hosts {
|
||||
endPoints = append(endPoints, fmt.Sprintf("%s://%s:%s", scheme, host, port))
|
||||
}
|
||||
|
||||
// Success.
|
||||
return endPoints
|
||||
}
|
||||
|
||||
// loadRootCAs fetches CA files provided in minio config and adds them to globalRootCAs
|
||||
// Currently under Windows, there is no way to load system + user CAs at the same time
|
||||
func loadRootCAs() {
|
||||
caFiles := mustGetCAFiles()
|
||||
if len(caFiles) == 0 {
|
||||
return
|
||||
}
|
||||
// Get system cert pool, and empty cert pool under Windows because it is not supported
|
||||
globalRootCAs = mustGetSystemCertPool()
|
||||
// Load custom root CAs for client requests
|
||||
for _, caFile := range mustGetCAFiles() {
|
||||
caCert, err := ioutil.ReadFile(caFile)
|
||||
if err != nil {
|
||||
fatalIf(err, "Unable to load a CA file")
|
||||
}
|
||||
globalRootCAs.AppendCertsFromPEM(caCert)
|
||||
}
|
||||
}
|
||||
|
||||
// initServerConfig initialize server config.
|
||||
func initServerConfig(c *cli.Context) {
|
||||
// Create certs path.
|
||||
@ -430,6 +365,9 @@ func serverMain(c *cli.Context) {
|
||||
fatalIf(errInvalidArgument, "None of the disks passed as command line args are local to this server.")
|
||||
}
|
||||
|
||||
// Is TLS configured?.
|
||||
tls := isSSL()
|
||||
|
||||
// Sort endpoints for consistent ordering across multiple
|
||||
// nodes in a distributed setup. This is to avoid format.json
|
||||
// corruption if the disks aren't supplied in the same order
|
||||
@ -473,15 +411,15 @@ func serverMain(c *cli.Context) {
|
||||
// Initialize a new HTTP server.
|
||||
apiServer := NewServerMux(serverAddr, handler)
|
||||
|
||||
// If https.
|
||||
tls := isSSL()
|
||||
|
||||
// Fetch endpoints which we are going to serve from.
|
||||
endPoints := finalizeEndpoints(tls, apiServer.Server)
|
||||
|
||||
// Initialize local server address
|
||||
// Set the global minio addr for this server.
|
||||
globalMinioAddr = getLocalAddress(srvConfig)
|
||||
|
||||
// Determine API endpoints where we are going to serve the S3 API from.
|
||||
apiEndPoints := finalizeAPIEndpoints(tls, apiServer.Server)
|
||||
|
||||
// Set the global API endpoints value.
|
||||
globalAPIEndpoints = apiEndPoints
|
||||
|
||||
// Initialize S3 Peers inter-node communication
|
||||
initGlobalS3Peers(endpoints)
|
||||
|
||||
@ -512,7 +450,7 @@ func serverMain(c *cli.Context) {
|
||||
globalObjLayerMutex.Unlock()
|
||||
|
||||
// Prints the formatted startup message once object layer is initialized.
|
||||
printStartupMessage(endPoints)
|
||||
printStartupMessage(apiEndPoints)
|
||||
|
||||
// Waits on the server.
|
||||
<-globalServiceDoneCh
|
||||
|
@ -63,7 +63,7 @@ func TestGetListenIPs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFinalizeEndpoints(t *testing.T) {
|
||||
func TestFinalizeAPIEndpoints(t *testing.T) {
|
||||
testCases := []struct {
|
||||
tls bool
|
||||
addr string
|
||||
@ -75,7 +75,7 @@ func TestFinalizeEndpoints(t *testing.T) {
|
||||
}
|
||||
|
||||
for i, test := range testCases {
|
||||
endPoints := finalizeEndpoints(test.tls, &http.Server{Addr: test.addr})
|
||||
endPoints := finalizeAPIEndpoints(test.tls, &http.Server{Addr: test.addr})
|
||||
if len(endPoints) <= 0 {
|
||||
t.Errorf("Test case %d returned with no API end points for %s",
|
||||
i+1, test.addr)
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
* Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -43,20 +43,30 @@ func getFormatStr(strLen int, padding int) string {
|
||||
}
|
||||
|
||||
// Prints the formatted startup message.
|
||||
func printStartupMessage(endPoints []string) {
|
||||
func printStartupMessage(apiEndPoints []string) {
|
||||
// If quiet flag is set do not print startup message.
|
||||
if globalQuiet {
|
||||
return
|
||||
}
|
||||
printServerCommonMsg(endPoints)
|
||||
printCLIAccessMsg(endPoints[0])
|
||||
|
||||
// Prints credential, region and browser access.
|
||||
printServerCommonMsg(apiEndPoints)
|
||||
|
||||
// Prints `mc` cli configuration message chooses
|
||||
// first endpoint as default.
|
||||
printCLIAccessMsg(apiEndPoints[0])
|
||||
|
||||
// Prints documentation message.
|
||||
printObjectAPIMsg()
|
||||
|
||||
// Object layer is initialized then print StorageInfo.
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI != nil {
|
||||
printStorageInfo(objAPI.StorageInfo())
|
||||
}
|
||||
|
||||
// SSL is configured reads certification chain, prints
|
||||
// authority and expiry.
|
||||
if isSSL() {
|
||||
certs, err := readCertificateChain()
|
||||
fatalIf(err, "Unable to read certificate chain.")
|
||||
@ -65,23 +75,23 @@ func printStartupMessage(endPoints []string) {
|
||||
}
|
||||
|
||||
// Prints common server startup message. Prints credential, region and browser access.
|
||||
func printServerCommonMsg(endPoints []string) {
|
||||
func printServerCommonMsg(apiEndpoints []string) {
|
||||
// Get saved credentials.
|
||||
cred := serverConfig.GetCredential()
|
||||
|
||||
// Get saved region.
|
||||
region := serverConfig.GetRegion()
|
||||
|
||||
endPointStr := strings.Join(endPoints, " ")
|
||||
apiEndpointStr := strings.Join(apiEndpoints, " ")
|
||||
// Colorize the message and print.
|
||||
console.Println(colorBlue("\nEndpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(endPointStr), 1), endPointStr)))
|
||||
console.Println(colorBlue("\nEndpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
|
||||
console.Println(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
|
||||
console.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
|
||||
console.Println(colorBlue("Region: ") + colorBold(fmt.Sprintf(getFormatStr(len(region), 3), region)))
|
||||
printEventNotifiers()
|
||||
|
||||
console.Println(colorBlue("\nBrowser Access:"))
|
||||
console.Println(fmt.Sprintf(getFormatStr(len(endPointStr), 3), endPointStr))
|
||||
console.Println(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 3), apiEndpointStr))
|
||||
}
|
||||
|
||||
// Prints bucket notification configurations.
|
||||
|
@ -94,3 +94,39 @@ func TestCertificateNotExpired(t *testing.T) {
|
||||
t.Fatalf("Expected empty message was: %s", msg)
|
||||
}
|
||||
}
|
||||
|
||||
// Test printing server common message.
|
||||
func TestPrintServerCommonMessage(t *testing.T) {
|
||||
root, err := newTestConfig("us-east-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer removeAll(root)
|
||||
|
||||
apiEndpoints := []string{"127.0.0.1:9000"}
|
||||
printServerCommonMsg(apiEndpoints)
|
||||
}
|
||||
|
||||
// Tests print cli access message.
|
||||
func TestPrintCLIAccessMsg(t *testing.T) {
|
||||
root, err := newTestConfig("us-east-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer removeAll(root)
|
||||
|
||||
apiEndpoints := []string{"127.0.0.1:9000"}
|
||||
printCLIAccessMsg(apiEndpoints[0])
|
||||
}
|
||||
|
||||
// Test print startup message.
|
||||
func TestPrintStartupMessage(t *testing.T) {
|
||||
root, err := newTestConfig("us-east-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer removeAll(root)
|
||||
|
||||
apiEndpoints := []string{"127.0.0.1:9000"}
|
||||
printStartupMessage(apiEndpoints)
|
||||
}
|
||||
|
67
cmd/server-startup-utils.go
Normal file
67
cmd/server-startup-utils.go
Normal file
@ -0,0 +1,67 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// getListenIPs - gets all the ips to listen on.
|
||||
func getListenIPs(serverAddr string) (hosts []string, port string, err error) {
|
||||
var host string
|
||||
host, port, err = net.SplitHostPort(serverAddr)
|
||||
if err != nil {
|
||||
return nil, port, fmt.Errorf("Unable to parse host address %s", err)
|
||||
}
|
||||
if host == "" {
|
||||
var ipv4s []net.IP
|
||||
ipv4s, err = getInterfaceIPv4s()
|
||||
if err != nil {
|
||||
return nil, port, fmt.Errorf("Unable reverse sort ips from hosts %s", err)
|
||||
}
|
||||
for _, ip := range ipv4s {
|
||||
hosts = append(hosts, ip.String())
|
||||
}
|
||||
return hosts, port, nil
|
||||
} // if host != "" {
|
||||
// Proceed to append itself, since user requested a specific endpoint.
|
||||
hosts = append(hosts, host)
|
||||
return hosts, port, nil
|
||||
}
|
||||
|
||||
// Finalizes the API endpoints based on the host list and port.
|
||||
func finalizeAPIEndpoints(tls bool, apiServer *http.Server) (endPoints []string) {
|
||||
// Verify current scheme.
|
||||
scheme := "http"
|
||||
if tls {
|
||||
scheme = "https"
|
||||
}
|
||||
|
||||
// Get list of listen ips and port.
|
||||
hosts, port, err := getListenIPs(apiServer.Addr)
|
||||
fatalIf(err, "Unable to get list of ips to listen on")
|
||||
|
||||
// Construct proper endpoints.
|
||||
for _, host := range hosts {
|
||||
endPoints = append(endPoints, fmt.Sprintf("%s://%s:%s", scheme, host, port))
|
||||
}
|
||||
|
||||
// Success.
|
||||
return endPoints
|
||||
}
|
@ -948,14 +948,14 @@ func testWebGetBucketPolicyHandler(obj ObjectLayer, instanceType string, t TestE
|
||||
Actions: set.CreateStringSet("s3:GetBucketLocation", "s3:ListBucket"),
|
||||
Effect: "Allow",
|
||||
Principal: map[string][]string{"AWS": {"*"}},
|
||||
Resources: set.CreateStringSet("arn:aws:s3:::" + bucketName),
|
||||
Resources: set.CreateStringSet(bucketARNPrefix + bucketName),
|
||||
Sid: "",
|
||||
},
|
||||
{
|
||||
Actions: set.CreateStringSet("s3:GetObject"),
|
||||
Effect: "Allow",
|
||||
Principal: map[string][]string{"AWS": {"*"}},
|
||||
Resources: set.CreateStringSet("arn:aws:s3:::" + bucketName + "/*"),
|
||||
Resources: set.CreateStringSet(bucketARNPrefix + bucketName + "/*"),
|
||||
Sid: "",
|
||||
},
|
||||
},
|
||||
@ -1031,7 +1031,7 @@ func testWebListAllBucketPoliciesHandler(obj ObjectLayer, instanceType string, t
|
||||
Actions: set.CreateStringSet("s3:GetBucketLocation"),
|
||||
Effect: "Allow",
|
||||
Principal: map[string][]string{"AWS": {"*"}},
|
||||
Resources: set.CreateStringSet("arn:aws:s3:::" + bucketName),
|
||||
Resources: set.CreateStringSet(bucketARNPrefix + bucketName),
|
||||
Sid: "",
|
||||
},
|
||||
{
|
||||
@ -1043,14 +1043,14 @@ func testWebListAllBucketPoliciesHandler(obj ObjectLayer, instanceType string, t
|
||||
},
|
||||
Effect: "Allow",
|
||||
Principal: map[string][]string{"AWS": {"*"}},
|
||||
Resources: set.CreateStringSet("arn:aws:s3:::" + bucketName),
|
||||
Resources: set.CreateStringSet(bucketARNPrefix + bucketName),
|
||||
Sid: "",
|
||||
},
|
||||
{
|
||||
Actions: set.CreateStringSet("s3:ListBucketMultipartUploads"),
|
||||
Effect: "Allow",
|
||||
Principal: map[string][]string{"AWS": {"*"}},
|
||||
Resources: set.CreateStringSet("arn:aws:s3:::" + bucketName),
|
||||
Resources: set.CreateStringSet(bucketARNPrefix + bucketName),
|
||||
Sid: "",
|
||||
},
|
||||
{
|
||||
@ -1058,7 +1058,7 @@ func testWebListAllBucketPoliciesHandler(obj ObjectLayer, instanceType string, t
|
||||
"s3:GetObject", "s3:ListMultipartUploadParts", "s3:PutObject"),
|
||||
Effect: "Allow",
|
||||
Principal: map[string][]string{"AWS": {"*"}},
|
||||
Resources: set.CreateStringSet("arn:aws:s3:::" + bucketName + "/hello*"),
|
||||
Resources: set.CreateStringSet(bucketARNPrefix + bucketName + "/hello*"),
|
||||
Sid: "",
|
||||
},
|
||||
},
|
||||
|
Loading…
Reference in New Issue
Block a user