From b0cfceb2110e44f20a9095c82d9dedf78ab25397 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 10 Jan 2017 16:43:48 -0800 Subject: [PATCH] event: Enhance event message struct to provide origin server. (#3557) `principalId` i.e user identity is kept as AccessKey in accordance with S3 spec. Additionally responseElements{} are added starting with `x-amz-request-id` is a hexadecimal of the event time itself in nanosecs. `x-minio-origin-server` - points to the server generating the event. Fixes #3556 --- cmd/api-datatypes.go | 5 ++ cmd/api-headers.go | 23 +++----- cmd/api-headers_test.go | 3 +- cmd/bucket-handlers.go | 2 +- cmd/bucket-notification-datatypes.go | 31 ++++++++-- cmd/bucket-policy-handlers_test.go | 24 ++++---- cmd/bucket-policy-parser.go | 11 +--- cmd/bucket-policy-parser_test.go | 28 +++++----- cmd/bucket-policy.go | 6 ++ cmd/certs.go | 19 +++++++ cmd/event-notifier.go | 55 +++++++++++++----- cmd/globals.go | 4 ++ cmd/lock-instrument.go | 11 +++- cmd/lock-instrument_test.go | 19 +++++++ cmd/server-main.go | 84 ++++------------------------ cmd/server-main_test.go | 4 +- cmd/server-startup-msg.go | 26 ++++++--- cmd/server-startup-msg_test.go | 36 ++++++++++++ cmd/server-startup-utils.go | 67 ++++++++++++++++++++++ cmd/web-handlers_test.go | 12 ++-- 20 files changed, 309 insertions(+), 161 deletions(-) create mode 100644 cmd/server-startup-utils.go diff --git a/cmd/api-datatypes.go b/cmd/api-datatypes.go index 12b7f4bf7..f74c64440 100644 --- a/cmd/api-datatypes.go +++ b/cmd/api-datatypes.go @@ -20,6 +20,11 @@ import ( "encoding/xml" ) +const ( + // Response request id. + responseRequestIDKey = "x-amz-request-id" +) + // ObjectIdentifier carries key name for the object to delete. type ObjectIdentifier struct { ObjectName string `xml:"Key"` diff --git a/cmd/api-headers.go b/cmd/api-headers.go index a16470ca0..62d32a0ec 100644 --- a/cmd/api-headers.go +++ b/cmd/api-headers.go @@ -18,33 +18,24 @@ package cmd import ( "bytes" - "crypto/rand" "encoding/xml" + "fmt" "net/http" "runtime" "strconv" + "time" ) -const requestIDLen = 16 - -// mustGetRequestID generates and returns request ID string. -func mustGetRequestID() string { - reqBytes := make([]byte, requestIDLen) - if _, err := rand.Read(reqBytes); err != nil { - panic(err) - } - - for i := 0; i < requestIDLen; i++ { - reqBytes[i] = alphaNumericTable[reqBytes[i]%alphaNumericTableLen] - } - - return string(reqBytes) +// Returns a hexadecimal representation of time at the +// time response is sent to the client. +func mustGetRequestID(t time.Time) string { + return fmt.Sprintf("%X", t.UnixNano()) } // Write http common headers func setCommonHeaders(w http.ResponseWriter) { // Set unique request ID for each reply. - w.Header().Set("X-Amz-Request-Id", mustGetRequestID()) + w.Header().Set(responseRequestIDKey, mustGetRequestID(time.Now().UTC())) w.Header().Set("Server", ("Minio/" + ReleaseTag + " (" + runtime.GOOS + "; " + runtime.GOARCH + ")")) w.Header().Set("Accept-Ranges", "bytes") } diff --git a/cmd/api-headers_test.go b/cmd/api-headers_test.go index 540d136ee..0d19f13dd 100644 --- a/cmd/api-headers_test.go +++ b/cmd/api-headers_test.go @@ -18,11 +18,12 @@ package cmd import ( "testing" + "time" ) func TestNewRequestID(t *testing.T) { // Ensure that it returns an alphanumeric result of length 16. - var id = mustGetRequestID() + var id = mustGetRequestID(time.Now().UTC()) if len(id) != 16 { t.Fail() diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index 5aa99885c..8f845a046 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -56,7 +56,7 @@ func enforceBucketPolicy(bucket string, action string, reqURL *url.URL) (s3Error } // Construct resource in 'arn:aws:s3:::examplebucket/object' format. - resource := AWSResourcePrefix + strings.TrimSuffix(strings.TrimPrefix(reqURL.Path, "/"), "/") + resource := bucketARNPrefix + strings.TrimSuffix(strings.TrimPrefix(reqURL.Path, "/"), "/") // Get conditions for policy verification. conditionKeyMap := make(map[string]set.StringSet) diff --git a/cmd/bucket-notification-datatypes.go b/cmd/bucket-notification-datatypes.go index 252947602..02695b36f 100644 --- a/cmd/bucket-notification-datatypes.go +++ b/cmd/bucket-notification-datatypes.go @@ -114,15 +114,11 @@ func (eventName EventName) String() string { } } -// Indentity represents the user id, this is a compliance field. +// Indentity represents the accessKey who caused the event. type identity struct { PrincipalID string `json:"principalId"` } -func defaultIdentity() identity { - return identity{"minio"} -} - // Notification event bucket metadata. type bucketMeta struct { Name string `json:"name"` @@ -139,6 +135,21 @@ type objectMeta struct { Sequencer string `json:"sequencer"` } +const ( + // Event schema version number defaulting to the value in S3 spec. + // ref: http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html + eventSchemaVersion = "1.0" + + // Default ID found in bucket notification configuration. + // ref: http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html + eventConfigID = "Config" +) + +const ( + // Response element origin endpoint key. + responseOriginEndpointKey = "x-minio-origin-endpoint" +) + // Notification event server specific metadata. type eventMeta struct { SchemaVersion string `json:"s3SchemaVersion"` @@ -147,6 +158,16 @@ type eventMeta struct { Object objectMeta `json:"object"` } +const ( + // Event source static value defaulting to the value in S3 spec. + // ref: http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html + eventSource = "aws:s3" + + // Event version number defaulting to the value in S3 spec. + // ref: http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html + eventVersion = "2.0" +) + // NotificationEvent represents an Amazon an S3 bucket notification event. type NotificationEvent struct { EventVersion string `json:"eventVersion"` diff --git a/cmd/bucket-policy-handlers_test.go b/cmd/bucket-policy-handlers_test.go index 16536de51..cfc690ef7 100644 --- a/cmd/bucket-policy-handlers_test.go +++ b/cmd/bucket-policy-handlers_test.go @@ -40,7 +40,7 @@ func TestBucketPolicyResourceMatch(t *testing.T) { // generates resource prefix. generateResource := func(bucketName, objectName string) string { - return AWSResourcePrefix + bucketName + "/" + objectName + return bucketARNPrefix + bucketName + "/" + objectName } testCases := []struct { @@ -50,30 +50,30 @@ func TestBucketPolicyResourceMatch(t *testing.T) { }{ // Test case 1-4. // Policy with resource ending with bucket/* allows access to all objects inside the given bucket. - {generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/*")), true}, - {generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/*")), true}, - {generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/*")), true}, - {generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/*")), true}, + {generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/*")), true}, + {generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/*")), true}, + {generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/*")), true}, + {generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/*")), true}, // Test case - 5. // Policy with resource ending with bucket/oo* should not allow access to bucket/output.txt. - {generateResource("minio-bucket", "output.txt"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/oo*")), false}, + {generateResource("minio-bucket", "output.txt"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/oo*")), false}, // Test case - 6. // Policy with resource ending with bucket/oo* should allow access to bucket/ootput.txt. - {generateResource("minio-bucket", "ootput.txt"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/oo*")), true}, + {generateResource("minio-bucket", "ootput.txt"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/oo*")), true}, // Test case - 7. // Policy with resource ending with bucket/oo* allows access to all sub-dirs starting with "oo" inside given bucket. - {generateResource("minio-bucket", "oop-bucket/my-file"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/oo*")), true}, + {generateResource("minio-bucket", "oop-bucket/my-file"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/oo*")), true}, // Test case - 8. - {generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/Asia/Japan/*")), false}, + {generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/Asia/Japan/*")), false}, // Test case - 9. - {generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/Asia/Japan/*")), false}, + {generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/Asia/Japan/*")), false}, // Test case - 10. // Proves that the name space is flat. - {generateResource("minio-bucket", "Africa/Bihar/India/design_info.doc/Bihar"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, + {generateResource("minio-bucket", "Africa/Bihar/India/design_info.doc/Bihar"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/*/India/*/Bihar")), true}, // Test case - 11. // Proves that the name space is flat. - {generateResource("minio-bucket", "Asia/China/India/States/Bihar/output.txt"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, + {generateResource("minio-bucket", "Asia/China/India/States/Bihar/output.txt"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/*/India/*/Bihar/*")), true}, } for i, testCase := range testCases { diff --git a/cmd/bucket-policy-parser.go b/cmd/bucket-policy-parser.go index da47c3d8e..1264a610c 100644 --- a/cmd/bucket-policy-parser.go +++ b/cmd/bucket-policy-parser.go @@ -29,11 +29,6 @@ import ( "github.com/minio/minio-go/pkg/set" ) -const ( - // AWSResourcePrefix - bucket policy resource prefix. - AWSResourcePrefix = "arn:aws:s3:::" -) - // supportedActionMap - lists all the actions supported by minio. var supportedActionMap = set.CreateStringSet("*", "s3:*", "s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:GetBucketLocation", "s3:DeleteObject", @@ -111,11 +106,11 @@ func isValidResources(resources set.StringSet) (err error) { return err } for resource := range resources { - if !strings.HasPrefix(resource, AWSResourcePrefix) { + if !strings.HasPrefix(resource, bucketARNPrefix) { err = errors.New("Unsupported resource style found: ‘" + resource + "’, please validate your policy document") return err } - resourceSuffix := strings.SplitAfter(resource, AWSResourcePrefix)[1] + resourceSuffix := strings.SplitAfter(resource, bucketARNPrefix)[1] if len(resourceSuffix) == 0 || strings.HasPrefix(resourceSuffix, "/") { err = errors.New("Invalid resource style found: ‘" + resource + "’, please validate your policy document") return err @@ -236,7 +231,7 @@ func checkBucketPolicyResources(bucket string, bucketPolicy *bucketPolicy) APIEr for _, statement := range bucketPolicy.Statements { for action := range statement.Actions { for resource := range statement.Resources { - resourcePrefix := strings.SplitAfter(resource, AWSResourcePrefix)[1] + resourcePrefix := strings.SplitAfter(resource, bucketARNPrefix)[1] if _, ok := invalidPrefixActions[action]; ok { // Resource prefix is not equal to bucket for // prefix invalid actions, reject them. diff --git a/cmd/bucket-policy-parser_test.go b/cmd/bucket-policy-parser_test.go index e7fd82dab..522dc2c63 100644 --- a/cmd/bucket-policy-parser_test.go +++ b/cmd/bucket-policy-parser_test.go @@ -79,7 +79,7 @@ func getReadWriteObjectStatement(bucketName, objectPrefix string) policyStatemen objectResourceStatement.Principal = map[string]interface{}{ "AWS": "*", } - objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}...) + objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...) objectResourceStatement.Actions = set.CreateStringSet(readWriteObjectActions...) return objectResourceStatement } @@ -91,7 +91,7 @@ func getReadWriteBucketStatement(bucketName, objectPrefix string) policyStatemen bucketResourceStatement.Principal = map[string]interface{}{ "AWS": "*", } - bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}...) + bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...) bucketResourceStatement.Actions = set.CreateStringSet(readWriteBucketActions...) return bucketResourceStatement } @@ -111,7 +111,7 @@ func getReadOnlyBucketStatement(bucketName, objectPrefix string) policyStatement bucketResourceStatement.Principal = map[string]interface{}{ "AWS": "*", } - bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}...) + bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...) bucketResourceStatement.Actions = set.CreateStringSet(readOnlyBucketActions...) return bucketResourceStatement } @@ -123,7 +123,7 @@ func getReadOnlyObjectStatement(bucketName, objectPrefix string) policyStatement objectResourceStatement.Principal = map[string]interface{}{ "AWS": "*", } - objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}...) + objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...) objectResourceStatement.Actions = set.CreateStringSet(readOnlyObjectActions...) return objectResourceStatement } @@ -144,7 +144,7 @@ func getWriteOnlyBucketStatement(bucketName, objectPrefix string) policyStatemen bucketResourceStatement.Principal = map[string]interface{}{ "AWS": "*", } - bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}...) + bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...) bucketResourceStatement.Actions = set.CreateStringSet(writeOnlyBucketActions...) return bucketResourceStatement } @@ -156,7 +156,7 @@ func getWriteOnlyObjectStatement(bucketName, objectPrefix string) policyStatemen objectResourceStatement.Principal = map[string]interface{}{ "AWS": "*", } - objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}...) + objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...) objectResourceStatement.Actions = set.CreateStringSet(writeOnlyObjectActions...) return objectResourceStatement } @@ -269,19 +269,19 @@ func TestIsValidResources(t *testing.T) { // Empty Resources. {[]string{}, errors.New("Resource list cannot be empty"), false}, // Test case - 2. - // A valid resource should have prefix "arn:aws:s3:::". + // A valid resource should have prefix bucketARNPrefix. {[]string{"my-resource"}, errors.New("Unsupported resource style found: ‘my-resource’, please validate your policy document"), false}, // Test case - 3. - // A Valid resource should have bucket name followed by "arn:aws:s3:::". - {[]string{"arn:aws:s3:::"}, errors.New("Invalid resource style found: ‘arn:aws:s3:::’, please validate your policy document"), false}, + // A Valid resource should have bucket name followed by bucketARNPrefix. + {[]string{bucketARNPrefix}, errors.New("Invalid resource style found: ‘arn:aws:s3:::’, please validate your policy document"), false}, // Test Case - 4. - // Valid resource shouldn't have slash('/') followed by "arn:aws:s3:::". - {[]string{"arn:aws:s3:::/"}, errors.New("Invalid resource style found: ‘arn:aws:s3:::/’, please validate your policy document"), false}, + // Valid resource shouldn't have slash('/') followed by bucketARNPrefix. + {[]string{bucketARNPrefix + "/"}, errors.New("Invalid resource style found: ‘arn:aws:s3:::/’, please validate your policy document"), false}, // Test cases with valid Resources. - {[]string{"arn:aws:s3:::my-bucket"}, nil, true}, - {[]string{"arn:aws:s3:::my-bucket/Asia/*"}, nil, true}, - {[]string{"arn:aws:s3:::my-bucket/Asia/India/*"}, nil, true}, + {[]string{bucketARNPrefix + "my-bucket"}, nil, true}, + {[]string{bucketARNPrefix + "my-bucket/Asia/*"}, nil, true}, + {[]string{bucketARNPrefix + "my-bucket/Asia/India/*"}, nil, true}, } for i, testCase := range testCases { err := isValidResources(set.CreateStringSet(testCase.resources...)) diff --git a/cmd/bucket-policy.go b/cmd/bucket-policy.go index 0f12aee62..9619e1c05 100644 --- a/cmd/bucket-policy.go +++ b/cmd/bucket-policy.go @@ -24,6 +24,12 @@ import ( "sync" ) +const ( + // Static prefix to be used while constructing bucket ARN. + // refer to S3 docs for more info. + bucketARNPrefix = "arn:" + eventSource + ":::" +) + // Variable represents bucket policies in memory. var globalBucketPolicies *bucketPolicies diff --git a/cmd/certs.go b/cmd/certs.go index 31896afd1..ce24bf37f 100644 --- a/cmd/certs.go +++ b/cmd/certs.go @@ -145,3 +145,22 @@ func parseCertificateChain(bytes []byte) ([]*x509.Certificate, error) { } return certs, nil } + +// loadRootCAs fetches CA files provided in minio config and adds them to globalRootCAs +// Currently under Windows, there is no way to load system + user CAs at the same time +func loadRootCAs() { + caFiles := mustGetCAFiles() + if len(caFiles) == 0 { + return + } + // Get system cert pool, and empty cert pool under Windows because it is not supported + globalRootCAs = mustGetSystemCertPool() + // Load custom root CAs for client requests + for _, caFile := range caFiles { + caCert, err := ioutil.ReadFile(caFile) + if err != nil { + fatalIf(err, "Unable to load a CA file") + } + globalRootCAs.AppendCertsFromPEM(caCert) + } +} diff --git a/cmd/event-notifier.go b/cmd/event-notifier.go index 1a1e63c98..85c600242 100644 --- a/cmd/event-notifier.go +++ b/cmd/event-notifier.go @@ -88,49 +88,76 @@ type eventData struct { // New notification event constructs a new notification event message from // input request metadata which completed successfully. func newNotificationEvent(event eventData) NotificationEvent { - /// Construct a new object created event. + // Fetch the region. region := serverConfig.GetRegion() - tnow := time.Now().UTC() - sequencer := fmt.Sprintf("%X", tnow.UnixNano()) + + // Fetch the credentials. + creds := serverConfig.GetCredential() + + // Time when Minio finished processing the request. + eventTime := time.Now().UTC() + + // API endpoint is captured here to be returned back + // to the client for it to differentiate from which + // server the request came from. + var apiEndpoint string + if len(globalAPIEndpoints) >= 1 { + apiEndpoint = globalAPIEndpoints[0] + } + + // Fetch a hexadecimal representation of event time in nano seconds. + uniqueID := mustGetRequestID(eventTime) + + /// Construct a new object created event. + // Following blocks fills in all the necessary details of s3 // event message structure. // http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html nEvent := NotificationEvent{ - EventVersion: "2.0", - EventSource: "aws:s3", + EventVersion: eventVersion, + EventSource: eventSource, AwsRegion: region, - EventTime: tnow.Format(timeFormatAMZ), + EventTime: eventTime.Format(timeFormatAMZ), EventName: event.Type.String(), - UserIdentity: defaultIdentity(), + UserIdentity: identity{creds.AccessKey}, RequestParameters: event.ReqParams, - ResponseElements: map[string]string{}, + ResponseElements: map[string]string{ + responseRequestIDKey: uniqueID, + // Following is a custom response element to indicate + // event origin server endpoint. + responseOriginEndpointKey: apiEndpoint, + }, S3: eventMeta{ - SchemaVersion: "1.0", - ConfigurationID: "Config", + SchemaVersion: eventSchemaVersion, + ConfigurationID: eventConfigID, Bucket: bucketMeta{ Name: event.Bucket, - OwnerIdentity: defaultIdentity(), - ARN: "arn:aws:s3:::" + event.Bucket, + OwnerIdentity: identity{creds.AccessKey}, + ARN: bucketARNPrefix + event.Bucket, }, }, } + // Escape the object name. For example "red flower.jpg" becomes "red+flower.jpg". escapedObj := url.QueryEscape(event.ObjInfo.Name) + // For delete object event type, we do not need to set ETag and Size. if event.Type == ObjectRemovedDelete { nEvent.S3.Object = objectMeta{ Key: escapedObj, - Sequencer: sequencer, + Sequencer: uniqueID, } return nEvent } + // For all other events we should set ETag and Size. nEvent.S3.Object = objectMeta{ Key: escapedObj, ETag: event.ObjInfo.MD5Sum, Size: event.ObjInfo.Size, - Sequencer: sequencer, + Sequencer: uniqueID, } + // Success. return nEvent } diff --git a/cmd/globals.go b/cmd/globals.go index 3eadd1902..9f31944eb 100644 --- a/cmd/globals.go +++ b/cmd/globals.go @@ -73,6 +73,7 @@ var ( // Cache expiry. globalCacheExpiry = objcache.DefaultExpiry + // Minio local server address (in `host:port` format) globalMinioAddr = "" // Minio default port, can be changed through command line. @@ -80,6 +81,9 @@ var ( // Holds the host that was passed using --address globalMinioHost = "" + // Holds the list of API endpoints for a given server. + globalAPIEndpoints = []string{} + // Peer communication struct globalS3Peers = s3Peers{} diff --git a/cmd/lock-instrument.go b/cmd/lock-instrument.go index 0b5fea730..b11dd81e5 100644 --- a/cmd/lock-instrument.go +++ b/cmd/lock-instrument.go @@ -17,6 +17,7 @@ package cmd import ( + "crypto/rand" "fmt" "time" ) @@ -225,5 +226,13 @@ func (n *nsLockMap) deleteLockInfoEntryForOps(param nsParam, opsID string) error // Return randomly generated string ID func getOpsID() string { - return mustGetRequestID() + const opsIDLen = 16 + opsIDBytes := make([]byte, opsIDLen) + if _, err := rand.Read(opsIDBytes); err != nil { + panic(err) + } + for i := 0; i < opsIDLen; i++ { + opsIDBytes[i] = alphaNumericTable[opsIDBytes[i]%alphaNumericTableLen] + } + return string(opsIDBytes) } diff --git a/cmd/lock-instrument_test.go b/cmd/lock-instrument_test.go index 503dc7876..d77030559 100644 --- a/cmd/lock-instrument_test.go +++ b/cmd/lock-instrument_test.go @@ -228,6 +228,25 @@ func verifyLockState(l lockStateCase, t *testing.T, testNum int) { verifyLockStats(l, t, testNum) } +func TestGetOpsID(t *testing.T) { + // Ensure that it returns an alphanumeric result of length 16. + var id = getOpsID() + + if len(id) != 16 { + t.Fail() + } + + var e rune + for _, char := range id { + e = rune(char) + + // Ensure that it is alphanumeric, in this case, between 0-9 and A-Z. + if !(('0' <= e && e <= '9') || ('A' <= e && e <= 'Z')) { + t.Fail() + } + } +} + // TestNewDebugLockInfoPerVolumePath - Validates the values initialized by newDebugLockInfoPerVolumePath(). func TestNewDebugLockInfoPerVolumePath(t *testing.T) { lockInfo := &debugLockInfoPerVolumePath{ diff --git a/cmd/server-main.go b/cmd/server-main.go index 23ba3cac1..36436ead2 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -18,9 +18,7 @@ package cmd import ( "fmt" - "io/ioutil" "net" - "net/http" "net/url" "os" "path" @@ -129,69 +127,6 @@ func parseStorageEndpoints(eps []string) (endpoints []*url.URL, err error) { return endpoints, nil } -// getListenIPs - gets all the ips to listen on. -func getListenIPs(serverAddr string) (hosts []string, port string, err error) { - var host string - host, port, err = net.SplitHostPort(serverAddr) - if err != nil { - return nil, port, fmt.Errorf("Unable to parse host address %s", err) - } - if host == "" { - var ipv4s []net.IP - ipv4s, err = getInterfaceIPv4s() - if err != nil { - return nil, port, fmt.Errorf("Unable reverse sorted ips from hosts %s", err) - } - for _, ip := range ipv4s { - hosts = append(hosts, ip.String()) - } - return hosts, port, nil - } // if host != "" { - // Proceed to append itself, since user requested a specific endpoint. - hosts = append(hosts, host) - return hosts, port, nil -} - -// Finalizes the endpoints based on the host list and port. -func finalizeEndpoints(tls bool, apiServer *http.Server) (endPoints []string) { - // Verify current scheme. - scheme := "http" - if tls { - scheme = "https" - } - - // Get list of listen ips and port. - hosts, port, err := getListenIPs(apiServer.Addr) - fatalIf(err, "Unable to get list of ips to listen on") - - // Construct proper endpoints. - for _, host := range hosts { - endPoints = append(endPoints, fmt.Sprintf("%s://%s:%s", scheme, host, port)) - } - - // Success. - return endPoints -} - -// loadRootCAs fetches CA files provided in minio config and adds them to globalRootCAs -// Currently under Windows, there is no way to load system + user CAs at the same time -func loadRootCAs() { - caFiles := mustGetCAFiles() - if len(caFiles) == 0 { - return - } - // Get system cert pool, and empty cert pool under Windows because it is not supported - globalRootCAs = mustGetSystemCertPool() - // Load custom root CAs for client requests - for _, caFile := range mustGetCAFiles() { - caCert, err := ioutil.ReadFile(caFile) - if err != nil { - fatalIf(err, "Unable to load a CA file") - } - globalRootCAs.AppendCertsFromPEM(caCert) - } -} - // initServerConfig initialize server config. func initServerConfig(c *cli.Context) { // Create certs path. @@ -430,6 +365,9 @@ func serverMain(c *cli.Context) { fatalIf(errInvalidArgument, "None of the disks passed as command line args are local to this server.") } + // Is TLS configured?. + tls := isSSL() + // Sort endpoints for consistent ordering across multiple // nodes in a distributed setup. This is to avoid format.json // corruption if the disks aren't supplied in the same order @@ -473,15 +411,15 @@ func serverMain(c *cli.Context) { // Initialize a new HTTP server. apiServer := NewServerMux(serverAddr, handler) - // If https. - tls := isSSL() - - // Fetch endpoints which we are going to serve from. - endPoints := finalizeEndpoints(tls, apiServer.Server) - - // Initialize local server address + // Set the global minio addr for this server. globalMinioAddr = getLocalAddress(srvConfig) + // Determine API endpoints where we are going to serve the S3 API from. + apiEndPoints := finalizeAPIEndpoints(tls, apiServer.Server) + + // Set the global API endpoints value. + globalAPIEndpoints = apiEndPoints + // Initialize S3 Peers inter-node communication initGlobalS3Peers(endpoints) @@ -512,7 +450,7 @@ func serverMain(c *cli.Context) { globalObjLayerMutex.Unlock() // Prints the formatted startup message once object layer is initialized. - printStartupMessage(endPoints) + printStartupMessage(apiEndPoints) // Waits on the server. <-globalServiceDoneCh diff --git a/cmd/server-main_test.go b/cmd/server-main_test.go index d666d31c9..3c64d6880 100644 --- a/cmd/server-main_test.go +++ b/cmd/server-main_test.go @@ -63,7 +63,7 @@ func TestGetListenIPs(t *testing.T) { } } -func TestFinalizeEndpoints(t *testing.T) { +func TestFinalizeAPIEndpoints(t *testing.T) { testCases := []struct { tls bool addr string @@ -75,7 +75,7 @@ func TestFinalizeEndpoints(t *testing.T) { } for i, test := range testCases { - endPoints := finalizeEndpoints(test.tls, &http.Server{Addr: test.addr}) + endPoints := finalizeAPIEndpoints(test.tls, &http.Server{Addr: test.addr}) if len(endPoints) <= 0 { t.Errorf("Test case %d returned with no API end points for %s", i+1, test.addr) diff --git a/cmd/server-startup-msg.go b/cmd/server-startup-msg.go index 720ede6c7..af2fbae25 100644 --- a/cmd/server-startup-msg.go +++ b/cmd/server-startup-msg.go @@ -1,5 +1,5 @@ /* - * Minio Cloud Storage, (C) 2016 Minio, Inc. + * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,20 +43,30 @@ func getFormatStr(strLen int, padding int) string { } // Prints the formatted startup message. -func printStartupMessage(endPoints []string) { +func printStartupMessage(apiEndPoints []string) { // If quiet flag is set do not print startup message. if globalQuiet { return } - printServerCommonMsg(endPoints) - printCLIAccessMsg(endPoints[0]) + + // Prints credential, region and browser access. + printServerCommonMsg(apiEndPoints) + + // Prints `mc` cli configuration message chooses + // first endpoint as default. + printCLIAccessMsg(apiEndPoints[0]) + + // Prints documentation message. printObjectAPIMsg() + // Object layer is initialized then print StorageInfo. objAPI := newObjectLayerFn() if objAPI != nil { printStorageInfo(objAPI.StorageInfo()) } + // SSL is configured reads certification chain, prints + // authority and expiry. if isSSL() { certs, err := readCertificateChain() fatalIf(err, "Unable to read certificate chain.") @@ -65,23 +75,23 @@ func printStartupMessage(endPoints []string) { } // Prints common server startup message. Prints credential, region and browser access. -func printServerCommonMsg(endPoints []string) { +func printServerCommonMsg(apiEndpoints []string) { // Get saved credentials. cred := serverConfig.GetCredential() // Get saved region. region := serverConfig.GetRegion() - endPointStr := strings.Join(endPoints, " ") + apiEndpointStr := strings.Join(apiEndpoints, " ") // Colorize the message and print. - console.Println(colorBlue("\nEndpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(endPointStr), 1), endPointStr))) + console.Println(colorBlue("\nEndpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr))) console.Println(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey))) console.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey))) console.Println(colorBlue("Region: ") + colorBold(fmt.Sprintf(getFormatStr(len(region), 3), region))) printEventNotifiers() console.Println(colorBlue("\nBrowser Access:")) - console.Println(fmt.Sprintf(getFormatStr(len(endPointStr), 3), endPointStr)) + console.Println(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 3), apiEndpointStr)) } // Prints bucket notification configurations. diff --git a/cmd/server-startup-msg_test.go b/cmd/server-startup-msg_test.go index 4a291546d..1a219858c 100644 --- a/cmd/server-startup-msg_test.go +++ b/cmd/server-startup-msg_test.go @@ -94,3 +94,39 @@ func TestCertificateNotExpired(t *testing.T) { t.Fatalf("Expected empty message was: %s", msg) } } + +// Test printing server common message. +func TestPrintServerCommonMessage(t *testing.T) { + root, err := newTestConfig("us-east-1") + if err != nil { + t.Fatal(err) + } + defer removeAll(root) + + apiEndpoints := []string{"127.0.0.1:9000"} + printServerCommonMsg(apiEndpoints) +} + +// Tests print cli access message. +func TestPrintCLIAccessMsg(t *testing.T) { + root, err := newTestConfig("us-east-1") + if err != nil { + t.Fatal(err) + } + defer removeAll(root) + + apiEndpoints := []string{"127.0.0.1:9000"} + printCLIAccessMsg(apiEndpoints[0]) +} + +// Test print startup message. +func TestPrintStartupMessage(t *testing.T) { + root, err := newTestConfig("us-east-1") + if err != nil { + t.Fatal(err) + } + defer removeAll(root) + + apiEndpoints := []string{"127.0.0.1:9000"} + printStartupMessage(apiEndpoints) +} diff --git a/cmd/server-startup-utils.go b/cmd/server-startup-utils.go new file mode 100644 index 000000000..fa884f934 --- /dev/null +++ b/cmd/server-startup-utils.go @@ -0,0 +1,67 @@ +/* + * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "fmt" + "net" + "net/http" +) + +// getListenIPs - gets all the ips to listen on. +func getListenIPs(serverAddr string) (hosts []string, port string, err error) { + var host string + host, port, err = net.SplitHostPort(serverAddr) + if err != nil { + return nil, port, fmt.Errorf("Unable to parse host address %s", err) + } + if host == "" { + var ipv4s []net.IP + ipv4s, err = getInterfaceIPv4s() + if err != nil { + return nil, port, fmt.Errorf("Unable reverse sort ips from hosts %s", err) + } + for _, ip := range ipv4s { + hosts = append(hosts, ip.String()) + } + return hosts, port, nil + } // if host != "" { + // Proceed to append itself, since user requested a specific endpoint. + hosts = append(hosts, host) + return hosts, port, nil +} + +// Finalizes the API endpoints based on the host list and port. +func finalizeAPIEndpoints(tls bool, apiServer *http.Server) (endPoints []string) { + // Verify current scheme. + scheme := "http" + if tls { + scheme = "https" + } + + // Get list of listen ips and port. + hosts, port, err := getListenIPs(apiServer.Addr) + fatalIf(err, "Unable to get list of ips to listen on") + + // Construct proper endpoints. + for _, host := range hosts { + endPoints = append(endPoints, fmt.Sprintf("%s://%s:%s", scheme, host, port)) + } + + // Success. + return endPoints +} diff --git a/cmd/web-handlers_test.go b/cmd/web-handlers_test.go index 057b28999..e793d202c 100644 --- a/cmd/web-handlers_test.go +++ b/cmd/web-handlers_test.go @@ -948,14 +948,14 @@ func testWebGetBucketPolicyHandler(obj ObjectLayer, instanceType string, t TestE Actions: set.CreateStringSet("s3:GetBucketLocation", "s3:ListBucket"), Effect: "Allow", Principal: map[string][]string{"AWS": {"*"}}, - Resources: set.CreateStringSet("arn:aws:s3:::" + bucketName), + Resources: set.CreateStringSet(bucketARNPrefix + bucketName), Sid: "", }, { Actions: set.CreateStringSet("s3:GetObject"), Effect: "Allow", Principal: map[string][]string{"AWS": {"*"}}, - Resources: set.CreateStringSet("arn:aws:s3:::" + bucketName + "/*"), + Resources: set.CreateStringSet(bucketARNPrefix + bucketName + "/*"), Sid: "", }, }, @@ -1031,7 +1031,7 @@ func testWebListAllBucketPoliciesHandler(obj ObjectLayer, instanceType string, t Actions: set.CreateStringSet("s3:GetBucketLocation"), Effect: "Allow", Principal: map[string][]string{"AWS": {"*"}}, - Resources: set.CreateStringSet("arn:aws:s3:::" + bucketName), + Resources: set.CreateStringSet(bucketARNPrefix + bucketName), Sid: "", }, { @@ -1043,14 +1043,14 @@ func testWebListAllBucketPoliciesHandler(obj ObjectLayer, instanceType string, t }, Effect: "Allow", Principal: map[string][]string{"AWS": {"*"}}, - Resources: set.CreateStringSet("arn:aws:s3:::" + bucketName), + Resources: set.CreateStringSet(bucketARNPrefix + bucketName), Sid: "", }, { Actions: set.CreateStringSet("s3:ListBucketMultipartUploads"), Effect: "Allow", Principal: map[string][]string{"AWS": {"*"}}, - Resources: set.CreateStringSet("arn:aws:s3:::" + bucketName), + Resources: set.CreateStringSet(bucketARNPrefix + bucketName), Sid: "", }, { @@ -1058,7 +1058,7 @@ func testWebListAllBucketPoliciesHandler(obj ObjectLayer, instanceType string, t "s3:GetObject", "s3:ListMultipartUploadParts", "s3:PutObject"), Effect: "Allow", Principal: map[string][]string{"AWS": {"*"}}, - Resources: set.CreateStringSet("arn:aws:s3:::" + bucketName + "/hello*"), + Resources: set.CreateStringSet(bucketARNPrefix + bucketName + "/hello*"), Sid: "", }, },