mirror of
https://github.com/minio/minio.git
synced 2025-11-07 21:02:58 -05:00
feat: add lambda transformation functions target (#16507)
This commit is contained in:
@@ -137,6 +137,11 @@ const (
|
||||
// Add new constants here (similar to above) if you add new fields to config.
|
||||
)
|
||||
|
||||
// Lambda config constants.
|
||||
const (
|
||||
LambdaWebhookSubSys = madmin.LambdaWebhookSubSys
|
||||
)
|
||||
|
||||
// NotifySubSystems - all notification sub-systems
|
||||
var NotifySubSystems = set.CreateStringSet(
|
||||
NotifyKafkaSubSys,
|
||||
@@ -151,6 +156,11 @@ var NotifySubSystems = set.CreateStringSet(
|
||||
NotifyWebhookSubSys,
|
||||
)
|
||||
|
||||
// LambdaSubSystems - all lambda sub-systesm
|
||||
var LambdaSubSystems = set.CreateStringSet(
|
||||
LambdaWebhookSubSys,
|
||||
)
|
||||
|
||||
// LoggerSubSystems - all sub-systems related to logger
|
||||
var LoggerSubSystems = set.CreateStringSet(
|
||||
LoggerWebhookSubSys,
|
||||
|
||||
40
internal/config/lambda/config.go
Normal file
40
internal/config/lambda/config.go
Normal file
@@ -0,0 +1,40 @@
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package lambda
|
||||
|
||||
import "github.com/minio/minio/internal/event/target"
|
||||
|
||||
// Config - lambda target configuration structure, holds
|
||||
// information about various lambda targets.
|
||||
type Config struct {
|
||||
Webhook map[string]target.WebhookArgs `json:"webhook"`
|
||||
}
|
||||
|
||||
const (
|
||||
defaultTarget = "1"
|
||||
)
|
||||
|
||||
// NewConfig - initialize lambda config.
|
||||
func NewConfig() Config {
|
||||
// Make sure to initialize lambda targets
|
||||
cfg := Config{
|
||||
Webhook: make(map[string]target.WebhookArgs),
|
||||
}
|
||||
cfg.Webhook[defaultTarget] = target.WebhookArgs{}
|
||||
return cfg
|
||||
}
|
||||
62
internal/config/lambda/event/arn.go
Normal file
62
internal/config/lambda/event/arn.go
Normal file
@@ -0,0 +1,62 @@
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package event
|
||||
|
||||
import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ARN - SQS resource name representation.
|
||||
type ARN struct {
|
||||
TargetID
|
||||
region string
|
||||
}
|
||||
|
||||
// String - returns string representation.
|
||||
func (arn ARN) String() string {
|
||||
if arn.TargetID.ID == "" && arn.TargetID.Name == "" && arn.region == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
return "arn:minio:s3-object-lambda:" + arn.region + ":" + arn.TargetID.String()
|
||||
}
|
||||
|
||||
// ParseARN - parses string to ARN.
|
||||
func ParseARN(s string) (*ARN, error) {
|
||||
// ARN must be in the format of arn:minio:s3-object-lambda:<REGION>:<ID>:<TYPE>
|
||||
if !strings.HasPrefix(s, "arn:minio:s3-object-lambda:") {
|
||||
return nil, &ErrInvalidARN{s}
|
||||
}
|
||||
|
||||
tokens := strings.Split(s, ":")
|
||||
if len(tokens) != 6 {
|
||||
return nil, &ErrInvalidARN{s}
|
||||
}
|
||||
|
||||
if tokens[4] == "" || tokens[5] == "" {
|
||||
return nil, &ErrInvalidARN{s}
|
||||
}
|
||||
|
||||
return &ARN{
|
||||
region: tokens[3],
|
||||
TargetID: TargetID{
|
||||
ID: tokens[4],
|
||||
Name: tokens[5],
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
72
internal/config/lambda/event/arn_test.go
Normal file
72
internal/config/lambda/event/arn_test.go
Normal file
@@ -0,0 +1,72 @@
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package event
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestARNString(t *testing.T) {
|
||||
testCases := []struct {
|
||||
arn ARN
|
||||
expectedResult string
|
||||
}{
|
||||
{ARN{}, ""},
|
||||
{ARN{TargetID{"1", "webhook"}, ""}, "arn:minio:s3-object-lambda::1:webhook"},
|
||||
{ARN{TargetID{"1", "webhook"}, "us-east-1"}, "arn:minio:s3-object-lambda:us-east-1:1:webhook"},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
result := testCase.arn.String()
|
||||
|
||||
if result != testCase.expectedResult {
|
||||
t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseARN(t *testing.T) {
|
||||
testCases := []struct {
|
||||
s string
|
||||
expectedARN *ARN
|
||||
expectErr bool
|
||||
}{
|
||||
{"", nil, true},
|
||||
{"arn:minio:s3-object-lambda:::", nil, true},
|
||||
{"arn:minio:s3-object-lambda::1:webhook:remote", nil, true},
|
||||
{"arn:aws:s3-object-lambda::1:webhook", nil, true},
|
||||
{"arn:minio:sns::1:webhook", nil, true},
|
||||
{"arn:minio:s3-object-lambda::1:webhook", &ARN{TargetID{"1", "webhook"}, ""}, false},
|
||||
{"arn:minio:s3-object-lambda:us-east-1:1:webhook", &ARN{TargetID{"1", "webhook"}, "us-east-1"}, false},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
arn, err := ParseARN(testCase.s)
|
||||
expectErr := (err != nil)
|
||||
|
||||
if expectErr != testCase.expectErr {
|
||||
t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
|
||||
}
|
||||
|
||||
if !testCase.expectErr {
|
||||
if *arn != *testCase.expectedARN {
|
||||
t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedARN, arn)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
49
internal/config/lambda/event/errors.go
Normal file
49
internal/config/lambda/event/errors.go
Normal file
@@ -0,0 +1,49 @@
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package event
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// ErrUnknownRegion - unknown region error.
|
||||
type ErrUnknownRegion struct {
|
||||
Region string
|
||||
}
|
||||
|
||||
func (err ErrUnknownRegion) Error() string {
|
||||
return fmt.Sprintf("unknown region '%v'", err.Region)
|
||||
}
|
||||
|
||||
// ErrARNNotFound - ARN not found error.
|
||||
type ErrARNNotFound struct {
|
||||
ARN ARN
|
||||
}
|
||||
|
||||
func (err ErrARNNotFound) Error() string {
|
||||
return fmt.Sprintf("ARN '%v' not found", err.ARN)
|
||||
}
|
||||
|
||||
// ErrInvalidARN - invalid ARN error.
|
||||
type ErrInvalidARN struct {
|
||||
ARN string
|
||||
}
|
||||
|
||||
func (err ErrInvalidARN) Error() string {
|
||||
return fmt.Sprintf("invalid ARN '%v'", err.ARN)
|
||||
}
|
||||
80
internal/config/lambda/event/event.go
Normal file
80
internal/config/lambda/event/event.go
Normal file
@@ -0,0 +1,80 @@
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package event
|
||||
|
||||
import "net/http"
|
||||
|
||||
// Identity represents access key who caused the event.
|
||||
type Identity struct {
|
||||
Type string `json:"type"`
|
||||
PrincipalID string `json:"principalId"`
|
||||
AccessKeyID string `json:"accessKeyId"`
|
||||
}
|
||||
|
||||
// UserRequest user request headers
|
||||
type UserRequest struct {
|
||||
URL string `json:"url"`
|
||||
Headers http.Header `json:"headers"`
|
||||
}
|
||||
|
||||
// GetObjectContext provides the necessary details to perform
|
||||
// download of the object, and return back the processed response
|
||||
// to the server.
|
||||
type GetObjectContext struct {
|
||||
OutputRoute string `json:"outputRoute"`
|
||||
OutputToken string `json:"outputToken"`
|
||||
InputS3URL string `json:"inputS3Url"`
|
||||
}
|
||||
|
||||
// Event represents lambda function event, this is undocumented in AWS S3. This
|
||||
// structure bases itself on this structure but there is no binding.
|
||||
//
|
||||
// {
|
||||
// "xAmzRequestId": "a2871150-1df5-4dc9-ad9f-3da283ca1bf3",
|
||||
// "getObjectContext": {
|
||||
// "outputRoute": "...",
|
||||
// "outputToken": "...",
|
||||
// "inputS3Url": "<presignedURL>"
|
||||
// },
|
||||
// "configuration": { // not useful in MinIO
|
||||
// "accessPointArn": "...",
|
||||
// "supportingAccessPointArn": "...",
|
||||
// "payload": ""
|
||||
// },
|
||||
// "userRequest": {
|
||||
// "url": "...",
|
||||
// "headers": {
|
||||
// "Host": "...",
|
||||
// "X-Amz-Content-SHA256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
// }
|
||||
// },
|
||||
// "userIdentity": {
|
||||
// "type": "IAMUser",
|
||||
// "principalId": "AIDAJF5MO57RFXQCE5ZNC",
|
||||
// "arn": "...",
|
||||
// "accountId": "...",
|
||||
// "accessKeyId": "AKIA3WNQJCXE2DYPAU7R"
|
||||
// },
|
||||
// "protocolVersion": "1.00"
|
||||
// }
|
||||
type Event struct {
|
||||
ProtocolVersion string `json:"protocolVersion"`
|
||||
GetObjectContext *GetObjectContext `json:"getObjectContext"`
|
||||
UserIdentity Identity `json:"userIdentity"`
|
||||
UserRequest UserRequest `json:"userRequest"`
|
||||
}
|
||||
74
internal/config/lambda/event/targetid.go
Normal file
74
internal/config/lambda/event/targetid.go
Normal file
@@ -0,0 +1,74 @@
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package event
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// TargetID - holds identification and name strings of notification target.
|
||||
type TargetID struct {
|
||||
ID string
|
||||
Name string
|
||||
}
|
||||
|
||||
// String - returns string representation.
|
||||
func (tid TargetID) String() string {
|
||||
return tid.ID + ":" + tid.Name
|
||||
}
|
||||
|
||||
// ToARN - converts to ARN.
|
||||
func (tid TargetID) ToARN(region string) ARN {
|
||||
return ARN{TargetID: tid, region: region}
|
||||
}
|
||||
|
||||
// MarshalJSON - encodes to JSON data.
|
||||
func (tid TargetID) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal(tid.String())
|
||||
}
|
||||
|
||||
// UnmarshalJSON - decodes JSON data.
|
||||
func (tid *TargetID) UnmarshalJSON(data []byte) error {
|
||||
var s string
|
||||
if err := json.Unmarshal(data, &s); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
targetID, err := parseTargetID(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
*tid = *targetID
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseTargetID - parses string to TargetID.
|
||||
func parseTargetID(s string) (*TargetID, error) {
|
||||
tokens := strings.Split(s, ":")
|
||||
if len(tokens) != 2 {
|
||||
return nil, fmt.Errorf("invalid TargetID format '%v'", s)
|
||||
}
|
||||
|
||||
return &TargetID{
|
||||
ID: tokens[0],
|
||||
Name: tokens[1],
|
||||
}, nil
|
||||
}
|
||||
118
internal/config/lambda/event/targetid_test.go
Normal file
118
internal/config/lambda/event/targetid_test.go
Normal file
@@ -0,0 +1,118 @@
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package event
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTargetDString(t *testing.T) {
|
||||
testCases := []struct {
|
||||
tid TargetID
|
||||
expectedResult string
|
||||
}{
|
||||
{TargetID{}, ":"},
|
||||
{TargetID{"1", "webhook"}, "1:webhook"},
|
||||
{TargetID{"httpclient+2e33cdee-fbec-4bdd-917e-7d8e3c5a2531", "localhost:55638"}, "httpclient+2e33cdee-fbec-4bdd-917e-7d8e3c5a2531:localhost:55638"},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
result := testCase.tid.String()
|
||||
|
||||
if result != testCase.expectedResult {
|
||||
t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTargetDToARN(t *testing.T) {
|
||||
tid := TargetID{"1", "webhook"}
|
||||
testCases := []struct {
|
||||
tid TargetID
|
||||
region string
|
||||
expectedARN ARN
|
||||
}{
|
||||
{tid, "", ARN{TargetID: tid, region: ""}},
|
||||
{tid, "us-east-1", ARN{TargetID: tid, region: "us-east-1"}},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
arn := testCase.tid.ToARN(testCase.region)
|
||||
|
||||
if arn != testCase.expectedARN {
|
||||
t.Fatalf("test %v: ARN: expected: %v, got: %v", i+1, testCase.expectedARN, arn)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTargetDMarshalJSON(t *testing.T) {
|
||||
testCases := []struct {
|
||||
tid TargetID
|
||||
expectedData []byte
|
||||
expectErr bool
|
||||
}{
|
||||
{TargetID{}, []byte(`":"`), false},
|
||||
{TargetID{"1", "webhook"}, []byte(`"1:webhook"`), false},
|
||||
{TargetID{"httpclient+2e33cdee-fbec-4bdd-917e-7d8e3c5a2531", "localhost:55638"}, []byte(`"httpclient+2e33cdee-fbec-4bdd-917e-7d8e3c5a2531:localhost:55638"`), false},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
data, err := testCase.tid.MarshalJSON()
|
||||
expectErr := (err != nil)
|
||||
|
||||
if expectErr != testCase.expectErr {
|
||||
t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
|
||||
}
|
||||
|
||||
if !testCase.expectErr {
|
||||
if !reflect.DeepEqual(data, testCase.expectedData) {
|
||||
t.Fatalf("test %v: data: expected: %v, got: %v", i+1, string(testCase.expectedData), string(data))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTargetDUnmarshalJSON(t *testing.T) {
|
||||
testCases := []struct {
|
||||
data []byte
|
||||
expectedTargetID *TargetID
|
||||
expectErr bool
|
||||
}{
|
||||
{[]byte(`""`), nil, true},
|
||||
{[]byte(`"httpclient+2e33cdee-fbec-4bdd-917e-7d8e3c5a2531:localhost:55638"`), nil, true},
|
||||
{[]byte(`":"`), &TargetID{}, false},
|
||||
{[]byte(`"1:webhook"`), &TargetID{"1", "webhook"}, false},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
targetID := &TargetID{}
|
||||
err := targetID.UnmarshalJSON(testCase.data)
|
||||
expectErr := (err != nil)
|
||||
|
||||
if expectErr != testCase.expectErr {
|
||||
t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
|
||||
}
|
||||
|
||||
if !testCase.expectErr {
|
||||
if *targetID != *testCase.expectedTargetID {
|
||||
t.Fatalf("test %v: TargetID: expected: %v, got: %v", i+1, testCase.expectedTargetID, targetID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
72
internal/config/lambda/event/targetidset.go
Normal file
72
internal/config/lambda/event/targetidset.go
Normal file
@@ -0,0 +1,72 @@
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package event
|
||||
|
||||
// TargetIDSet - Set representation of TargetIDs.
|
||||
type TargetIDSet map[TargetID]struct{}
|
||||
|
||||
// IsEmpty returns true if the set is empty.
|
||||
func (set TargetIDSet) IsEmpty() bool {
|
||||
return len(set) != 0
|
||||
}
|
||||
|
||||
// Clone - returns copy of this set.
|
||||
func (set TargetIDSet) Clone() TargetIDSet {
|
||||
setCopy := NewTargetIDSet()
|
||||
for k, v := range set {
|
||||
setCopy[k] = v
|
||||
}
|
||||
return setCopy
|
||||
}
|
||||
|
||||
// add - adds TargetID to the set.
|
||||
func (set TargetIDSet) add(targetID TargetID) {
|
||||
set[targetID] = struct{}{}
|
||||
}
|
||||
|
||||
// Union - returns union with given set as new set.
|
||||
func (set TargetIDSet) Union(sset TargetIDSet) TargetIDSet {
|
||||
nset := set.Clone()
|
||||
|
||||
for k := range sset {
|
||||
nset.add(k)
|
||||
}
|
||||
|
||||
return nset
|
||||
}
|
||||
|
||||
// Difference - returns diffrence with given set as new set.
|
||||
func (set TargetIDSet) Difference(sset TargetIDSet) TargetIDSet {
|
||||
nset := NewTargetIDSet()
|
||||
for k := range set {
|
||||
if _, ok := sset[k]; !ok {
|
||||
nset.add(k)
|
||||
}
|
||||
}
|
||||
|
||||
return nset
|
||||
}
|
||||
|
||||
// NewTargetIDSet - creates new TargetID set with given TargetIDs.
|
||||
func NewTargetIDSet(targetIDs ...TargetID) TargetIDSet {
|
||||
set := make(TargetIDSet)
|
||||
for _, targetID := range targetIDs {
|
||||
set.add(targetID)
|
||||
}
|
||||
return set
|
||||
}
|
||||
110
internal/config/lambda/event/targetidset_test.go
Normal file
110
internal/config/lambda/event/targetidset_test.go
Normal file
@@ -0,0 +1,110 @@
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package event
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTargetIDSetClone(t *testing.T) {
|
||||
testCases := []struct {
|
||||
set TargetIDSet
|
||||
targetIDToAdd TargetID
|
||||
}{
|
||||
{NewTargetIDSet(), TargetID{"1", "webhook"}},
|
||||
{NewTargetIDSet(TargetID{"1", "webhook"}), TargetID{"2", "webhook"}},
|
||||
{NewTargetIDSet(TargetID{"1", "webhook"}, TargetID{"2", "amqp"}), TargetID{"2", "webhook"}},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
result := testCase.set.Clone()
|
||||
|
||||
if !reflect.DeepEqual(result, testCase.set) {
|
||||
t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.set, result)
|
||||
}
|
||||
|
||||
result.add(testCase.targetIDToAdd)
|
||||
if reflect.DeepEqual(result, testCase.set) {
|
||||
t.Fatalf("test %v: result: expected: not equal, got: equal", i+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTargetIDSetUnion(t *testing.T) {
|
||||
testCases := []struct {
|
||||
set TargetIDSet
|
||||
setToAdd TargetIDSet
|
||||
expectedResult TargetIDSet
|
||||
}{
|
||||
{NewTargetIDSet(), NewTargetIDSet(), NewTargetIDSet()},
|
||||
{NewTargetIDSet(), NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"1", "webhook"})},
|
||||
{NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(), NewTargetIDSet(TargetID{"1", "webhook"})},
|
||||
{NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"2", "amqp"}), NewTargetIDSet(TargetID{"1", "webhook"}, TargetID{"2", "amqp"})},
|
||||
{NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"1", "webhook"})},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
result := testCase.set.Union(testCase.setToAdd)
|
||||
|
||||
if !reflect.DeepEqual(testCase.expectedResult, result) {
|
||||
t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTargetIDSetDifference(t *testing.T) {
|
||||
testCases := []struct {
|
||||
set TargetIDSet
|
||||
setToRemove TargetIDSet
|
||||
expectedResult TargetIDSet
|
||||
}{
|
||||
{NewTargetIDSet(), NewTargetIDSet(), NewTargetIDSet()},
|
||||
{NewTargetIDSet(), NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet()},
|
||||
{NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(), NewTargetIDSet(TargetID{"1", "webhook"})},
|
||||
{NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"2", "amqp"}), NewTargetIDSet(TargetID{"1", "webhook"})},
|
||||
{NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet()},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
result := testCase.set.Difference(testCase.setToRemove)
|
||||
|
||||
if !reflect.DeepEqual(testCase.expectedResult, result) {
|
||||
t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewTargetIDSet(t *testing.T) {
|
||||
testCases := []struct {
|
||||
targetIDs []TargetID
|
||||
expectedResult TargetIDSet
|
||||
}{
|
||||
{[]TargetID{}, NewTargetIDSet()},
|
||||
{[]TargetID{{"1", "webhook"}}, NewTargetIDSet(TargetID{"1", "webhook"})},
|
||||
{[]TargetID{{"1", "webhook"}, {"2", "amqp"}}, NewTargetIDSet(TargetID{"1", "webhook"}, TargetID{"2", "amqp"})},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
result := NewTargetIDSet(testCase.targetIDs...)
|
||||
|
||||
if !reflect.DeepEqual(testCase.expectedResult, result) {
|
||||
t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
189
internal/config/lambda/event/targetlist.go
Normal file
189
internal/config/lambda/event/targetlist.go
Normal file
@@ -0,0 +1,189 @@
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package event
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Target - lambda target interface
|
||||
type Target interface {
|
||||
ID() TargetID
|
||||
IsActive() (bool, error)
|
||||
Send(Event) (*http.Response, error)
|
||||
Stat() TargetStat
|
||||
Close() error
|
||||
}
|
||||
|
||||
// TargetStats is a collection of stats for multiple targets.
|
||||
type TargetStats struct {
|
||||
TargetStats map[string]TargetStat
|
||||
}
|
||||
|
||||
// TargetStat is the stats of a single target.
|
||||
type TargetStat struct {
|
||||
ID TargetID
|
||||
ActiveRequests int64
|
||||
TotalRequests int64
|
||||
FailedRequests int64
|
||||
}
|
||||
|
||||
// TargetList - holds list of targets indexed by target ID.
|
||||
type TargetList struct {
|
||||
sync.RWMutex
|
||||
targets map[TargetID]Target
|
||||
}
|
||||
|
||||
// Add - adds unique target to target list.
|
||||
func (list *TargetList) Add(targets ...Target) error {
|
||||
list.Lock()
|
||||
defer list.Unlock()
|
||||
|
||||
for _, target := range targets {
|
||||
if _, ok := list.targets[target.ID()]; ok {
|
||||
return fmt.Errorf("target %v already exists", target.ID())
|
||||
}
|
||||
list.targets[target.ID()] = target
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Lookup - checks whether target by target ID exists is valid or not.
|
||||
func (list *TargetList) Lookup(arnStr string) (Target, error) {
|
||||
list.RLock()
|
||||
defer list.RUnlock()
|
||||
|
||||
arn, err := ParseARN(arnStr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
id, found := list.targets[arn.TargetID]
|
||||
if !found {
|
||||
return nil, &ErrARNNotFound{}
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// TargetIDResult returns result of Remove/Send operation, sets err if
|
||||
// any for the associated TargetID
|
||||
type TargetIDResult struct {
|
||||
// ID where the remove or send were initiated.
|
||||
ID TargetID
|
||||
// Stores any error while removing a target or while sending an event.
|
||||
Err error
|
||||
}
|
||||
|
||||
// Remove - closes and removes targets by given target IDs.
|
||||
func (list *TargetList) Remove(targetIDSet TargetIDSet) {
|
||||
list.Lock()
|
||||
defer list.Unlock()
|
||||
|
||||
for id := range targetIDSet {
|
||||
target, ok := list.targets[id]
|
||||
if ok {
|
||||
target.Close()
|
||||
delete(list.targets, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Targets - list all targets
|
||||
func (list *TargetList) Targets() []Target {
|
||||
if list == nil {
|
||||
return []Target{}
|
||||
}
|
||||
|
||||
list.RLock()
|
||||
defer list.RUnlock()
|
||||
|
||||
targets := make([]Target, 0, len(list.targets))
|
||||
for _, tgt := range list.targets {
|
||||
targets = append(targets, tgt)
|
||||
}
|
||||
|
||||
return targets
|
||||
}
|
||||
|
||||
// Empty returns true if targetList is empty.
|
||||
func (list *TargetList) Empty() bool {
|
||||
list.RLock()
|
||||
defer list.RUnlock()
|
||||
|
||||
return len(list.targets) == 0
|
||||
}
|
||||
|
||||
// List - returns available target IDs.
|
||||
func (list *TargetList) List(region string) []ARN {
|
||||
list.RLock()
|
||||
defer list.RUnlock()
|
||||
|
||||
keys := make([]ARN, 0, len(list.targets))
|
||||
for k := range list.targets {
|
||||
keys = append(keys, k.ToARN(region))
|
||||
}
|
||||
|
||||
return keys
|
||||
}
|
||||
|
||||
// TargetMap - returns available targets.
|
||||
func (list *TargetList) TargetMap() map[TargetID]Target {
|
||||
list.RLock()
|
||||
defer list.RUnlock()
|
||||
|
||||
ntargets := make(map[TargetID]Target, len(list.targets))
|
||||
for k, v := range list.targets {
|
||||
ntargets[k] = v
|
||||
}
|
||||
return ntargets
|
||||
}
|
||||
|
||||
// Send - sends events to targets identified by target IDs.
|
||||
func (list *TargetList) Send(event Event, id TargetID) (*http.Response, error) {
|
||||
list.RLock()
|
||||
target, ok := list.targets[id]
|
||||
list.RUnlock()
|
||||
if ok {
|
||||
return target.Send(event)
|
||||
}
|
||||
return nil, ErrARNNotFound{}
|
||||
}
|
||||
|
||||
// Stats returns stats for targets.
|
||||
func (list *TargetList) Stats() TargetStats {
|
||||
t := TargetStats{}
|
||||
if list == nil {
|
||||
return t
|
||||
}
|
||||
list.RLock()
|
||||
defer list.RUnlock()
|
||||
t.TargetStats = make(map[string]TargetStat, len(list.targets))
|
||||
for id, target := range list.targets {
|
||||
t.TargetStats[strings.ReplaceAll(id.String(), ":", "_")] = target.Stat()
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// NewTargetList - creates TargetList.
|
||||
func NewTargetList() *TargetList {
|
||||
return &TargetList{targets: make(map[TargetID]Target)}
|
||||
}
|
||||
62
internal/config/lambda/help.go
Normal file
62
internal/config/lambda/help.go
Normal file
@@ -0,0 +1,62 @@
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package lambda
|
||||
|
||||
import (
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/event/target"
|
||||
)
|
||||
|
||||
// Help template inputs for all lambda targets
|
||||
var (
|
||||
HelpWebhook = config.HelpKVS{
|
||||
config.HelpKV{
|
||||
Key: target.WebhookEndpoint,
|
||||
Description: "webhook server endpoint e.g. http://localhost:8080/minio/lambda",
|
||||
Type: "url",
|
||||
Sensitive: true,
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: target.WebhookAuthToken,
|
||||
Description: "opaque string or JWT authorization token",
|
||||
Optional: true,
|
||||
Type: "string",
|
||||
Sensitive: true,
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: config.Comment,
|
||||
Description: config.DefaultComment,
|
||||
Optional: true,
|
||||
Type: "sentence",
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: target.WebhookClientCert,
|
||||
Description: "client cert for Webhook mTLS auth",
|
||||
Optional: true,
|
||||
Type: "string",
|
||||
Sensitive: true,
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: target.WebhookClientKey,
|
||||
Description: "client cert key for Webhook mTLS auth",
|
||||
Optional: true,
|
||||
Type: "string",
|
||||
Sensitive: true,
|
||||
},
|
||||
}
|
||||
)
|
||||
211
internal/config/lambda/parse.go
Normal file
211
internal/config/lambda/parse.go
Normal file
@@ -0,0 +1,211 @@
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package lambda
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/config/lambda/event"
|
||||
"github.com/minio/minio/internal/config/lambda/target"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/env"
|
||||
xnet "github.com/minio/pkg/net"
|
||||
)
|
||||
|
||||
// ErrTargetsOffline - Indicates single/multiple target failures.
|
||||
var ErrTargetsOffline = errors.New("one or more targets are offline. Please use `mc admin info --json` to check the offline targets")
|
||||
|
||||
// TestSubSysLambdaTargets - tests notification targets of given subsystem
|
||||
func TestSubSysLambdaTargets(ctx context.Context, cfg config.Config, subSys string, transport *http.Transport) error {
|
||||
if err := checkValidLambdaKeysForSubSys(subSys, cfg[subSys]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
targetList, err := fetchSubSysTargets(ctx, cfg, subSys, transport)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, target := range targetList {
|
||||
defer target.Close()
|
||||
}
|
||||
|
||||
for _, target := range targetList {
|
||||
yes, err := target.IsActive()
|
||||
if err == nil && !yes {
|
||||
err = ErrTargetsOffline
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("error (%s): %w", target.ID(), err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func fetchSubSysTargets(ctx context.Context, cfg config.Config, subSys string, transport *http.Transport) (targets []event.Target, err error) {
|
||||
if err := checkValidLambdaKeysForSubSys(subSys, cfg[subSys]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if subSys == config.LambdaWebhookSubSys {
|
||||
webhookTargets, err := GetLambdaWebhook(cfg[config.LambdaWebhookSubSys], transport)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for id, args := range webhookTargets {
|
||||
if !args.Enable {
|
||||
continue
|
||||
}
|
||||
t, err := target.NewWebhookTarget(ctx, id, args, logger.LogOnceIf, transport)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
targets = append(targets, t)
|
||||
}
|
||||
}
|
||||
return targets, nil
|
||||
}
|
||||
|
||||
// FetchEnabledTargets - Returns a set of configured TargetList
|
||||
func FetchEnabledTargets(ctx context.Context, cfg config.Config, transport *http.Transport) (*event.TargetList, error) {
|
||||
targetList := event.NewTargetList()
|
||||
for _, subSys := range config.LambdaSubSystems.ToSlice() {
|
||||
targets, err := fetchSubSysTargets(ctx, cfg, subSys, transport)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, t := range targets {
|
||||
if err = targetList.Add(t); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return targetList, nil
|
||||
}
|
||||
|
||||
// DefaultLambdaKVS - default notification list of kvs.
|
||||
var (
|
||||
DefaultLambdaKVS = map[string]config.KVS{
|
||||
config.LambdaWebhookSubSys: DefaultWebhookKVS,
|
||||
}
|
||||
)
|
||||
|
||||
// DefaultWebhookKVS - default KV for webhook config
|
||||
var (
|
||||
DefaultWebhookKVS = config.KVS{
|
||||
config.KV{
|
||||
Key: config.Enable,
|
||||
Value: config.EnableOff,
|
||||
},
|
||||
config.KV{
|
||||
Key: target.WebhookEndpoint,
|
||||
Value: "",
|
||||
},
|
||||
config.KV{
|
||||
Key: target.WebhookAuthToken,
|
||||
Value: "",
|
||||
},
|
||||
config.KV{
|
||||
Key: target.WebhookClientCert,
|
||||
Value: "",
|
||||
},
|
||||
config.KV{
|
||||
Key: target.WebhookClientKey,
|
||||
Value: "",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
func checkValidLambdaKeysForSubSys(subSys string, tgt map[string]config.KVS) error {
|
||||
validKVS, ok := DefaultLambdaKVS[subSys]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
for tname, kv := range tgt {
|
||||
subSysTarget := subSys
|
||||
if tname != config.Default {
|
||||
subSysTarget = subSys + config.SubSystemSeparator + tname
|
||||
}
|
||||
if v, ok := kv.Lookup(config.Enable); ok && v == config.EnableOn {
|
||||
if err := config.CheckValidKeys(subSysTarget, kv, validKVS); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetLambdaWebhook - returns a map of registered notification 'webhook' targets
|
||||
func GetLambdaWebhook(webhookKVS map[string]config.KVS, transport *http.Transport) (
|
||||
map[string]target.WebhookArgs, error,
|
||||
) {
|
||||
webhookTargets := make(map[string]target.WebhookArgs)
|
||||
for k, kv := range config.Merge(webhookKVS, target.EnvWebhookEnable, DefaultWebhookKVS) {
|
||||
enableEnv := target.EnvWebhookEnable
|
||||
if k != config.Default {
|
||||
enableEnv = enableEnv + config.Default + k
|
||||
}
|
||||
enabled, err := config.ParseBool(env.Get(enableEnv, kv.Get(config.Enable)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !enabled {
|
||||
continue
|
||||
}
|
||||
urlEnv := target.EnvWebhookEndpoint
|
||||
if k != config.Default {
|
||||
urlEnv = urlEnv + config.Default + k
|
||||
}
|
||||
url, err := xnet.ParseHTTPURL(env.Get(urlEnv, kv.Get(target.WebhookEndpoint)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
authEnv := target.EnvWebhookAuthToken
|
||||
if k != config.Default {
|
||||
authEnv = authEnv + config.Default + k
|
||||
}
|
||||
clientCertEnv := target.EnvWebhookClientCert
|
||||
if k != config.Default {
|
||||
clientCertEnv = clientCertEnv + config.Default + k
|
||||
}
|
||||
|
||||
clientKeyEnv := target.EnvWebhookClientKey
|
||||
if k != config.Default {
|
||||
clientKeyEnv = clientKeyEnv + config.Default + k
|
||||
}
|
||||
|
||||
webhookArgs := target.WebhookArgs{
|
||||
Enable: enabled,
|
||||
Endpoint: *url,
|
||||
Transport: transport,
|
||||
AuthToken: env.Get(authEnv, kv.Get(target.WebhookAuthToken)),
|
||||
ClientCert: env.Get(clientCertEnv, kv.Get(target.WebhookClientCert)),
|
||||
ClientKey: env.Get(clientKeyEnv, kv.Get(target.WebhookClientKey)),
|
||||
}
|
||||
if err = webhookArgs.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
webhookTargets[k] = webhookArgs
|
||||
}
|
||||
return webhookTargets, nil
|
||||
}
|
||||
51
internal/config/lambda/target/lazyinit.go
Normal file
51
internal/config/lambda/target/lazyinit.go
Normal file
@@ -0,0 +1,51 @@
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package target
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// Inspired from Golang sync.Once but it is only marked
|
||||
// initialized when the provided function returns nil.
|
||||
|
||||
type lazyInit struct {
|
||||
done uint32
|
||||
m sync.Mutex
|
||||
}
|
||||
|
||||
func (l *lazyInit) Do(f func() error) error {
|
||||
if atomic.LoadUint32(&l.done) == 0 {
|
||||
return l.doSlow(f)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *lazyInit) doSlow(f func() error) error {
|
||||
l.m.Lock()
|
||||
defer l.m.Unlock()
|
||||
if atomic.LoadUint32(&l.done) == 0 {
|
||||
if err := f(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Mark as done only when f() is successful
|
||||
atomic.StoreUint32(&l.done, 1)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
247
internal/config/lambda/target/webhook.go
Normal file
247
internal/config/lambda/target/webhook.go
Normal file
@@ -0,0 +1,247 @@
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package target
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/config/lambda/event"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/certs"
|
||||
xnet "github.com/minio/pkg/net"
|
||||
)
|
||||
|
||||
// Webhook constants
|
||||
const (
|
||||
WebhookEndpoint = "endpoint"
|
||||
WebhookAuthToken = "auth_token"
|
||||
WebhookClientCert = "client_cert"
|
||||
WebhookClientKey = "client_key"
|
||||
|
||||
EnvWebhookEnable = "MINIO_LAMBDA_WEBHOOK_ENABLE"
|
||||
EnvWebhookEndpoint = "MINIO_LAMBDA_WEBHOOK_ENDPOINT"
|
||||
EnvWebhookAuthToken = "MINIO_LAMBDA_WEBHOOK_AUTH_TOKEN"
|
||||
EnvWebhookClientCert = "MINIO_LAMBDA_WEBHOOK_CLIENT_CERT"
|
||||
EnvWebhookClientKey = "MINIO_LAMBDA_WEBHOOK_CLIENT_KEY"
|
||||
)
|
||||
|
||||
// WebhookArgs - Webhook target arguments.
|
||||
type WebhookArgs struct {
|
||||
Enable bool `json:"enable"`
|
||||
Endpoint xnet.URL `json:"endpoint"`
|
||||
AuthToken string `json:"authToken"`
|
||||
Transport *http.Transport `json:"-"`
|
||||
ClientCert string `json:"clientCert"`
|
||||
ClientKey string `json:"clientKey"`
|
||||
}
|
||||
|
||||
// Validate WebhookArgs fields
|
||||
func (w WebhookArgs) Validate() error {
|
||||
if !w.Enable {
|
||||
return nil
|
||||
}
|
||||
if w.Endpoint.IsEmpty() {
|
||||
return errors.New("endpoint empty")
|
||||
}
|
||||
if w.ClientCert != "" && w.ClientKey == "" || w.ClientCert == "" && w.ClientKey != "" {
|
||||
return errors.New("cert and key must be specified as a pair")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WebhookTarget - Webhook target.
|
||||
type WebhookTarget struct {
|
||||
activeRequests int64
|
||||
totalRequests int64
|
||||
failedRequests int64
|
||||
|
||||
lazyInit lazyInit
|
||||
|
||||
id event.TargetID
|
||||
args WebhookArgs
|
||||
transport *http.Transport
|
||||
httpClient *http.Client
|
||||
loggerOnce logger.LogOnce
|
||||
cancel context.CancelFunc
|
||||
cancelCh <-chan struct{}
|
||||
}
|
||||
|
||||
// ID - returns target ID.
|
||||
func (target *WebhookTarget) ID() event.TargetID {
|
||||
return target.id
|
||||
}
|
||||
|
||||
// IsActive - Return true if target is up and active
|
||||
func (target *WebhookTarget) IsActive() (bool, error) {
|
||||
if err := target.init(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return target.isActive()
|
||||
}
|
||||
|
||||
// errNotConnected - indicates that the target connection is not active.
|
||||
var errNotConnected = errors.New("not connected to target server/service")
|
||||
|
||||
func (target *WebhookTarget) isActive() (bool, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodHead, target.args.Endpoint.String(), nil)
|
||||
if err != nil {
|
||||
if xnet.IsNetworkOrHostDown(err, false) {
|
||||
return false, errNotConnected
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
tokens := strings.Fields(target.args.AuthToken)
|
||||
switch len(tokens) {
|
||||
case 2:
|
||||
req.Header.Set("Authorization", target.args.AuthToken)
|
||||
case 1:
|
||||
req.Header.Set("Authorization", "Bearer "+target.args.AuthToken)
|
||||
}
|
||||
|
||||
resp, err := target.httpClient.Do(req)
|
||||
if err != nil {
|
||||
if xnet.IsNetworkOrHostDown(err, true) {
|
||||
return false, errNotConnected
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
// No network failure i.e response from the target means its up
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Stat - returns lamdba webhook target statistics such as
|
||||
// current calls in progress, successfully completed functions
|
||||
// failed functions.
|
||||
func (target *WebhookTarget) Stat() event.TargetStat {
|
||||
return event.TargetStat{
|
||||
ID: target.id,
|
||||
ActiveRequests: atomic.LoadInt64(&target.activeRequests),
|
||||
TotalRequests: atomic.LoadInt64(&target.totalRequests),
|
||||
FailedRequests: atomic.LoadInt64(&target.failedRequests),
|
||||
}
|
||||
}
|
||||
|
||||
// Send - sends an event to the webhook.
|
||||
func (target *WebhookTarget) Send(eventData event.Event) (resp *http.Response, err error) {
|
||||
atomic.AddInt64(&target.activeRequests, 1)
|
||||
defer atomic.AddInt64(&target.activeRequests, -1)
|
||||
|
||||
atomic.AddInt64(&target.totalRequests, 1)
|
||||
defer func() {
|
||||
if err != nil {
|
||||
atomic.AddInt64(&target.failedRequests, 1)
|
||||
}
|
||||
}()
|
||||
|
||||
if err = target.init(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
data, err := json.Marshal(eventData)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, target.args.Endpoint.String(), bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Verify if the authToken already contains
|
||||
// <Key> <Token> like format, if this is
|
||||
// already present we can blindly use the
|
||||
// authToken as is instead of adding 'Bearer'
|
||||
tokens := strings.Fields(target.args.AuthToken)
|
||||
switch len(tokens) {
|
||||
case 2:
|
||||
req.Header.Set("Authorization", target.args.AuthToken)
|
||||
case 1:
|
||||
req.Header.Set("Authorization", "Bearer "+target.args.AuthToken)
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
return target.httpClient.Do(req)
|
||||
}
|
||||
|
||||
// Close the target. Will cancel all active requests.
|
||||
func (target *WebhookTarget) Close() error {
|
||||
target.cancel()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (target *WebhookTarget) init() error {
|
||||
return target.lazyInit.Do(target.initWebhook)
|
||||
}
|
||||
|
||||
// Only called from init()
|
||||
func (target *WebhookTarget) initWebhook() error {
|
||||
args := target.args
|
||||
transport := target.transport
|
||||
|
||||
if args.ClientCert != "" && args.ClientKey != "" {
|
||||
manager, err := certs.NewManager(context.Background(), args.ClientCert, args.ClientKey, tls.LoadX509KeyPair)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manager.ReloadOnSignal(syscall.SIGHUP) // allow reloads upon SIGHUP
|
||||
transport.TLSClientConfig.GetClientCertificate = manager.GetClientCertificate
|
||||
}
|
||||
target.httpClient = &http.Client{Transport: transport}
|
||||
|
||||
yes, err := target.isActive()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !yes {
|
||||
return errNotConnected
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewWebhookTarget - creates new Webhook target.
|
||||
func NewWebhookTarget(ctx context.Context, id string, args WebhookArgs, loggerOnce logger.LogOnce, transport *http.Transport) (*WebhookTarget, error) {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
|
||||
target := &WebhookTarget{
|
||||
id: event.TargetID{ID: id, Name: "webhook"},
|
||||
args: args,
|
||||
loggerOnce: loggerOnce,
|
||||
transport: transport,
|
||||
cancel: cancel,
|
||||
cancelCh: ctx.Done(),
|
||||
}
|
||||
|
||||
return target, nil
|
||||
}
|
||||
Reference in New Issue
Block a user