mirror of https://github.com/minio/minio.git
Simplify gateway backend registration (#5111)
This commit is contained in:
parent
7195ac7f14
commit
6400f506da
|
@ -33,13 +33,86 @@ import (
|
||||||
|
|
||||||
"github.com/Azure/azure-sdk-for-go/storage"
|
"github.com/Azure/azure-sdk-for-go/storage"
|
||||||
humanize "github.com/dustin/go-humanize"
|
humanize "github.com/dustin/go-humanize"
|
||||||
|
"github.com/minio/cli"
|
||||||
"github.com/minio/minio-go/pkg/policy"
|
"github.com/minio/minio-go/pkg/policy"
|
||||||
"github.com/minio/minio/pkg/hash"
|
"github.com/minio/minio/pkg/hash"
|
||||||
)
|
)
|
||||||
|
|
||||||
const globalAzureAPIVersion = "2016-05-31"
|
const (
|
||||||
const azureBlockSize = 100 * humanize.MiByte
|
globalAzureAPIVersion = "2016-05-31"
|
||||||
const metadataObjectNameTemplate = globalMinioSysTmp + "multipart/v1/%s.%x/azure.json"
|
azureBlockSize = 100 * humanize.MiByte
|
||||||
|
metadataObjectNameTemplate = globalMinioSysTmp + "multipart/v1/%s.%x/azure.json"
|
||||||
|
azureBackend = "azure"
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
const azureGatewayTemplate = `NAME:
|
||||||
|
{{.HelpName}} - {{.Usage}}
|
||||||
|
|
||||||
|
USAGE:
|
||||||
|
{{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [ENDPOINT]
|
||||||
|
{{if .VisibleFlags}}
|
||||||
|
FLAGS:
|
||||||
|
{{range .VisibleFlags}}{{.}}
|
||||||
|
{{end}}{{end}}
|
||||||
|
ENDPOINT:
|
||||||
|
Azure server endpoint. Default ENDPOINT is https://core.windows.net
|
||||||
|
|
||||||
|
ENVIRONMENT VARIABLES:
|
||||||
|
ACCESS:
|
||||||
|
MINIO_ACCESS_KEY: Username or access key of Azure storage.
|
||||||
|
MINIO_SECRET_KEY: Password or secret key of Azure storage.
|
||||||
|
|
||||||
|
BROWSER:
|
||||||
|
MINIO_BROWSER: To disable web browser access, set this value to "off".
|
||||||
|
|
||||||
|
EXAMPLES:
|
||||||
|
1. Start minio gateway server for Azure Blob Storage backend.
|
||||||
|
$ export MINIO_ACCESS_KEY=azureaccountname
|
||||||
|
$ export MINIO_SECRET_KEY=azureaccountkey
|
||||||
|
$ {{.HelpName}}
|
||||||
|
|
||||||
|
2. Start minio gateway server for Azure Blob Storage backend on custom endpoint.
|
||||||
|
$ export MINIO_ACCESS_KEY=azureaccountname
|
||||||
|
$ export MINIO_SECRET_KEY=azureaccountkey
|
||||||
|
$ {{.HelpName}} https://azure.example.com
|
||||||
|
|
||||||
|
`
|
||||||
|
|
||||||
|
MustRegisterGatewayCommand(cli.Command{
|
||||||
|
Name: azureBackend,
|
||||||
|
Usage: "Microsoft Azure Blob Storage.",
|
||||||
|
Action: azureGatewayMain,
|
||||||
|
CustomHelpTemplate: azureGatewayTemplate,
|
||||||
|
Flags: append(serverFlags, globalFlags...),
|
||||||
|
HideHelpCommand: true,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handler for 'minio gateway azure' command line.
|
||||||
|
func azureGatewayMain(ctx *cli.Context) {
|
||||||
|
// Validate gateway arguments.
|
||||||
|
host := ctx.Args().First()
|
||||||
|
// Validate gateway arguments.
|
||||||
|
fatalIf(validateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
|
||||||
|
|
||||||
|
startGateway(ctx, &AzureGateway{host})
|
||||||
|
}
|
||||||
|
|
||||||
|
// AzureGateway implements Gateway.
|
||||||
|
type AzureGateway struct {
|
||||||
|
host string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name implements Gateway interface.
|
||||||
|
func (g *AzureGateway) Name() string {
|
||||||
|
return azureBackend
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewGatewayLayer initializes azure blob storage client and returns AzureObjects.
|
||||||
|
func (g *AzureGateway) NewGatewayLayer() (GatewayLayer, error) {
|
||||||
|
return newAzureLayer(g.host)
|
||||||
|
}
|
||||||
|
|
||||||
// s3MetaToAzureProperties converts metadata meant for S3 PUT/COPY
|
// s3MetaToAzureProperties converts metadata meant for S3 PUT/COPY
|
||||||
// object into Azure data structures - BlobMetadata and
|
// object into Azure data structures - BlobMetadata and
|
||||||
|
|
|
@ -29,6 +29,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
b2 "github.com/minio/blazer/base"
|
b2 "github.com/minio/blazer/base"
|
||||||
|
"github.com/minio/cli"
|
||||||
"github.com/minio/minio-go/pkg/policy"
|
"github.com/minio/minio-go/pkg/policy"
|
||||||
h2 "github.com/minio/minio/pkg/hash"
|
h2 "github.com/minio/minio/pkg/hash"
|
||||||
)
|
)
|
||||||
|
@ -37,8 +38,65 @@ import (
|
||||||
const (
|
const (
|
||||||
bucketTypePrivate = "allPrivate"
|
bucketTypePrivate = "allPrivate"
|
||||||
bucketTypeReadOnly = "allPublic"
|
bucketTypeReadOnly = "allPublic"
|
||||||
|
b2Backend = "b2"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
const b2GatewayTemplate = `NAME:
|
||||||
|
{{.HelpName}} - {{.Usage}}
|
||||||
|
|
||||||
|
USAGE:
|
||||||
|
{{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}}
|
||||||
|
{{if .VisibleFlags}}
|
||||||
|
FLAGS:
|
||||||
|
{{range .VisibleFlags}}{{.}}
|
||||||
|
{{end}}{{end}}
|
||||||
|
ENVIRONMENT VARIABLES:
|
||||||
|
ACCESS:
|
||||||
|
MINIO_ACCESS_KEY: B2 account id.
|
||||||
|
MINIO_SECRET_KEY: B2 application key.
|
||||||
|
|
||||||
|
BROWSER:
|
||||||
|
MINIO_BROWSER: To disable web browser access, set this value to "off".
|
||||||
|
|
||||||
|
EXAMPLES:
|
||||||
|
1. Start minio gateway server for B2 backend.
|
||||||
|
$ export MINIO_ACCESS_KEY=accountID
|
||||||
|
$ export MINIO_SECRET_KEY=applicationKey
|
||||||
|
$ {{.HelpName}}
|
||||||
|
|
||||||
|
`
|
||||||
|
|
||||||
|
MustRegisterGatewayCommand(cli.Command{
|
||||||
|
Name: b2Backend,
|
||||||
|
Usage: "Backblaze B2.",
|
||||||
|
Action: b2GatewayMain,
|
||||||
|
CustomHelpTemplate: b2GatewayTemplate,
|
||||||
|
Flags: append(serverFlags, globalFlags...),
|
||||||
|
HideHelpCommand: true,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handler for 'minio gateway b2' command line.
|
||||||
|
func b2GatewayMain(ctx *cli.Context) {
|
||||||
|
startGateway(ctx, &B2Gateway{})
|
||||||
|
}
|
||||||
|
|
||||||
|
// B2Gateway implements Gateway.
|
||||||
|
type B2Gateway struct{}
|
||||||
|
|
||||||
|
// Name implements Gateway interface.
|
||||||
|
func (g *B2Gateway) Name() string {
|
||||||
|
return b2Backend
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewGatewayLayer returns b2 gateway layer, implements GatewayLayer interface to
|
||||||
|
// talk to B2 remote backend.
|
||||||
|
func (g *B2Gateway) NewGatewayLayer() (GatewayLayer, error) {
|
||||||
|
log.Println(colorYellow("\n *** Warning: Not Ready for Production ***"))
|
||||||
|
return newB2GatewayLayer()
|
||||||
|
}
|
||||||
|
|
||||||
// b2Object implements gateway for Minio and BackBlaze B2 compatible object storage servers.
|
// b2Object implements gateway for Minio and BackBlaze B2 compatible object storage servers.
|
||||||
type b2Objects struct {
|
type b2Objects struct {
|
||||||
gatewayUnsupported
|
gatewayUnsupported
|
||||||
|
@ -49,9 +107,8 @@ type b2Objects struct {
|
||||||
ctx context.Context
|
ctx context.Context
|
||||||
}
|
}
|
||||||
|
|
||||||
// newB2Gateway returns b2 gateway layer, implements GatewayLayer interface to
|
// newB2GatewayLayer returns b2 gateway layer.
|
||||||
// talk to B2 remote backend.
|
func newB2GatewayLayer() (GatewayLayer, error) {
|
||||||
func newB2Gateway() (GatewayLayer, error) {
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
creds := serverConfig.GetCredential()
|
creds := serverConfig.GetCredential()
|
||||||
|
|
||||||
|
|
|
@ -30,16 +30,15 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"golang.org/x/oauth2/google"
|
|
||||||
|
|
||||||
"cloud.google.com/go/storage"
|
"cloud.google.com/go/storage"
|
||||||
cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1"
|
"github.com/minio/cli"
|
||||||
"google.golang.org/api/googleapi"
|
|
||||||
"google.golang.org/api/iterator"
|
|
||||||
|
|
||||||
minio "github.com/minio/minio-go"
|
minio "github.com/minio/minio-go"
|
||||||
"github.com/minio/minio-go/pkg/policy"
|
"github.com/minio/minio-go/pkg/policy"
|
||||||
"github.com/minio/minio/pkg/hash"
|
"github.com/minio/minio/pkg/hash"
|
||||||
|
"golang.org/x/oauth2/google"
|
||||||
|
cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1"
|
||||||
|
"google.golang.org/api/googleapi"
|
||||||
|
"google.golang.org/api/iterator"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
@ -78,8 +77,85 @@ const (
|
||||||
|
|
||||||
// Project ID key in credentials.json
|
// Project ID key in credentials.json
|
||||||
gcsProjectIDKey = "project_id"
|
gcsProjectIDKey = "project_id"
|
||||||
|
|
||||||
|
gcsBackend = "gcs"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
const gcsGatewayTemplate = `NAME:
|
||||||
|
{{.HelpName}} - {{.Usage}}
|
||||||
|
|
||||||
|
USAGE:
|
||||||
|
{{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [PROJECTID]
|
||||||
|
{{if .VisibleFlags}}
|
||||||
|
FLAGS:
|
||||||
|
{{range .VisibleFlags}}{{.}}
|
||||||
|
{{end}}{{end}}
|
||||||
|
PROJECTID:
|
||||||
|
GCS project-id should be provided if GOOGLE_APPLICATION_CREDENTIALS environmental variable is not set.
|
||||||
|
|
||||||
|
ENVIRONMENT VARIABLES:
|
||||||
|
ACCESS:
|
||||||
|
MINIO_ACCESS_KEY: Username or access key of GCS.
|
||||||
|
MINIO_SECRET_KEY: Password or secret key of GCS.
|
||||||
|
|
||||||
|
BROWSER:
|
||||||
|
MINIO_BROWSER: To disable web browser access, set this value to "off".
|
||||||
|
|
||||||
|
GCS credentials file:
|
||||||
|
GOOGLE_APPLICATION_CREDENTIALS: Path to credentials.json
|
||||||
|
|
||||||
|
EXAMPLES:
|
||||||
|
1. Start minio gateway server for GCS backend.
|
||||||
|
$ export GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json
|
||||||
|
(Instructions to generate credentials : https://developers.google.com/identity/protocols/application-default-credentials)
|
||||||
|
$ export MINIO_ACCESS_KEY=accesskey
|
||||||
|
$ export MINIO_SECRET_KEY=secretkey
|
||||||
|
$ {{.HelpName}} mygcsprojectid
|
||||||
|
|
||||||
|
`
|
||||||
|
|
||||||
|
MustRegisterGatewayCommand(cli.Command{
|
||||||
|
Name: gcsBackend,
|
||||||
|
Usage: "Google Cloud Storage.",
|
||||||
|
Action: gcsGatewayMain,
|
||||||
|
CustomHelpTemplate: gcsGatewayTemplate,
|
||||||
|
Flags: append(serverFlags, globalFlags...),
|
||||||
|
HideHelpCommand: true,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handler for 'minio gateway gcs' command line.
|
||||||
|
func gcsGatewayMain(ctx *cli.Context) {
|
||||||
|
projectID := ctx.Args().First()
|
||||||
|
if projectID == "" && os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" {
|
||||||
|
errorIf(errGCSProjectIDNotFound, "project-id should be provided as argument or GOOGLE_APPLICATION_CREDENTIALS should be set with path to credentials.json")
|
||||||
|
cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
|
||||||
|
}
|
||||||
|
if projectID != "" && !isValidGCSProjectIDFormat(projectID) {
|
||||||
|
errorIf(errGCSInvalidProjectID, "Unable to start GCS gateway with %s", ctx.Args().First())
|
||||||
|
cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
startGateway(ctx, &GCSGateway{projectID})
|
||||||
|
}
|
||||||
|
|
||||||
|
// GCSGateway implements Gateway.
|
||||||
|
type GCSGateway struct {
|
||||||
|
projectID string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name returns the name of gcs gatewaylayer.
|
||||||
|
func (g *GCSGateway) Name() string {
|
||||||
|
return gcsBackend
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewGatewayLayer returns gcs gatewaylayer.
|
||||||
|
func (g *GCSGateway) NewGatewayLayer() (GatewayLayer, error) {
|
||||||
|
log.Println(colorYellow("\n *** Warning: Not Ready for Production ***"))
|
||||||
|
return newGCSGatewayLayer(g.projectID)
|
||||||
|
}
|
||||||
|
|
||||||
// Stored in gcs.json - Contents of this file is not used anywhere. It can be
|
// Stored in gcs.json - Contents of this file is not used anywhere. It can be
|
||||||
// used for debugging purposes.
|
// used for debugging purposes.
|
||||||
type gcsMultipartMetaV1 struct {
|
type gcsMultipartMetaV1 struct {
|
||||||
|
@ -275,8 +351,8 @@ func gcsParseProjectID(credsFile string) (projectID string, err error) {
|
||||||
return googleCreds[gcsProjectIDKey], err
|
return googleCreds[gcsProjectIDKey], err
|
||||||
}
|
}
|
||||||
|
|
||||||
// newGCSGateway returns gcs gatewaylayer
|
// newGCSGatewayLayer returns gcs gatewaylayer
|
||||||
func newGCSGateway(projectID string) (GatewayLayer, error) {
|
func newGCSGatewayLayer(projectID string) (GatewayLayer, error) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
|
|
|
@ -31,212 +31,41 @@ import (
|
||||||
miniohttp "github.com/minio/minio/pkg/http"
|
miniohttp "github.com/minio/minio/pkg/http"
|
||||||
)
|
)
|
||||||
|
|
||||||
const azureGatewayTemplate = `NAME:
|
|
||||||
{{.HelpName}} - {{.Usage}}
|
|
||||||
|
|
||||||
USAGE:
|
|
||||||
{{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [ENDPOINT]
|
|
||||||
{{if .VisibleFlags}}
|
|
||||||
FLAGS:
|
|
||||||
{{range .VisibleFlags}}{{.}}
|
|
||||||
{{end}}{{end}}
|
|
||||||
ENDPOINT:
|
|
||||||
Azure server endpoint. Default ENDPOINT is https://core.windows.net
|
|
||||||
|
|
||||||
ENVIRONMENT VARIABLES:
|
|
||||||
ACCESS:
|
|
||||||
MINIO_ACCESS_KEY: Username or access key of Azure storage.
|
|
||||||
MINIO_SECRET_KEY: Password or secret key of Azure storage.
|
|
||||||
|
|
||||||
BROWSER:
|
|
||||||
MINIO_BROWSER: To disable web browser access, set this value to "off".
|
|
||||||
|
|
||||||
EXAMPLES:
|
|
||||||
1. Start minio gateway server for Azure Blob Storage backend.
|
|
||||||
$ export MINIO_ACCESS_KEY=azureaccountname
|
|
||||||
$ export MINIO_SECRET_KEY=azureaccountkey
|
|
||||||
$ {{.HelpName}}
|
|
||||||
|
|
||||||
2. Start minio gateway server for Azure Blob Storage backend on custom endpoint.
|
|
||||||
$ export MINIO_ACCESS_KEY=azureaccountname
|
|
||||||
$ export MINIO_SECRET_KEY=azureaccountkey
|
|
||||||
$ {{.HelpName}} https://azure.example.com
|
|
||||||
`
|
|
||||||
|
|
||||||
const s3GatewayTemplate = `NAME:
|
|
||||||
{{.HelpName}} - {{.Usage}}
|
|
||||||
|
|
||||||
USAGE:
|
|
||||||
{{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [ENDPOINT]
|
|
||||||
{{if .VisibleFlags}}
|
|
||||||
FLAGS:
|
|
||||||
{{range .VisibleFlags}}{{.}}
|
|
||||||
{{end}}{{end}}
|
|
||||||
ENDPOINT:
|
|
||||||
S3 server endpoint. Default ENDPOINT is https://s3.amazonaws.com
|
|
||||||
|
|
||||||
ENVIRONMENT VARIABLES:
|
|
||||||
ACCESS:
|
|
||||||
MINIO_ACCESS_KEY: Username or access key of S3 storage.
|
|
||||||
MINIO_SECRET_KEY: Password or secret key of S3 storage.
|
|
||||||
|
|
||||||
BROWSER:
|
|
||||||
MINIO_BROWSER: To disable web browser access, set this value to "off".
|
|
||||||
|
|
||||||
EXAMPLES:
|
|
||||||
1. Start minio gateway server for AWS S3 backend.
|
|
||||||
$ export MINIO_ACCESS_KEY=accesskey
|
|
||||||
$ export MINIO_SECRET_KEY=secretkey
|
|
||||||
$ {{.HelpName}}
|
|
||||||
|
|
||||||
2. Start minio gateway server for S3 backend on custom endpoint.
|
|
||||||
$ export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
|
|
||||||
$ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
|
|
||||||
$ {{.HelpName}} https://play.minio.io:9000
|
|
||||||
`
|
|
||||||
|
|
||||||
const gcsGatewayTemplate = `NAME:
|
|
||||||
{{.HelpName}} - {{.Usage}}
|
|
||||||
|
|
||||||
USAGE:
|
|
||||||
{{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [PROJECTID]
|
|
||||||
{{if .VisibleFlags}}
|
|
||||||
FLAGS:
|
|
||||||
{{range .VisibleFlags}}{{.}}
|
|
||||||
{{end}}{{end}}
|
|
||||||
PROJECTID:
|
|
||||||
GCS project-id should be provided if GOOGLE_APPLICATION_CREDENTIALS environmental variable is not set.
|
|
||||||
|
|
||||||
ENVIRONMENT VARIABLES:
|
|
||||||
ACCESS:
|
|
||||||
MINIO_ACCESS_KEY: Username or access key of GCS.
|
|
||||||
MINIO_SECRET_KEY: Password or secret key of GCS.
|
|
||||||
|
|
||||||
BROWSER:
|
|
||||||
MINIO_BROWSER: To disable web browser access, set this value to "off".
|
|
||||||
|
|
||||||
GCS credentials file:
|
|
||||||
GOOGLE_APPLICATION_CREDENTIALS: Path to credentials.json
|
|
||||||
|
|
||||||
EXAMPLES:
|
|
||||||
1. Start minio gateway server for GCS backend.
|
|
||||||
$ export GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json
|
|
||||||
(Instructions to generate credentials : https://developers.google.com/identity/protocols/application-default-credentials)
|
|
||||||
$ export MINIO_ACCESS_KEY=accesskey
|
|
||||||
$ export MINIO_SECRET_KEY=secretkey
|
|
||||||
$ {{.HelpName}} mygcsprojectid
|
|
||||||
|
|
||||||
`
|
|
||||||
|
|
||||||
const b2GatewayTemplate = `NAME:
|
|
||||||
{{.HelpName}} - {{.Usage}}
|
|
||||||
|
|
||||||
USAGE:
|
|
||||||
{{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}}
|
|
||||||
{{if .VisibleFlags}}
|
|
||||||
FLAGS:
|
|
||||||
{{range .VisibleFlags}}{{.}}
|
|
||||||
{{end}}{{end}}
|
|
||||||
ENVIRONMENT VARIABLES:
|
|
||||||
ACCESS:
|
|
||||||
MINIO_ACCESS_KEY: B2 account id.
|
|
||||||
MINIO_SECRET_KEY: B2 application key.
|
|
||||||
|
|
||||||
BROWSER:
|
|
||||||
MINIO_BROWSER: To disable web browser access, set this value to "off".
|
|
||||||
|
|
||||||
EXAMPLES:
|
|
||||||
1. Start minio gateway server for B2 backend.
|
|
||||||
$ export MINIO_ACCESS_KEY=accountID
|
|
||||||
$ export MINIO_SECRET_KEY=applicationKey
|
|
||||||
$ {{.HelpName}}
|
|
||||||
|
|
||||||
`
|
|
||||||
|
|
||||||
var (
|
var (
|
||||||
azureBackendCmd = cli.Command{
|
|
||||||
Name: "azure",
|
|
||||||
Usage: "Microsoft Azure Blob Storage.",
|
|
||||||
Action: azureGatewayMain,
|
|
||||||
CustomHelpTemplate: azureGatewayTemplate,
|
|
||||||
Flags: append(serverFlags, globalFlags...),
|
|
||||||
HideHelpCommand: true,
|
|
||||||
}
|
|
||||||
|
|
||||||
s3BackendCmd = cli.Command{
|
|
||||||
Name: "s3",
|
|
||||||
Usage: "Amazon Simple Storage Service (S3).",
|
|
||||||
Action: s3GatewayMain,
|
|
||||||
CustomHelpTemplate: s3GatewayTemplate,
|
|
||||||
Flags: append(serverFlags, globalFlags...),
|
|
||||||
HideHelpCommand: true,
|
|
||||||
}
|
|
||||||
|
|
||||||
gcsBackendCmd = cli.Command{
|
|
||||||
Name: "gcs",
|
|
||||||
Usage: "Google Cloud Storage.",
|
|
||||||
Action: gcsGatewayMain,
|
|
||||||
CustomHelpTemplate: gcsGatewayTemplate,
|
|
||||||
Flags: append(serverFlags, globalFlags...),
|
|
||||||
HideHelpCommand: true,
|
|
||||||
}
|
|
||||||
|
|
||||||
b2BackendCmd = cli.Command{
|
|
||||||
Name: "b2",
|
|
||||||
Usage: "Backblaze B2.",
|
|
||||||
Action: b2GatewayMain,
|
|
||||||
CustomHelpTemplate: b2GatewayTemplate,
|
|
||||||
Flags: append(serverFlags, globalFlags...),
|
|
||||||
HideHelpCommand: true,
|
|
||||||
}
|
|
||||||
|
|
||||||
gatewayCmd = cli.Command{
|
gatewayCmd = cli.Command{
|
||||||
Name: "gateway",
|
Name: "gateway",
|
||||||
Usage: "Start object storage gateway.",
|
Usage: "Start object storage gateway.",
|
||||||
Flags: append(serverFlags, globalFlags...),
|
Flags: append(serverFlags, globalFlags...),
|
||||||
HideHelpCommand: true,
|
HideHelpCommand: true,
|
||||||
Subcommands: []cli.Command{azureBackendCmd, s3BackendCmd, gcsBackendCmd, b2BackendCmd},
|
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
// Represents the type of the gateway backend.
|
// Gateway represents a gateway backend.
|
||||||
type gatewayBackend string
|
type Gateway interface {
|
||||||
|
// Name returns the unique name of the gateway.
|
||||||
const (
|
Name() string
|
||||||
azureBackend gatewayBackend = "azure"
|
// NewGatewayLayer returns a new gateway layer.
|
||||||
s3Backend gatewayBackend = "s3"
|
NewGatewayLayer() (GatewayLayer, error)
|
||||||
gcsBackend gatewayBackend = "gcs"
|
|
||||||
b2Backend gatewayBackend = "b2"
|
|
||||||
// Add more backends here.
|
|
||||||
)
|
|
||||||
|
|
||||||
// Initialize gateway layer depending on the backend type.
|
|
||||||
// Supported backend types are
|
|
||||||
//
|
|
||||||
// - Azure Blob Storage.
|
|
||||||
// - AWS S3.
|
|
||||||
// - Google Cloud Storage.
|
|
||||||
// - Backblaze B2.
|
|
||||||
// - Add your favorite backend here.
|
|
||||||
func newGatewayLayer(backendType gatewayBackend, arg string) (GatewayLayer, error) {
|
|
||||||
switch backendType {
|
|
||||||
case azureBackend:
|
|
||||||
return newAzureLayer(arg)
|
|
||||||
case s3Backend:
|
|
||||||
return newS3Gateway(arg)
|
|
||||||
case gcsBackend:
|
|
||||||
// FIXME: The following print command is temporary and
|
|
||||||
// will be removed when gcs is ready for production use.
|
|
||||||
log.Println(colorYellow("\n *** Warning: Not Ready for Production ***"))
|
|
||||||
return newGCSGateway(arg)
|
|
||||||
case b2Backend:
|
|
||||||
// FIXME: The following print command is temporary and
|
|
||||||
// will be removed when B2 is ready for production use.
|
|
||||||
log.Println(colorYellow("\n *** Warning: Not Ready for Production ***"))
|
|
||||||
return newB2Gateway()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, fmt.Errorf("Unrecognized backend type %s", backendType)
|
// RegisterGatewayCommand registers a new command for gateway.
|
||||||
|
func RegisterGatewayCommand(cmd cli.Command) error {
|
||||||
|
// We should not have multiple subcommands with same name.
|
||||||
|
for _, c := range gatewayCmd.Subcommands {
|
||||||
|
if c.Name == cmd.Name {
|
||||||
|
return fmt.Errorf("duplicate gateway: %s", cmd.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
gatewayCmd.Subcommands = append(gatewayCmd.Subcommands, cmd)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// MustRegisterGatewayCommand is like RegisterGatewayCommand but panics instead of returning error.
|
||||||
|
func MustRegisterGatewayCommand(cmd cli.Command) {
|
||||||
|
if err := RegisterGatewayCommand(cmd); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Return endpoint.
|
// Return endpoint.
|
||||||
|
@ -292,62 +121,8 @@ func validateGatewayArguments(serverAddr, endpointAddr string) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handler for 'minio gateway azure' command line.
|
// Handler for 'minio gateway <name>'.
|
||||||
func azureGatewayMain(ctx *cli.Context) {
|
func startGateway(ctx *cli.Context, gw Gateway) {
|
||||||
if ctx.Args().Present() && ctx.Args().First() == "help" {
|
|
||||||
cli.ShowCommandHelpAndExit(ctx, "azure", 1)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate gateway arguments.
|
|
||||||
fatalIf(validateGatewayArguments(ctx.GlobalString("address"), ctx.Args().First()), "Invalid argument")
|
|
||||||
|
|
||||||
gatewayMain(ctx, azureBackend)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Handler for 'minio gateway s3' command line.
|
|
||||||
func s3GatewayMain(ctx *cli.Context) {
|
|
||||||
if ctx.Args().Present() && ctx.Args().First() == "help" {
|
|
||||||
cli.ShowCommandHelpAndExit(ctx, "s3", 1)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate gateway arguments.
|
|
||||||
fatalIf(validateGatewayArguments(ctx.GlobalString("address"), ctx.Args().First()), "Invalid argument")
|
|
||||||
|
|
||||||
gatewayMain(ctx, s3Backend)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Handler for 'minio gateway gcs' command line
|
|
||||||
func gcsGatewayMain(ctx *cli.Context) {
|
|
||||||
if ctx.Args().Present() && ctx.Args().First() == "help" {
|
|
||||||
cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
|
|
||||||
}
|
|
||||||
|
|
||||||
projectID := ctx.Args().First()
|
|
||||||
if projectID == "" && os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" {
|
|
||||||
errorIf(errGCSProjectIDNotFound, "project-id should be provided as argument or GOOGLE_APPLICATION_CREDENTIALS should be set with path to credentials.json")
|
|
||||||
cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
|
|
||||||
}
|
|
||||||
if projectID != "" && !isValidGCSProjectIDFormat(projectID) {
|
|
||||||
errorIf(errGCSInvalidProjectID, "Unable to start GCS gateway with %s", ctx.Args().First())
|
|
||||||
cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
|
|
||||||
}
|
|
||||||
|
|
||||||
gatewayMain(ctx, gcsBackend)
|
|
||||||
}
|
|
||||||
|
|
||||||
func b2GatewayMain(ctx *cli.Context) {
|
|
||||||
if ctx.Args().Present() && ctx.Args().First() == "help" {
|
|
||||||
cli.ShowCommandHelpAndExit(ctx, "b2", 1)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate gateway arguments.
|
|
||||||
fatalIf(validateGatewayArguments(ctx.GlobalString("address"), ctx.Args().First()), "Invalid argument")
|
|
||||||
|
|
||||||
gatewayMain(ctx, b2Backend)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Handler for 'minio gateway'.
|
|
||||||
func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
|
|
||||||
// Get quiet flag from command line argument.
|
// Get quiet flag from command line argument.
|
||||||
quietFlag := ctx.Bool("quiet") || ctx.GlobalBool("quiet")
|
quietFlag := ctx.Bool("quiet") || ctx.GlobalBool("quiet")
|
||||||
if quietFlag {
|
if quietFlag {
|
||||||
|
@ -367,8 +142,9 @@ func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
|
||||||
handleCommonEnvVars()
|
handleCommonEnvVars()
|
||||||
|
|
||||||
// Validate if we have access, secret set through environment.
|
// Validate if we have access, secret set through environment.
|
||||||
|
gatewayName := gw.Name()
|
||||||
if !globalIsEnvCreds {
|
if !globalIsEnvCreds {
|
||||||
fatalIf(fmt.Errorf("Access and Secret keys should be set through ENVs for backend [%s]", backendType), "")
|
fatalIf(fmt.Errorf("Access and Secret keys should be set through ENVs for backend [%s]", gatewayName), "")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create certs path.
|
// Create certs path.
|
||||||
|
@ -390,7 +166,11 @@ func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
|
||||||
|
|
||||||
initNSLock(false) // Enable local namespace lock.
|
initNSLock(false) // Enable local namespace lock.
|
||||||
|
|
||||||
newObject, err := newGatewayLayer(backendType, ctx.Args().First())
|
if ctx.Args().First() == "help" {
|
||||||
|
cli.ShowCommandHelpAndExit(ctx, gatewayName, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
newObject, err := gw.NewGatewayLayer()
|
||||||
fatalIf(err, "Unable to initialize gateway layer")
|
fatalIf(err, "Unable to initialize gateway layer")
|
||||||
|
|
||||||
router := mux.NewRouter().SkipClean(true)
|
router := mux.NewRouter().SkipClean(true)
|
||||||
|
@ -445,23 +225,12 @@ func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
|
||||||
|
|
||||||
// Prints the formatted startup message once object layer is initialized.
|
// Prints the formatted startup message once object layer is initialized.
|
||||||
if !quietFlag {
|
if !quietFlag {
|
||||||
mode := ""
|
mode := globalMinioModeGatewayPrefix + gatewayName
|
||||||
switch gatewayBackend(backendType) {
|
|
||||||
case azureBackend:
|
|
||||||
mode = globalMinioModeGatewayAzure
|
|
||||||
case gcsBackend:
|
|
||||||
mode = globalMinioModeGatewayGCS
|
|
||||||
case s3Backend:
|
|
||||||
mode = globalMinioModeGatewayS3
|
|
||||||
case b2Backend:
|
|
||||||
mode = globalMinioModeGatewayB2
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check update mode.
|
// Check update mode.
|
||||||
checkUpdate(mode)
|
checkUpdate(mode)
|
||||||
|
|
||||||
// Print gateway startup message.
|
// Print gateway startup message.
|
||||||
printGatewayStartupMessage(getAPIEndpoints(gatewayAddr), backendType)
|
printGatewayStartupMessage(getAPIEndpoints(gatewayAddr), gatewayName)
|
||||||
}
|
}
|
||||||
|
|
||||||
handleSignals()
|
handleSignals()
|
||||||
|
|
|
@ -19,8 +19,31 @@ package cmd
|
||||||
import (
|
import (
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/minio/cli"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Test RegisterGatewayCommand
|
||||||
|
func TestRegisterGatewayCommand(t *testing.T) {
|
||||||
|
var err error
|
||||||
|
|
||||||
|
cmd := cli.Command{Name: "test"}
|
||||||
|
err = RegisterGatewayCommand(cmd)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("RegisterGatewayCommand got unexpected error: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Should returns 'duplicated' error
|
||||||
|
err = RegisterGatewayCommand(cmd)
|
||||||
|
if err == nil {
|
||||||
|
t.Errorf("RegisterGatewayCommand twice with same name should return error")
|
||||||
|
} else {
|
||||||
|
if err.Error() != "duplicate gateway: test" {
|
||||||
|
t.Errorf("RegisterGatewayCommand got unexpected error: %s", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Test parseGatewayEndpoint
|
// Test parseGatewayEndpoint
|
||||||
func TestParseGatewayEndpoint(t *testing.T) {
|
func TestParseGatewayEndpoint(t *testing.T) {
|
||||||
testCases := []struct {
|
testCases := []struct {
|
||||||
|
|
|
@ -20,12 +20,85 @@ import (
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
|
"github.com/minio/cli"
|
||||||
minio "github.com/minio/minio-go"
|
minio "github.com/minio/minio-go"
|
||||||
"github.com/minio/minio-go/pkg/policy"
|
"github.com/minio/minio-go/pkg/policy"
|
||||||
"github.com/minio/minio-go/pkg/s3utils"
|
"github.com/minio/minio-go/pkg/s3utils"
|
||||||
"github.com/minio/minio/pkg/hash"
|
"github.com/minio/minio/pkg/hash"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
s3Backend = "s3"
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
const s3GatewayTemplate = `NAME:
|
||||||
|
{{.HelpName}} - {{.Usage}}
|
||||||
|
|
||||||
|
USAGE:
|
||||||
|
{{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [ENDPOINT]
|
||||||
|
{{if .VisibleFlags}}
|
||||||
|
FLAGS:
|
||||||
|
{{range .VisibleFlags}}{{.}}
|
||||||
|
{{end}}{{end}}
|
||||||
|
ENDPOINT:
|
||||||
|
S3 server endpoint. Default ENDPOINT is https://s3.amazonaws.com
|
||||||
|
|
||||||
|
ENVIRONMENT VARIABLES:
|
||||||
|
ACCESS:
|
||||||
|
MINIO_ACCESS_KEY: Username or access key of S3 storage.
|
||||||
|
MINIO_SECRET_KEY: Password or secret key of S3 storage.
|
||||||
|
|
||||||
|
BROWSER:
|
||||||
|
MINIO_BROWSER: To disable web browser access, set this value to "off".
|
||||||
|
|
||||||
|
EXAMPLES:
|
||||||
|
1. Start minio gateway server for AWS S3 backend.
|
||||||
|
$ export MINIO_ACCESS_KEY=accesskey
|
||||||
|
$ export MINIO_SECRET_KEY=secretkey
|
||||||
|
$ {{.HelpName}}
|
||||||
|
|
||||||
|
2. Start minio gateway server for S3 backend on custom endpoint.
|
||||||
|
$ export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
|
||||||
|
$ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
|
||||||
|
$ {{.HelpName}} https://play.minio.io:9000
|
||||||
|
`
|
||||||
|
|
||||||
|
MustRegisterGatewayCommand(cli.Command{
|
||||||
|
Name: s3Backend,
|
||||||
|
Usage: "Amazon Simple Storage Service (S3).",
|
||||||
|
Action: s3GatewayMain,
|
||||||
|
CustomHelpTemplate: s3GatewayTemplate,
|
||||||
|
Flags: append(serverFlags, globalFlags...),
|
||||||
|
HideHelpCommand: true,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handler for 'minio gateway s3' command line.
|
||||||
|
func s3GatewayMain(ctx *cli.Context) {
|
||||||
|
// Validate gateway arguments.
|
||||||
|
host := ctx.Args().First()
|
||||||
|
// Validate gateway arguments.
|
||||||
|
fatalIf(validateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
|
||||||
|
|
||||||
|
startGateway(ctx, &S3Gateway{host})
|
||||||
|
}
|
||||||
|
|
||||||
|
// S3Gateway implements Gateway.
|
||||||
|
type S3Gateway struct {
|
||||||
|
host string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name implements Gateway interface.
|
||||||
|
func (g *S3Gateway) Name() string {
|
||||||
|
return s3Backend
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewGatewayLayer returns s3 gatewaylayer.
|
||||||
|
func (g *S3Gateway) NewGatewayLayer() (GatewayLayer, error) {
|
||||||
|
return newS3GatewayLayer(g.host)
|
||||||
|
}
|
||||||
|
|
||||||
// s3ToObjectError converts Minio errors to minio object layer errors.
|
// s3ToObjectError converts Minio errors to minio object layer errors.
|
||||||
func s3ToObjectError(err error, params ...string) error {
|
func s3ToObjectError(err error, params ...string) error {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
|
@ -101,8 +174,8 @@ type s3Objects struct {
|
||||||
anonClient *minio.Core
|
anonClient *minio.Core
|
||||||
}
|
}
|
||||||
|
|
||||||
// newS3Gateway returns s3 gatewaylayer
|
// newS3GatewayLayer returns s3 gatewaylayer
|
||||||
func newS3Gateway(host string) (GatewayLayer, error) {
|
func newS3GatewayLayer(host string) (GatewayLayer, error) {
|
||||||
var err error
|
var err error
|
||||||
var endpoint string
|
var endpoint string
|
||||||
var secure = true
|
var secure = true
|
||||||
|
|
|
@ -22,7 +22,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
// Prints the formatted startup message.
|
// Prints the formatted startup message.
|
||||||
func printGatewayStartupMessage(apiEndPoints []string, backendType gatewayBackend) {
|
func printGatewayStartupMessage(apiEndPoints []string, backendType string) {
|
||||||
strippedAPIEndpoints := stripStandardPorts(apiEndPoints)
|
strippedAPIEndpoints := stripStandardPorts(apiEndPoints)
|
||||||
|
|
||||||
// Prints credential.
|
// Prints credential.
|
||||||
|
|
|
@ -50,10 +50,7 @@ const (
|
||||||
globalMinioModeFS = "mode-server-fs"
|
globalMinioModeFS = "mode-server-fs"
|
||||||
globalMinioModeXL = "mode-server-xl"
|
globalMinioModeXL = "mode-server-xl"
|
||||||
globalMinioModeDistXL = "mode-server-distributed-xl"
|
globalMinioModeDistXL = "mode-server-distributed-xl"
|
||||||
globalMinioModeGatewayAzure = "mode-gateway-azure"
|
globalMinioModeGatewayPrefix = "mode-gateway-"
|
||||||
globalMinioModeGatewayS3 = "mode-gateway-s3"
|
|
||||||
globalMinioModeGatewayGCS = "mode-gateway-gcs"
|
|
||||||
globalMinioModeGatewayB2 = "mode-gateway-b2"
|
|
||||||
|
|
||||||
// globalMinioSysTmp prefix is used in Azure/GCS gateway for save metadata sent by Initialize Multipart Upload API.
|
// globalMinioSysTmp prefix is used in Azure/GCS gateway for save metadata sent by Initialize Multipart Upload API.
|
||||||
globalMinioSysTmp = "minio.sys.tmp/"
|
globalMinioSysTmp = "minio.sys.tmp/"
|
||||||
|
|
|
@ -728,20 +728,10 @@ func readBucketAccessPolicy(objAPI ObjectLayer, bucketName string) (policy.Bucke
|
||||||
|
|
||||||
func getBucketAccessPolicy(objAPI ObjectLayer, bucketName string) (policy.BucketAccessPolicy, error) {
|
func getBucketAccessPolicy(objAPI ObjectLayer, bucketName string) (policy.BucketAccessPolicy, error) {
|
||||||
// FIXME: remove this code when S3 layer for gateway and server is unified.
|
// FIXME: remove this code when S3 layer for gateway and server is unified.
|
||||||
var policyInfo policy.BucketAccessPolicy
|
if layer, ok := objAPI.(GatewayLayer); ok {
|
||||||
var err error
|
return layer.GetBucketPolicies(bucketName)
|
||||||
|
|
||||||
switch layer := objAPI.(type) {
|
|
||||||
case *s3Objects:
|
|
||||||
policyInfo, err = layer.GetBucketPolicies(bucketName)
|
|
||||||
case *azureObjects:
|
|
||||||
policyInfo, err = layer.GetBucketPolicies(bucketName)
|
|
||||||
case *gcsGateway:
|
|
||||||
policyInfo, err = layer.GetBucketPolicies(bucketName)
|
|
||||||
default:
|
|
||||||
policyInfo, err = readBucketAccessPolicy(objAPI, bucketName)
|
|
||||||
}
|
}
|
||||||
return policyInfo, err
|
return readBucketAccessPolicy(objAPI, bucketName)
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetBucketPolicy - get bucket policy for the requested prefix.
|
// GetBucketPolicy - get bucket policy for the requested prefix.
|
||||||
|
|
Loading…
Reference in New Issue