Add support for server side bucket replication (#9882)

pkg/madmin/examples/bucket-replication.go

@@ -0,0 +1,61 @@
+// +build ignore
+
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package main
+
+import (
+	"context"
+	"log"
+
+	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/madmin"
+)
+
+func main() {
+	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are
+	// dummy values, please replace them with original values.
+
+	// API requests are secure (HTTPS) if secure=true and insecure (HTTP) otherwise.
+	// New returns an MinIO Admin client object.
+	madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	ctx := context.Background()
+	creds, err := auth.CreateCredentials("access-key", "secret-key")
+	if err != nil {
+		log.Fatalln(err)
+	}
+	target := madmin.BucketReplicationTarget{Endpoint: "site2:9000", Credentials: creds, TargetBucket: "destbucket", IsSSL: false}
+	// Set bucket replication target
+	if err := madmClnt.SetBucketReplicationTarget(ctx, "srcbucket", &target); err != nil {
+		log.Fatalln(err)
+	}
+	// Get bucket replication target
+	target, err = madmClnt.GetBucketReplicationTarget(ctx, "srcbucket")
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	// Remove bucket replication target
+	if err := madmClnt.SetBucketReplicationTarget(ctx, "srcbucket", nil); err != nil {
+		log.Fatalln(err)
+	}
+
+}

pkg/madmin/replication-commands.go

@@ -0,0 +1,163 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package madmin
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+
+	"github.com/minio/minio/pkg/auth"
+)
+
+// BucketReplicationTarget represents the target bucket and site to be replicated to.
+type BucketReplicationTarget struct {
+	Endpoint     string            `json:"endpoint"`
+	Credentials  *auth.Credentials `json:"credentials"`
+	TargetBucket string            `json:"targetbucket"`
+	IsSSL        bool              `json:"isssl"`
+	Path         string            `json:"path,omitempty"`
+	API          string            `json:"api,omitempty"`
+	Arn          string            `json:"arn,omitempty"`
+}
+
+// URL returns replication target url
+func (t BucketReplicationTarget) URL() string {
+	scheme := "http"
+	if t.IsSSL {
+		scheme = "https"
+	}
+	return fmt.Sprintf("%s://%s", scheme, t.Endpoint)
+}
+
+// Empty returns true if struct is empty.
+func (t BucketReplicationTarget) Empty() bool {
+	return t.String() == "" || t.Credentials == nil
+}
+
+func (t *BucketReplicationTarget) String() string {
+	return fmt.Sprintf("%s %s", t.Endpoint, t.TargetBucket)
+}
+
+// GetBucketReplicationTarget - gets replication target for this bucket
+func (adm *AdminClient) GetBucketReplicationTarget(ctx context.Context, bucket string) (target BucketReplicationTarget, err error) {
+	queryValues := url.Values{}
+	queryValues.Set("bucket", bucket)
+
+	reqData := requestData{
+		relPath:     adminAPIPrefix + "/get-bucket-replication-target",
+		queryValues: queryValues,
+	}
+
+	// Execute GET on /minio/admin/v3/get-bucket-replication-target
+	resp, err := adm.executeMethod(ctx, http.MethodGet, reqData)
+
+	defer closeResponse(resp)
+	if err != nil {
+		return target, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return target, httpRespToErrorResponse(resp)
+	}
+
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return target, err
+	}
+	if err = json.Unmarshal(b, &target); err != nil {
+		return target, err
+	}
+	if target.Empty() {
+		return target, errors.New("No Replication target configured")
+	}
+	return target, nil
+}
+
+// SetBucketReplicationTarget sets up a replication target for this bucket
+func (adm *AdminClient) SetBucketReplicationTarget(ctx context.Context, bucket string, target *BucketReplicationTarget) error {
+	data, err := json.Marshal(target)
+	if err != nil {
+		return err
+	}
+	encData, err := EncryptData(adm.getSecretKey(), data)
+	if err != nil {
+		return err
+	}
+	queryValues := url.Values{}
+	queryValues.Set("bucket", bucket)
+
+	reqData := requestData{
+		relPath:     adminAPIPrefix + "/set-bucket-replication-target",
+		queryValues: queryValues,
+		content:     encData,
+	}
+
+	// Execute PUT on /minio/admin/v3/set-bucket-replication-target to set a replication target for this bucket.
+	resp, err := adm.executeMethod(ctx, http.MethodPut, reqData)
+
+	defer closeResponse(resp)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return httpRespToErrorResponse(resp)
+	}
+
+	return nil
+}
+
+// GetBucketReplicationARN - gets replication Arn for this remote
+func (adm *AdminClient) GetBucketReplicationARN(ctx context.Context, rURL string) (arn string, err error) {
+	queryValues := url.Values{}
+	queryValues.Set("url", rURL)
+
+	reqData := requestData{
+		relPath:     adminAPIPrefix + "/get-bucket-replication-arn",
+		queryValues: queryValues,
+	}
+
+	// Execute GET on /minio/admin/v3/list-bucket-replication-arn
+	resp, err := adm.executeMethod(ctx, http.MethodGet, reqData)
+
+	defer closeResponse(resp)
+	if err != nil {
+		return arn, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return arn, httpRespToErrorResponse(resp)
+	}
+
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return arn, err
+	}
+	if err = json.Unmarshal(b, &arn); err != nil {
+		return arn, err
+	}
+	if arn == "" {
+		return arn, fmt.Errorf("Missing Replication ARN")
+	}
+	return arn, nil
+}