MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-24 06:05:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add access format support for Redis notification target (#3989)

Browse Source 
This change adds `access` format support for notifications to a Redis
server, and it refactors `namespace` format support.

In the case of `access` format, a list is used to store Minio
operations in Redis. Each entry in the list is a JSON encoded list of
two items - the first is the Minio server timestamp of the event, and
the second is an object describing the operation that created/replaced
the object in the server.

In the case of `namespace` format, a hash is used. Entries in the hash
may be updated or removed if objects in Minio are updated or deleted
respectively. The field values in the Redis hash are JSON encoded.

Also updates documentation on Redis notification target usage.

Towards resolving #3928
This commit is contained in:
Aditya Manthramurthy 2017-03-29 21:25:53 +05:30 committed by Harshavardhana
parent 1caad902cb
commit 61b08137b0
2 changed files with 122 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
cmd/notify-redis.go
View File

@ -17,6 +17,7 @@
package cmd package cmd
import ( import (
"encoding/json"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"time" "time"
@ -25,6 +26,16 @@ import (
"github.com/garyburd/redigo/redis" "github.com/garyburd/redigo/redis"
) )
func makeRedisError(msg string, a ...interface{}) error {
s := fmt.Sprintf(msg, a...)
return fmt.Errorf("Redis Notifier Error: %s", s)
}
var (
rdNFormatError = makeRedisError(`"format" value is invalid - it must be one of "access" or "namespace".`)
rdNKeyError = makeRedisError("Key was not specified in the configuration.")
)
// redisNotify to send logs to Redis server // redisNotify to send logs to Redis server
type redisNotify struct { type redisNotify struct {
Enable bool `json:"enable"` Enable bool `json:"enable"`
@ -38,14 +49,15 @@ func (r *redisNotify) Validate() error {
if !r.Enable { if !r.Enable {
return nil return nil
} }
if r.Format != formatNamespace { if r.Format != formatNamespace && r.Format != formatAccess {
return fmt.Errorf( return rdNFormatError
"Redis Notifier Error: \"format\" must be \"%s\"",
formatNamespace)
} }
if _, err := checkURL(r.Addr); err != nil { if _, err := checkURL(r.Addr); err != nil {
return err return err
} }
if r.Key == "" {
return rdNKeyError
}
return nil return nil
} }
@ -92,7 +104,25 @@ func dialRedis(rNotify redisNotify) (*redis.Pool, error) {
// Check connection. // Check connection.
_, err := rConn.Do("PING") _, err := rConn.Do("PING")
if err != nil { if err != nil {
return nil, err return nil, makeRedisError("Error connecting to server: %v", err)
}
// Test that Key is of desired type
reply, err := redis.String(rConn.Do("TYPE", rNotify.Key))
if err != nil {
return nil, makeRedisError("Error getting type of Key=%s: %v",
rNotify.Key, err)
}
if reply != "none" {
expectedType := "hash"
if rNotify.Format == formatAccess {
expectedType = "list"
}
if reply != expectedType {
return nil, makeRedisError(
"Key=%s has type %s, but we expect it to be a %s",
rNotify.Key, reply, expectedType)
}
} }
// Return pool. // Return pool.
@ -137,17 +167,51 @@ func (r redisConn) Fire(entry *logrus.Entry) error {
return nil return nil
} }
// Match the event if its a delete request, attempt to delete the key switch r.params.Format {
if eventMatch(entryStr, []string{"s3:ObjectRemoved:*"}) { case formatNamespace:
if _, err := rConn.Do("DEL", entry.Data["Key"]); err != nil { // Match the event if its a delete request, attempt to delete the key
return err if eventMatch(entryStr, []string{"s3:ObjectRemoved:*"}) {
_, err := rConn.Do("HDEL", r.params.Key, entry.Data["Key"])
if err != nil {
return makeRedisError("Error deleting entry: %v",
err)
}
return nil
} // else save this as new entry or update any existing ones.
value, err := json.Marshal(map[string]interface{}{
"Records": entry.Data["Records"],
})
if err != nil {
return makeRedisError(
"Unable to encode event %v to JSON: %v",
entry.Data["Records"], err)
}
_, err = rConn.Do("HSET", r.params.Key, entry.Data["Key"],
value)
if err != nil {
return makeRedisError("Error updating hash entry: %v",
err)
}
case formatAccess:
// eventTime is taken from the first entry in the
// records.
events, ok := entry.Data["Records"].([]NotificationEvent)
if !ok {
return makeRedisError("unable to extract event time due to conversion error of entry.Data[\"Records\"]=%v", entry.Data["Records"])
}
eventTime := events[0].EventTime
listEntry := []interface{}{eventTime, entry.Data["Records"]}
jsonValue, err := json.Marshal(listEntry)
if err != nil {
return makeRedisError("JSON encoding error: %v", err)
}
_, err = rConn.Do("RPUSH", r.params.Key, jsonValue)
if err != nil {
return makeRedisError("Error appending to Redis list: %v",
err)
} }
return nil
} // else save this as new entry or update any existing ones.
if _, err := rConn.Do("SET", entry.Data["Key"], map[string]interface{}{
"Records": entry.Data["Records"],
}); err != nil {
return err
} }
return nil return nil
} }

56
docs/bucket/notifications/README.md
View File

@ -223,28 +223,53 @@ curl -XGET '127.0.0.1:9200/bucketevents/_search?pretty=1'
<a name="Redis"></a> <a name="Redis"></a>
## Publish Minio events via Redis ## Publish Minio events via Redis
Install Redis from [here](http://redis.io/download). Install [Redis](http://redis.io/download) server. For illustrative purposes, we have set the database password as "yoursecret".
This notification target supports two formats: _namespace_ and _access_.
When the _namespace_ format is used, Minio synchronizes objects in the bucket with entries in a hash. For each entry, the key is formatted as "bucketName/objectName" for an object that exists in the bucket, and the value is the JSON-encoded event data about the operation that created/replaced the object in Minio. When objects are updated or deleted, the corresponding entry int he hash is updated or deleted respectively.
When the _access_ format is used, Minio appends events to a list using [RPUSH](https://redis.io/commands/rpush). Each item in the list is a JSON encoded list with two items, where the first item is a timestamp string, and second item is a JSON object containing evnet data about the operation that happened in the bucket. No entries appended to the list are updated or deleted by Minio in this format.
The steps below show how to use this notification target in `namespace` and `access` format.
### Step 1: Add Redis endpoint to Minio ### Step 1: Add Redis endpoint to Minio
The default location of Minio server configuration file is ``~/.minio/config.json``. Update the Redis configuration block in ``config.json`` as follows: The default location of Minio server configuration file is ``~/.minio/config.json``. The Redis configuration is located in the `redis` key under the `notify` top-level key. Create a configuration key-value pair here for your Redis instance. The key is a name for your Redis endpoint, and the value is a collection of key-value parameters described in the table below.
| Parameter | Type | Description |
|:---|:---|:---|
| `enable` | _bool_ | (Required) Is this server endpoint configuration active/enabled? |
| `format` | _string_ | (Required) Either `namespace` or `access`. |
| `address` | _string_ | (Required) The Redis server's address. For example: `localhost:6379`. |
| `password` | _string_ | (Optional) The Redis server's password. |
| `key` | _string_ | (Required) The name of the redis key under which events are stored. A hash is used in case of `namespace` format and a list in case of `access` format.|
An example of Redis configuration is as follows:
```json ```json
"redis": { "redis": {
"1": { "1": {
"enable": true, "enable": true,
"address": "127.0.0.1:6379", "address": "127.0.0.1:6379",
"password": "yoursecret", "password": "yoursecret",
"key": "bucketevents" "key": "bucketevents"
} }
} }
``` ```
Restart Minio server to reflect config changes. ``bucketevents`` is the key used by Redis in this example. After updating the configuration file, restart the Minio server to put the changes into effect. The server will print a line like `SQS ARNs: arn:minio:sqs:us-east-1:1:redis` at start-up if there were no errors.
Note that, you can add as many Redis server endpoint configurations as needed by providing an identifier (like "1" in the example above) for the Redis instance and an object of per-server configuration parameters.
### Step 2: Enable bucket notification using Minio client ### Step 2: Enable bucket notification using Minio client
We will enable bucket event notification to trigger whenever a JPEG image is uploaded or deleted from ``images`` bucket on ``myminio`` server. Here ARN value is ``arn:minio:sqs:us-east-1:1:redis``. To understand more about ARN please follow [AWS ARN](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) documentation. We will now enable bucket event notifications on a bucket named `images`. Whenever a JPEG image is created/overwritten, a new key is added or an existing key is updated in the Redis hash configured above. When an existing object is deleted, the corresponding key is deleted from the Redis hash. Thus, the rows in the Redis hash, reflect the `.jpg` objects in the `images` bucket.
To configure this bucket notification, we need the ARN printed by Minio in the previous step. Additional information about ARN is available [here](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html).
With the `mc` tool, the configuration is very simple to add. Let us say that the Minio server is aliased as `myminio` in our mc configuration. Execute the following:
``` ```
mc mb myminio/images mc mb myminio/images
@ -255,10 +280,12 @@ arn:minio:sqs:us-east-1:1:redis s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: su
### Step 3: Test on Redis ### Step 3: Test on Redis
Redis comes with a handy command line interface ``redis-cli`` to print all notifications on the console. Start the `redis-cli` Redis client program to inspect the contents in Redis. Run the `monitor` Redis command. This prints each operation performed on Redis as it occurs.
``` ```
redis-cli -a yoursecret redis-cli -a yoursecret
127.0.0.1:6379> monitor
OK
``` ```
Open another terminal and upload a JPEG image into ``images`` bucket. Open another terminal and upload a JPEG image into ``images`` bucket.
@ -267,16 +294,19 @@ Open another terminal and upload a JPEG image into ``images`` bucket.
mc cp myphoto.jpg myminio/images mc cp myphoto.jpg myminio/images
``` ```
``redis-cli`` prints event notification to the console. In the previous terminal, you will now see the operation that Minio performs on Redis:
``` ```
redis-cli -a yoursecret
127.0.0.1:6379> monitor 127.0.0.1:6379> monitor
OK OK
1474321638.556108 [0 127.0.0.1:40190] "AUTH" "yoursecret" 1490686879.650649 [0 172.17.0.1:44710] "PING"
1474321638.556477 [0 127.0.0.1:40190] "RPUSH" "bucketevents" "{\"Records\":[{\"eventVersion\":\"2.0\",\"eventSource\":\"aws:s3\",\"awsRegion\":\"us-east-1\",\"eventTime\":\"2016-09-19T21:47:18.555Z\",\"eventName\":\"s3:ObjectCreated:Put\",\"userIdentity\":{\"principalId\":\"minio\"},\"requestParameters\":{\"sourceIPAddress\":\"[::1]:39250\"},\"responseElements\":{},\"s3\":{\"s3SchemaVersion\":\"1.0\",\"configurationId\":\"Config\",\"bucket\":{\"name\":\"images\",\"ownerIdentity\":{\"principalId\":\"minio\"},\"arn\":\"arn:aws:s3:::images\"},\"object\":{\"key\":\"myphoto.jpg\",\"size\":23745,\"sequencer\":\"1475D7B80ECBD853\"}}}],\"level\":\"info\",\"msg\":\"\",\"time\":\"2016-09-19T14:47:18-07:00\"}\n" 1490686879.651061 [0 172.17.0.1:44710] "HSET" "minio_events" "images/myphoto.jpg" "{\"Records\":[{\"eventVersion\":\"2.0\",\"eventSource\":\"minio:s3\",\"awsRegion\":\"us-east-1\",\"eventTime\":\"2017-03-28T07:41:19Z\",\"eventName\":\"s3:ObjectCreated:Put\",\"userIdentity\":{\"principalId\":\"minio\"},\"requestParameters\":{\"sourceIPAddress\":\"127.0.0.1:52234\"},\"responseElements\":{\"x-amz-request-id\":\"14AFFBD1ACE5F632\",\"x-minio-origin-endpoint\":\"http://192.168.86.115:9000\"},\"s3\":{\"s3SchemaVersion\":\"1.0\",\"configurationId\":\"Config\",\"bucket\":{\"name\":\"images\",\"ownerIdentity\":{\"principalId\":\"minio\"},\"arn\":\"arn:aws:s3:::images\"},\"object\":{\"key\":\"myphoto.jpg\",\"size\":2586,\"eTag\":\"5d284463f9da279f060f0ea4d11af098\",\"sequencer\":\"14AFFBD1ACE5F632\"}},\"source\":{\"host\":\"127.0.0.1\",\"port\":\"52234\",\"userAgent\":\"Minio (linux; amd64) minio-go/2.0.3 mc/2017-02-15T17:57:25Z\"}}]}"
``` ```
Here we see that Minio performed `HSET` on `minio_events` key.
In case, `access` format was used, then `minio_events` would be a list, and the Minio server would have performed an `RPUSH` to append to the list. A consumer of this list would ideally use `BLPOP` to remove list items from the left-end of the list.
<a name="NATS"></a> <a name="NATS"></a>
## Publish Minio events via NATS ## Publish Minio events via NATS