docs, tweaks

LICENSE

@@ -0,0 +1,19 @@
+Copyright (C) 2015 by Patrick Stadler
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

README.md

@@ -124,6 +124,131 @@ network_eth1.out: 0.03
 ...
 ```
 
-## Writing custom metrics / reporters
+## Writing custom metrics and reporters
 
-TODO
+metrics.sh provides a simple interface based on hooks for writing custom metrics and reporters. Each hook is optional and only needs to be implemented if necessary. In order for metrics.sh to find and load custom metrics, they have to be placed in `./metrics/custom` or wherever `CUSTOM_METRICS_PATH` is pointing to. The same applies to custom reporters, whose default location is `./reporters/custom` or any folder specified by `CUSTOM_REPORTERS_PATH`.
+
+### Custom metrics
+
+```sh
+# Hooks for metrics in order of execution
+defaults () {}  # setting default variables
+start () {}     # called at the beginning
+collect () {}   # collect the actual metric
+stop () {}      # called before exiting
+docs () {}      # used for priting docs and creating output for configuration
+```
+
+Metrics run within an isolated scope. It's generally safe to create variables and helper functions within metrics.
+
+Below is an example script for monitoring the size of a specified folder. Assuming this script is located at `./metrics/custom/dir_size.sh`, it can be invoked by calling `./metrics.sh -m dir_size`.
+
+```sh
+#!/bin/sh
+
+# Set default values. This function should never fail.
+defaults () {
+  if [ -z $DIR_SIZE_PATH ]; then
+    DIR_SIZE_PATH="."
+  fi
+  if [ -z $DIR_SIZE_IN_MB ]; then
+    DIR_SIZE_IN_MB=false
+  fi
+}
+
+# Prepare the collector. Create helper functions to be used during collection
+# if needed. Returning 1 will disable this metric and report a warning.
+start () {
+  if [ $DIR_SIZE_IN_MB = false ]; then
+    DU_ARGS="-s -m $DIR_SIZE_PATH"
+  else
+    DU_ARGS="-s -k $DIR_SIZE_PATH"
+  fi
+}
+
+# Collect actual metric. This function is called every N seconds.
+collect () {
+  # Calling `report $val` will check if the value is a number (int or float)
+  # and then send it over to the reporter's report() function, together with
+  # the name of the metric, in this case "dir_size" if no alias is used.
+  report $(du $DU_ARGS | awk '{ print $1 }')
+  # If report is called with two arguments, the first one will be appended
+  # to the metric name, for example `report "foo" $val` would be reported as
+  # "dir_size.foo: $val". This is helpful when a metric is collecting multiple 
+  # values like `network_io`, which reports "network_io.in" / "network_io.out".
+}
+
+# Stop is not needed for this metric, there's nothing to clean up.
+# stop () {}
+
+# The output of this function is shown when calling `metrics.sh`
+# with `--docs` and is even more helpful when creating configuration
+# files with `--print-config`.
+docs () {
+  echo "Monitor size of a specific folder in Kb or Mb."
+  echo "DIR_SIZE_PATH=$DIR_SIZE_PATH"
+  echo "DIR_SIZE_REPORT_MB=$DIR_SIZE_IN_MB"
+}
+```
+
+### Custom reporters
+
+```sh
+# Hooks for reporters in order of execution
+defaults () {}  # setting default variables
+start () {}     # called at the beginning
+report () {}    # report the actual metric
+stop () {}      # called before exiting
+docs () {}      # used for priting docs and creating output for configuration
+```
+
+Below is an example script for sending metrics as JSON data to an API endpoint. Assuming this script is located at `./reporters/custom/json_api.sh`, it can be invoked by calling `./metrics.sh -r json_api`.
+
+```sh
+#!/bin/sh
+
+# Set default values. This function should never fail.
+defaults () {
+  if [ -z $JSON_API_METHOD ]; then
+    JSON_API_METHOD="POST"
+  fi
+}
+
+# Prepare the reporter. Create helper functions to be used during collection
+# if needed. Returning 1 will result in an error and exection will be stopped.
+start () {
+  if [ -z $JSON_API_ENDPOINT ]; then
+    echo "Error: json_api requires \$JSON_API_ENDPOINT to be specified"
+    return 1
+  fi
+}
+
+# Report metric. This function is called whenever there's a new value
+# to report. It's important to know that metrics don't call this function
+# directly, as there's some more work to be done before. You can safely assume 
+# that arguments passed to this function are sanitized and valid.
+report () {
+  local metric=$1 # the name of the metric, e.g. "cpu", "cpu_alias", "cpu.foo"
+  local value=$2  # int or float
+  curl -s -H "Content-Type: application/json" $JSON_API_ENDPOINT \
+       -X $JSON_API_METHOD -d "{\"metric\":\"$metric\",\"value\":$value}"
+}
+
+# Stop is not needed here, there's nothing to clean up.
+# stop () {}
+
+# The output of this function is shown when calling `metrics.sh`
+# with `--docs` and is even more helpful when creating configuration
+# files with `--print-config`.
+docs () {
+  echo "Send data as JSON to an API endpoint."
+  echo "JSON_API_ENDPOINT=$JSON_API_ENDPOINT"
+  echo "JSON_API_METHOD=$JSON_API_METHOD"
+}
+```
+
+## Roadmap
+
+- Test and improve init.d script and write docs for it
+- Implement StatsD reporter
+- Tests

lib/main.sh

@@ -68,7 +68,7 @@ main_collect () {
   trap '
     trap "" 13
     trap - INT TERM EXIT
-    echo Exit signal received.
+    echo Exit signal received, stopping...
     kill -13 -$$
   ' 13 INT TERM EXIT
 
@@ -88,6 +88,10 @@ main_collect () {
   verbose "Starting reporter '${reporter_alias}'"
   if is_function __r_${reporter_alias}_start; then
     __r_${reporter_alias}_start
+    if [ $? -ne 0 ]; then
+      echo "Error: failed to start reporter '${reporter_alias}'"
+      exit 1
+    fi
   fi
 
   # collect metrics
@@ -111,6 +115,10 @@ main_collect () {
       verbose "Starting metric '${metric_alias}'"
       if is_function __m_${metric_alias}_start; then
         __m_${metric_alias}_start
+        if [ $? -ne 0 ]; then
+          echo "Warning: metric '${metric_alias}' is disabled after failing to start"
+          continue
+        fi
       fi
 
       if ! is_function __m_${metric_alias}_collect; then

metrics/disk_io.sh

@@ -11,33 +11,33 @@ defaults () {
 }
 
 start () {
-  readonly __disk_io_fifo=$TEMP_DIR/$(unique_id)
-  mkfifo $__disk_io_fifo
+  readonly fifo=$TEMP_DIR/$(unique_id)
+  mkfifo $fifo
 
   if is_osx; then
-    __disk_io_bgproc () {
+    bg_proc () {
       iostat -K -d -w $INTERVAL $DISK_IO_MOUNTPOINT | while read line; do
-        echo $line | awk '{ print $3 }' > $__disk_io_fifo
+        echo $line | awk '{ print $3 }' > $fifo
       done
     }
   else
-    __disk_io_bgproc () {
+    bg_proc () {
       iostat -y -m -d $INTERVAL $DISK_IO_MOUNTPOINT | while read line; do
-        echo $line | awk '/[0-9.]/{ print $3 }' > $__disk_io_fifo
+        echo $line | awk '/[0-9.]/{ print $3 }' > $fifo
       done
     }
   fi
 
-  __disk_io_bgproc &
+  bg_proc &
 }
 
 collect () {
-  report $(cat $__disk_io_fifo)
+  report $(cat $fifo)
 }
 
 stop () {
-  if [ ! -z $__disk_io_fifo ] && [ -p $__disk_io_fifo ]; then
-    rm $__disk_io_fifo
+  if [ ! -z $fifo ] && [ -p $fifo ]; then
+    rm $fifo
   fi
 }
 

metrics/network_io.sh

@@ -11,35 +11,35 @@ defaults () {
 }
 
 start () {
-  readonly __network_io_divisor=$(($INTERVAL * 1024))
+  readonly divisor=$(($INTERVAL * 1024))
 
   if is_osx; then
-    get_netstat () {
+    get_sample () {
       netstat -b -I $NETWORK_IO_INTERFACE | awk '{ print $7" "$10 }' | tail -n 1
     }
   else
-    get_netstat () {
+    get_sample () {
       cat /proc/net/dev | awk -v iface_regex="$NETWORK_IO_INTERFACE:" \
                                       '$0 ~ iface_regex { print $2" "$10 }'
     }
   fi
 
   calc_kBps() {
-    echo $1 $2 | awk -v divisor=$__network_io_divisor \
+    echo $1 $2 | awk -v divisor=$divisor \
                 '{ printf "%.2f", ($1 - $2) / divisor }'
   }
 }
 
 collect () {
   local sample
-  sample=$(get_netstat)
-  if [ ! -z "$__network_io_sample" ]; then
+  sample=$(get_sample)
+  if [ ! -z "$previous_sample" ]; then
     report "in" $(calc_kBps $(echo $sample | awk '{print $1}') \
-                                $(echo $__network_io_sample | awk '{print $1}'))
+                                $(echo $previous_sample | awk '{print $1}'))
     report "out" $(calc_kBps $(echo $sample | awk '{print $2}') \
-                                $(echo $__network_io_sample | awk '{print $2}'))
+                                $(echo $previous_sample | awk '{print $2}'))
   fi
-  __network_io_sample="$sample"
+  previous_sample="$sample"
 }
 
 docs () {

metrics/ping.sh

@@ -2,8 +2,8 @@
 
 start () {
   if [ -z $PING_REMOTE_HOST ]; then
-    echo "Error: ping metric requires \$PING_REMOTE_HOST to be specified"
-    exit 1
+    echo "Warning: ping requires \$PING_REMOTE_HOST to be specified"
+    return 1
   fi
 }
 

reporters/file.sh

@@ -3,7 +3,7 @@
 start () {
   if [ -z $FILE_LOCATION ]; then
     echo "Error: file reporter requires \$FILE_LOCATION to be specified"
-    exit 1
+    return 1
   fi
 
   if [ ! -f $FILE_LOCATION ]; then

reporters/influxdb.sh

@@ -9,7 +9,7 @@ defaults () {
 start () {
   if [ -z $INFLUXDB_API_ENDPOINT ]; then
     echo "Error: influxdb requires \$INFLUXDB_API_ENDPOINT to be specified"
-    exit 1
+    return 1
   fi
 
   if [ "$INFLUXDB_SEND_HOSTNAME" = true ]; then
@@ -29,9 +29,8 @@ report () {
   else
     points="[$value]"
   fi
-
-  curl -X POST $INFLUXDB_API_ENDPOINT \
-    -d "[{\"name\":\"$metric\",\"columns\":$__influxdb_columns,\"points\":[$points]}]"
+  local data="[{\"name\":\"$metric\",\"columns\":$__influxdb_columns,\"points\":[$points]}]"
+  curl -s -X POST $INFLUXDB_API_ENDPOINT -d $data
 }
 
 docs () {

reporters/keen_io.sh

@@ -9,12 +9,12 @@ defaults () {
 start() {
   if [ -z $KEEN_IO_PROJECT_ID ]; then
     echo "Error: keen_io requires \$KEEN_IO_PROJECT_ID to be specified"
-    exit 1
+    return 1
   fi
 
   if [ -z $KEEN_IO_WRITE_KEY ]; then
     echo "Error: keen_io requires \$KEEN_IO_WRITE_KEY to be specified"
-    exit 1
+    return 1
   fi
 
   __keen_io_api_url="https://api.keen.io/3.0"
@@ -28,7 +28,7 @@ report () {
   local value=$2
 
   curl -s $__keen_io_api_url -H "Content-Type: application/json" \
-                  -d "{\"metric\":\"$metric\",\"value\":$value}" > /dev/null
+       -d "{\"metric\":\"$metric\",\"value\":$value}"
 }
 
 docs () {

reporters/stathat.sh

@@ -3,7 +3,7 @@
 start () {
   if [ -z $STATHAT_API_KEY ]; then
     echo "Error: stathat requires \$STATHAT_API_KEY to be specified"
-    exit 1
+    return 1
   fi
 }
 
@@ -11,8 +11,8 @@ report () {
   local metric=$1
   local value=$2
 
-  curl -s -d "ezkey=$STATHAT_API_KEY&stat=$metric&value=$value" \
-                                      http://api.stathat.com/ez > /dev/null
+  curl -s http://api.stathat.com/ez \
+       -d "ezkey=$STATHAT_API_KEY&stat=$metric&value=$value"
 }
 
 docs () {