WIP: Update logging to improve formatting a bit

Should now have pairity in terms of system utilities and scripts for
checking services

.drone.yml

@@ -3,14 +3,11 @@ kind: pipeline
 name: test
 
 steps:
-  - name: build
-    image: golang:1.12
-    commands:
-      - make build
 
   - name: test
     image: golang:1.12
     commands:
+      - make build
       - make test
 
   - name: check

Dockerfile

@@ -1,8 +1,23 @@
 ARG REPO=library
-FROM ${REPO}/busybox:latest
-WORKDIR /root/
+FROM ${REPO}/alpine:3.10
+RUN mkdir /app
+WORKDIR /app/
 
+# Copy minitor in
 ARG ARCH=amd64
 COPY ./minitor-go ./minitor
 
+# Add common checking tools
+RUN apk --no-cache add bash=~5.0 curl=~7.66 jq=~1.6
+
+# Add minitor user for running as non-root
+RUN addgroup -S minitor && adduser -S minitor -G minitor
+
+# Copy scripts
+COPY ./scripts /app/scripts
+RUN chmod -R 755 /app/scripts
+
+# Drop to non-root user
+USER minitor
+
 ENTRYPOINT [ "./minitor" ]

Dockerfile.multi-stage

@@ -1,7 +1,7 @@
 ARG REPO=library
 FROM golang:1.12-alpine AS builder
 
-RUN apk add --no-cache git
+RUN apk add --no-cache git=~2
 
 RUN mkdir /app
 WORKDIR /app
@@ -16,8 +16,26 @@ ARG VERSION=dev
 ENV CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH}
 RUN go build -ldflags "-X main.version=${VERSION}" -a -installsuffix nocgo -o minitor .
 
-FROM ${REPO}/busybox:latest
-WORKDIR /root/
+FROM ${REPO}/alpine:3.10
+RUN mkdir /app
+WORKDIR /app/
+
+# Copy minitor in
 COPY --from=builder /app/minitor .
 
+# Add common checking tools
+RUN apk --no-cache add bash=~5.0 curl=~7.66 jq=~1.6
+
+# Add minitor user for running as non-root
+RUN addgroup -S minitor && adduser -S minitor -G minitor
+
+# Copy scripts
+COPY ./scripts /app/scripts
+RUN chmod -R 755 /app/scripts
+
+# Drop to non-root user
+USER minitor
+
 ENTRYPOINT [ "./minitor" ]
+
+# vim: set filetype=dockerfile:

alert.go

@@ -3,10 +3,11 @@ package main
 import (
 	"bytes"
 	"fmt"
-	"log"
 	"os/exec"
 	"text/template"
 	"time"
+
+	log "github.com/sirupsen/logrus"
 )
 
 // Alert is a config driven mechanism for sending a notice
@@ -38,9 +39,7 @@ func (alert Alert) IsValid() bool {
 
 // BuildTemplates compiles command templates for the Alert
 func (alert *Alert) BuildTemplates() error {
-	if LogDebug {
-		log.Printf("DEBUG: Building template for alert %s", alert.Name)
-	}
+	log.Debugf("Building template for alert %s", alert.Name)
 	if alert.commandTemplate == nil && alert.Command != nil {
 		alert.commandTemplate = []*template.Template{}
 		for i, cmdPart := range alert.Command {
@@ -60,8 +59,8 @@ func (alert *Alert) BuildTemplates() error {
 }
 
 // Send will send an alert notice by executing the command template
-func (alert Alert) Send(notice AlertNotice) (output_str string, err error) {
-	log.Printf("INFO: Sending alert %s for %s", alert.Name, notice.MonitorName)
+func (alert Alert) Send(notice AlertNotice) (outputStr string, err error) {
+	log.Infof("Sending alert %s for %s", alert.Name, notice.MonitorName)
 	var cmd *exec.Cmd
 	if alert.commandTemplate != nil {
 		command := []string{}
@@ -95,10 +94,8 @@ func (alert Alert) Send(notice AlertNotice) (output_str string, err error) {
 
 	var output []byte
 	output, err = cmd.CombinedOutput()
-	output_str = string(output)
-	if LogDebug {
-		log.Printf("DEBUG: Alert output for: %s\n---\n%s\n---", alert.Name, output_str)
-	}
+	outputStr = string(output)
+	log.Debugf("Alert output for: %s\n---\n%s\n---", alert.Name, outputStr)
 
-	return output_str, err
+	return outputStr, err
 }

alert_test.go

@@ -1,8 +1,9 @@
 package main
 
 import (
-	"log"
 	"testing"
+
+	log "github.com/sirupsen/logrus"
 )
 
 func TestAlertIsValid(t *testing.T) {
@@ -22,13 +23,13 @@ func TestAlertIsValid(t *testing.T) {
 	}
 
 	for _, c := range cases {
-		log.Printf("Testing case %s", c.name)
+		log.Debugf("Testing case %s", c.name)
 		actual := c.alert.IsValid()
 		if actual != c.expected {
 			t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expected, actual)
-			log.Printf("Case failed: %s", c.name)
+			log.Debugf("Case failed: %s", c.name)
 		}
-		log.Println("-----")
+		log.Debugf("-----")
 	}
 }
 
@@ -71,19 +72,19 @@ func TestAlertSend(t *testing.T) {
 	}
 
 	for _, c := range cases {
-		log.Printf("Testing case %s", c.name)
+		log.Debugf("Testing case %s", c.name)
 		c.alert.BuildTemplates()
 		output, err := c.alert.Send(c.notice)
 		hasErr := (err != nil)
 		if output != c.expectedOutput {
 			t.Errorf("Send(%v output), expected=%v actual=%v", c.name, c.expectedOutput, output)
-			log.Printf("Case failed: %s", c.name)
+			log.Debugf("Case failed: %s", c.name)
 		}
 		if hasErr != c.expectErr {
 			t.Errorf("Send(%v err), expected=%v actual=%v", c.name, "Err", err)
-			log.Printf("Case failed: %s", c.name)
+			log.Debugf("Case failed: %s", c.name)
 		}
-		log.Println("-----")
+		log.Debugf("-----")
 	}
 }
 
@@ -94,7 +95,7 @@ func TestAlertSendNoTemplates(t *testing.T) {
 	if err == nil {
 		t.Errorf("Send(no template), expected=%v actual=%v", "Err", output)
 	}
-	log.Println("-----")
+	log.Debugf("-----")
 }
 
 func TestAlertBuildTemplate(t *testing.T) {
@@ -109,13 +110,13 @@ func TestAlertBuildTemplate(t *testing.T) {
 	}
 
 	for _, c := range cases {
-		log.Printf("Testing case %s", c.name)
+		log.Debugf("Testing case %s", c.name)
 		err := c.alert.BuildTemplates()
 		hasErr := (err != nil)
 		if hasErr != c.expectErr {
 			t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expectErr, err)
-			log.Printf("Case failed: %s", c.name)
+			log.Debugf("Case failed: %s", c.name)
 		}
-		log.Println("-----")
+		log.Debugf("-----")
 	}
 }

config.go

@@ -3,9 +3,9 @@ package main
 import (
 	"errors"
 	"io/ioutil"
-	"log"
 	"os"
 
+	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v2"
 )
 
@@ -22,20 +22,20 @@ func (config Config) IsValid() (isValid bool) {
 
 	// Validate monitors
 	if config.Monitors == nil || len(config.Monitors) == 0 {
-		log.Printf("ERROR: Invalid monitor configuration: Must provide at least one monitor")
+		log.Errorf("Invalid monitor configuration: Must provide at least one monitor")
 		isValid = false
 	}
 	for _, monitor := range config.Monitors {
 		if !monitor.IsValid() {
-			log.Printf("ERROR: Invalid monitor configuration: %s", monitor.Name)
+			log.Errorf("Invalid monitor configuration: %s", monitor.Name)
 			isValid = false
 		}
 		// Check that all Monitor alerts actually exist
 		for _, isUp := range []bool{true, false} {
 			for _, alertName := range monitor.GetAlertNames(isUp) {
 				if _, ok := config.Alerts[alertName]; !ok {
-					log.Printf(
-						"ERROR: Invalid monitor configuration: %s. Unknown alert %s",
+					log.Errorf(
+						"Invalid monitor configuration: %s. Unknown alert %s",
 						monitor.Name, alertName,
 					)
 					isValid = false
@@ -46,12 +46,12 @@ func (config Config) IsValid() (isValid bool) {
 
 	// Validate alerts
 	if config.Alerts == nil || len(config.Alerts) == 0 {
-		log.Printf("ERROR: Invalid alert configuration: Must provide at least one alert")
+		log.Errorf("Invalid alert configuration: Must provide at least one alert")
 		isValid = false
 	}
 	for _, alert := range config.Alerts {
 		if !alert.IsValid() {
-			log.Printf("ERROR: Invalid alert configuration: %s", alert.Name)
+			log.Errorf("Invalid alert configuration: %s", alert.Name)
 			isValid = false
 		}
 	}
@@ -85,9 +85,7 @@ func LoadConfig(filePath string) (config Config, err error) {
 		return
 	}
 
-	if LogDebug {
-		log.Printf("DEBUG: Config values:\n%v\n", config)
-	}
+	log.Debugf("Config values:\n%v\n", config)
 
 	if !config.IsValid() {
 		err = errors.New("Invalid configuration")

go.mod

@@ -4,5 +4,6 @@ go 1.12
 
 require (
 	github.com/prometheus/client_golang v1.2.1
+	github.com/sirupsen/logrus v1.4.2
 	gopkg.in/yaml.v2 v2.2.4
 )

go.sum

@@ -53,6 +53,7 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
 github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
 github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -68,6 +69,7 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
 golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=

main.go

@@ -3,14 +3,12 @@ package main
 import (
 	"flag"
 	"fmt"
-	"log"
 	"time"
+
+	log "github.com/sirupsen/logrus"
 )
 
 var (
-	// LogDebug will control whether debug messsages should be logged
-	LogDebug = false
-
 	// ExportMetrics will track whether or not we want to export metrics to prometheus
 	ExportMetrics = false
 	// MetricsPort is the port to expose metrics on
@@ -35,9 +33,7 @@ func checkMonitors(config *Config) error {
 
 			// Should probably consider refactoring everything below here
 			if alertNotice != nil {
-				if LogDebug {
-					log.Printf("DEBUG: Recieved an alert notice from %s", alertNotice.MonitorName)
-				}
+				log.Debugf("Recieved an alert notice from %s", alertNotice.MonitorName)
 				alertNames := monitor.GetAlertNames(alertNotice.IsUp)
 				if alertNames == nil {
 					// This should only happen for a recovery alert. AlertDown is validated not empty
@@ -81,11 +77,16 @@ func checkMonitors(config *Config) error {
 
 func main() {
 	// Get debug flag
-	flag.BoolVar(&LogDebug, "debug", false, "Enables debug logs (default: false)")
+	var debug = flag.Bool("debug", false, "Enables debug logs (default: false)")
 	flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)")
 	var showVersion = flag.Bool("version", false, "Display the version of minitor and exit")
 	flag.Parse()
 
+	// Set debug if flag is set
+	if *debug {
+		log.SetLevel(log.DebugLevel)
+	}
+
 	// Print version if flag is provided
 	if *showVersion {
 		log.Println("Minitor version:", version)

monitor.go

@@ -1,10 +1,11 @@
 package main
 
 import (
-	"log"
 	"math"
 	"os/exec"
 	"time"
+
+	log "github.com/sirupsen/logrus"
 )
 
 // Monitor represents a particular periodic check of a command
@@ -70,20 +71,18 @@ func (monitor *Monitor) Check() (bool, *AlertNotice) {
 		alertNotice = monitor.failure()
 	}
 
-	if LogDebug {
-		log.Printf("DEBUG: Command output: %s", monitor.lastOutput)
-	}
+	log.Debugf("Command output: %s", monitor.lastOutput)
 	if err != nil {
-		if LogDebug {
-			log.Printf("DEBUG: Command result: %v", err)
-		}
+		log.Debugf("Command result: %v", err)
 	}
 
-	log.Printf(
-		"INFO: %s success=%t, alert=%t",
+	log.WithFields(log.Fields{
+		"monitor": monitor.Name,
+		"success": isSuccess,
+		"alert":   alertNotice != nil,
+	}).Infof(
+		"%s checked",
 		monitor.Name,
-		isSuccess,
-		alertNotice != nil,
 	)
 
 	return isSuccess, alertNotice
@@ -109,15 +108,13 @@ func (monitor *Monitor) failure() (notice *AlertNotice) {
 	monitor.failureCount++
 	// If we haven't hit the minimum failures, we can exit
 	if monitor.failureCount < monitor.getAlertAfter() {
-		if LogDebug {
-			log.Printf(
-				"DEBUG: %s failed but did not hit minimum failures. "+
+		log.Debugf(
+			"%s failed but did not hit minimum failures. "+
 				"Count: %v alert after: %v",
 			monitor.Name,
 			monitor.failureCount,
 			monitor.getAlertAfter(),
 		)
-		}
 		return
 	}
 
@@ -155,18 +152,16 @@ func (monitor Monitor) getAlertAfter() int16 {
 	// Zero is one!
 	if monitor.AlertAfter == 0 {
 		return 1
-	} else {
-		return monitor.AlertAfter
 	}
+	return monitor.AlertAfter
 }
 
 // GetAlertNames gives a list of alert names for a given monitor status
 func (monitor Monitor) GetAlertNames(up bool) []string {
 	if up {
 		return monitor.AlertUp
-	} else {
-		return monitor.AlertDown
 	}
+	return monitor.AlertDown
 }
 
 func (monitor Monitor) createAlertNotice(isUp bool) *AlertNotice {

sample-config.yml

@@ -1,29 +1,41 @@
-check_interval: 30
+---
+check_interval: 5
 
 monitors:
-  - name: My Website
-    command: [ 'curl', '-s', '-o', '/dev/null', 'https://minitor.mon' ]
-    alert_down: [ log, mailgun_down, sms_down ]
-    alert_up: [ log, email_up ]
-    check_interval: 30 # Must be at minimum the global `check_interval`
+  - name: Fake Website
+    command: ['curl', '-s', '-o', '/dev/null', 'https://minitor.mon']
+    alert_down: [log_down, mailgun_down, sms_down]
+    alert_up: [log_up, email_up]
+    check_interval: 10  # Must be at minimum the global `check_interval`
     alert_after: 3
     alert_every: -1  # Defaults to -1 for exponential backoff. 0 to disable repeating
+  - name: Real Website
+    command: ['curl', '-s', '-o', '/dev/null', 'https://google.com']
+    alert_down: [log_down, mailgun_down, sms_down]
+    alert_up: [log_up, email_up]
+    check_interval: 5
+    alert_after: 3
+    alert_every: -1
 
 alerts:
+  log_down:
+    command: ["echo", "Minitor failure for {{.MonitorName}}"]
+  log_up:
+    command: ["echo", "Minitor recovery for {{.MonitorName}}"]
   email_up:
-    command: [ sendmail, "me@minitor.mon",  "Recovered: {monitor_name}",  "We're back!" ]
+    command: [sendmail, "me@minitor.mon", "Recovered: {monitor_name}", "We're back!"]
   mailgun_down:
-    command: >
+    command_shell: >
       curl -s -X POST
-      -F subject="Alert! {monitor_name} failed"
+      -F subject="Alert! {{.MonitorName}} failed"
       -F from="Minitor <minitor@minitor.mon>"
       -F to=me@minitor.mon
       -F text="Our monitor failed"
       https://api.mailgun.net/v3/minitor.mon/messages
       -u "api:${MAILGUN_API_KEY}"
   sms_down:
-    command: >
-      curl -s -X POST -F "Body=Failure! {monitor_name} has failed"
+    command_shell: >
+      curl -s -X POST -F "Body=Failure! {{.MonitorName}} has failed"
       -F "From=${AVAILABLE_NUMBER}" -F "To=${MY_PHONE}"
       "https://api.twilio.com/2010-04-01/Accounts/${ACCOUNT_SID}/Messages"
       -u "${ACCOUNT_SID}:${AUTH_TOKEN}"

scripts/README.md

@@ -0,0 +1,5 @@
+# Minitor Scripts
+
+A collection of some handy scripts to use with Minitor
+
+These are not included with the Python package, but they are included in the Docker image in `/app/scripts`.

scripts/docker_check.sh

@@ -0,0 +1,51 @@
+#! /bin/bash
+set -e
+
+#################
+# docker_check.sh
+#
+# Checks the most recent state exit code of a Docker container
+#################
+
+# Docker host will default to a socket
+# To override, export DOCKER_HOST to a new hostname
+DOCKER_HOST="${DOCKER_HOST:=socket}"
+container_name="$1"
+
+# Curls Docker either using a socket or URL
+function curl_docker {
+    local path="$1"
+    if [ "$DOCKER_HOST" == "socket" ]; then
+        curl --unix-socket /var/run/docker.sock "http://localhost/$path" 2>/dev/null
+    else
+        curl "http://${DOCKER_HOST}/$path" 2>/dev/null
+    fi
+}
+
+# Returns caintainer ID for a given container name
+function get_container_id {
+    local container_name="$1"
+    curl_docker 'containers/json?all=1' \
+        | jq -r ".[] | {Id, Name: .Names[]} | select(.Name == \"/${container_name}\") | .Id"
+}
+
+# Returns container JSON
+function inspect_container {
+    local container_id=$1
+    curl_docker "containers/$container_id/json"
+}
+
+if [ -z "$container_name" ]; then
+    echo "Usage: $0 container_name"
+    echo "Will exit with the last status code of continer with provided name"
+    exit 1
+fi
+
+container_id=$(get_container_id $container_name)
+if [ -z "$container_id" ]; then
+    echo "ERROR: Could not find container with name: $container_name"
+    exit 1
+fi
+exit_code=$(inspect_container "$container_id" | jq -r .State.ExitCode)
+
+exit "$exit_code"

scripts/docker_healthcheck.sh

@@ -0,0 +1,61 @@
+#! /bin/bash
+set -e
+
+#################
+# docker_healthcheck.sh
+#
+# Returns the results of a Docker Healthcheck for a container
+#################
+
+# Docker host will default to a socket
+# To override, export DOCKER_HOST to a new hostname
+DOCKER_HOST="${DOCKER_HOST:=socket}"
+container_name="$1"
+
+# Curls Docker either using a socket or URL
+function curl_docker {
+    local path="$1"
+    if [ "$DOCKER_HOST" == "socket" ]; then
+        curl --unix-socket /var/run/docker.sock "http://localhost/$path" 2>/dev/null
+    else
+        curl "http://${DOCKER_HOST}/$path" 2>/dev/null
+    fi
+}
+
+# Returns caintainer ID for a given container name
+function get_container_id {
+    local container_name="$1"
+    curl_docker 'containers/json?all=1' \
+        | jq -r ".[] | {Id, Name: .Names[]} | select(.Name == \"/${container_name}\") | .Id"
+}
+
+# Returns container JSON
+function inspect_container {
+    local container_id="$1"
+    curl_docker "containers/$container_id/json"
+}
+
+if [ -z "$container_name" ]; then
+    echo "Usage: $0 container_name"
+    echo "Will return results of healthcheck for continer with provided name"
+    exit 1
+fi
+
+container_id=$(get_container_id "$container_name")
+if [ -z "$container_id" ]; then
+    echo "ERROR: Could not find container with name: $container_name"
+    exit 1
+fi
+health=$(inspect_container "$container_id" | jq -r '.State.Health.Status')
+
+case "$health" in
+    null)
+        echo "No healthcheck results"
+        ;;
+    starting|healthy)
+        echo "Status: '$health'"
+        ;;
+    *)
+        echo "Status: '$health'"
+        exit 1
+esac