Add notify after docker builds

Should now have pairity in terms of system utilities and scripts for
checking services

.drone.yml

@@ -45,15 +45,70 @@ trigger:
     - refs/tags/v*
 
 steps:
+  - name: build all binaries
+    image: golang:1.12
+    commands:
+      - make all
 
-  # Might consider moving this step into the previous pipeline
-  - name: push image
+  - name: push image - arm
     image: plugins/docker
     settings:
       repo: iamthefij/minitor-go
-      dockerfile: Dockerfile.multi-stage
       auto_tag: true
+      auto_tag_suffix: linux-arm
       username:
         from_secret: docker_username
       password:
         from_secret: docker_password
+      build_args:
+        - ARCH=arm
+        - REPO=arm32v7
+
+  - name: push image - arm64
+    image: plugins/docker
+    settings:
+      repo: iamthefij/minitor-go
+      auto_tag: true
+      auto_tag_suffix: linux-arm64
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      build_args:
+        - ARCH=arm64
+        - REPO=arm64v8
+
+  - name: push image - amd64
+    image: plugins/docker
+    settings:
+      repo: iamthefij/minitor-go
+      auto_tag: true
+      auto_tag_suffix: linux-amd64
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+
+  - name: publish manifest
+    image: plugins/manifest
+    settings:
+      spec: manifest.tmpl
+      auto_tag: true
+      ignore_missing: true
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+
+  - name: notify
+    image: drillster/drone-email
+    settings:
+      host:
+        from_secret: SMTP_HOST
+      username:
+        from_secret: SMTP_USER
+      password:
+        from_secret: SMTP_PASS
+      from: drone@iamthefij.com
+    when:
+      status: [changed, failure]

.gitignore

@@ -16,4 +16,6 @@
 config.yml
 
 # Output binary
-minitor-go
+minitor
+minitor-linux-*
+minitor-darwin-amd64

Dockerfile

@@ -1,8 +1,28 @@
 ARG REPO=library
-FROM ${REPO}/busybox:latest
-WORKDIR /root/
+FROM multiarch/qemu-user-static:4.2.0-2 as qemu-user-static
+FROM ${REPO}/alpine:3.10
 
+# Copying all qemu files because amd64 doesn't exist and cannot condional copy
+COPY --from=qemu-user-static /usr/bin/qemu-* /usr/bin/
+
+RUN mkdir /app
+WORKDIR /app/
+
+# Add common checking tools
+RUN apk --no-cache add bash=~5.0 curl=~7.66 jq=~1.6
+
+# Add minitor user for running as non-root
+RUN addgroup -S minitor && adduser -S minitor -G minitor
+
+# Copy scripts
+COPY ./scripts /app/scripts
+RUN chmod -R 755 /app/scripts
+
+# Copy minitor in
 ARG ARCH=amd64
-COPY ./minitor-go ./minitor
+COPY ./minitor-linux-${ARCH} ./minitor
+
+# Drop to non-root user
+USER minitor
 
 ENTRYPOINT [ "./minitor" ]

Dockerfile.multi-stage

@@ -1,7 +1,7 @@
 ARG REPO=library
 FROM golang:1.12-alpine AS builder
 
-RUN apk add --no-cache git
+RUN apk add --no-cache git=~2
 
 RUN mkdir /app
 WORKDIR /app
@@ -16,8 +16,26 @@ ARG VERSION=dev
 ENV CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH}
 RUN go build -ldflags "-X main.version=${VERSION}" -a -installsuffix nocgo -o minitor .
 
-FROM ${REPO}/busybox:latest
-WORKDIR /root/
+FROM ${REPO}/alpine:3.10
+RUN mkdir /app
+WORKDIR /app/
+
+# Copy minitor in
 COPY --from=builder /app/minitor .
 
+# Add common checking tools
+RUN apk --no-cache add bash=~5.0 curl=~7.66 jq=~1.6
+
+# Add minitor user for running as non-root
+RUN addgroup -S minitor && adduser -S minitor -G minitor
+
+# Copy scripts
+COPY ./scripts /app/scripts
+RUN chmod -R 755 /app/scripts
+
+# Drop to non-root user
+USER minitor
+
 ENTRYPOINT [ "./minitor" ]
+
+# vim: set filetype=dockerfile:

Makefile

@@ -1,23 +1,28 @@
-.PHONY: all
 DOCKER_TAG ?= minitor-go-${USER}
+GIT_TAG_NAME := $(shell git tag -l --contains HEAD)
+GIT_SHA := $(shell git rev-parse HEAD)
+VERSION := $(if $(GIT_TAG_NAME),$(GIT_TAG_NAME),$(GIT_SHA))
+
+.PHONY: all
+all: minitor-linux-amd64 minitor-linux-arm minitor-linux-arm64
 
 .PHONY: default
 default: test
 
 .PHONY: build
-build:
-	go build
+build: minitor
 
-minitor-go:
-	go build
+minitor:
+	@echo Version: $(VERSION)
+	go build -ldflags '-X "main.version=${VERSION}"' -o minitor
 
 .PHONY: run
-run: minitor-go build
-	./minitor-go -debug
+run: minitor build
+	./minitor -debug
 
 .PHONY: run-metrics
-run-metrics: minitor-go build
-	./minitor-go -debug -metrics
+run-metrics: minitor build
+	./minitor -debug -metrics
 
 .PHONY: test
 test:
@@ -41,13 +46,56 @@ check:
 
 .PHONY: clean
 clean:
-	rm -f ./minitor-go
+	rm -f ./minitor
+	rm -f ./minitor-linux-*
+	rm -f ./minitor-darwin-amd64
 	rm -f ./coverage.out
 
 .PHONY: docker-build
 docker-build:
-	docker build -f ./Dockerfile.multi-stage -t $(DOCKER_TAG) .
+	docker build -f ./Dockerfile.multi-stage -t $(DOCKER_TAG)-linux-amd64 .
 
 .PHONY: docker-run
 docker-run: docker-build
 	docker run --rm -v $(shell pwd)/config.yml:/root/config.yml $(DOCKER_TAG)
+
+## Multi-arch targets
+
+# Arch specific go build targets
+minitor-darwin-amd64:
+	GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 \
+		   go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
+		   -o minitor-darwin-amd64
+
+minitor-linux-amd64:
+	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \
+		   go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
+		   -o minitor-linux-amd64
+
+minitor-linux-arm:
+	GOOS=linux GOARCH=arm CGO_ENABLED=0 \
+		   go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
+		   -o minitor-linux-arm
+
+minitor-linux-arm64:
+	GOOS=linux GOARCH=arm64 CGO_ENABLED=0 \
+		   go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
+		   -o minitor-linux-arm64
+
+# Arch specific docker build targets
+.PHONY: docker-build-arm
+docker-build-arm: minitor-linux-arm
+	docker build --build-arg REPO=arm32v7 --build-arg ARCH=arm . -t ${DOCKER_TAG}-linux-arm
+
+.PHONY: docker-build-arm
+docker-build-arm64: minitor-linux-arm64
+	docker build --build-arg REPO=arm64v8 --build-arg ARCH=arm64 . -t ${DOCKER_TAG}-linux-arm64
+
+# Cross run on host architechture
+.PHONY: docker-run-arm
+docker-run-arm: docker-build-arm
+	docker run --rm -v /var/run/docker.sock:/var/run/docker.sock --name $(DOCKER_TAG)-run ${DOCKER_TAG}-linux-arm
+
+.PHONY: docker-run-arm64
+docker-run-arm64: docker-build-arm64
+	docker run --rm -v /var/run/docker.sock:/var/run/docker.sock --name $(DOCKER_TAG)-run ${DOCKER_TAG}-linux-arm64

config.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"io/ioutil"
 	"log"
-	"os"
 
 	"gopkg.in/yaml.v2"
 )
@@ -78,9 +77,7 @@ func LoadConfig(filePath string) (config Config, err error) {
 		return
 	}
 
-	// TODO: Decide if this is better expanded here, or only when executing
-	envExpanded := os.ExpandEnv(string(data))
-	err = yaml.Unmarshal([]byte(envExpanded), &config)
+	err = yaml.Unmarshal(data, &config)
 	if err != nil {
 		return
 	}

config_test.go

@@ -23,9 +23,74 @@ func TestLoadConfig(t *testing.T) {
 		_, err := LoadConfig(c.configPath)
 		hasErr := (err != nil)
 		if hasErr != c.expectErr {
-			t.Errorf("LoadConfig(%v), expected=%v actual=%v", c.name, "Err", err)
+			t.Errorf("LoadConfig(%v), expected_error=%v actual=%v", c.name, c.expectErr, err)
 			log.Printf("Case failed: %s", c.name)
 		}
 		log.Println("-----")
 	}
 }
+
+// TestMultiLineConfig is a more complicated test stepping through the parsing
+// and execution of mutli-line strings presented in YAML
+func TestMultiLineConfig(t *testing.T) {
+	log.Println("Testing multi-line string config")
+	config, err := LoadConfig("./test/valid-verify-multi-line.yml")
+	if err != nil {
+		t.Fatalf("TestMultiLineConfig(load), expected=no_error actual=%v", err)
+	}
+
+	log.Println("-----")
+	log.Println("TestMultiLineConfig(parse > string)")
+	expected := "echo 'Some string with stuff'; echo \"<angle brackets>\"; exit 1\n"
+	actual := config.Monitors[0].CommandShell
+	if expected != actual {
+		t.Errorf("TestMultiLineConfig(>) failed")
+		t.Logf("string expected=`%v`", expected)
+		t.Logf("string actual  =`%v`", actual)
+		t.Logf("bytes expected=%v", []byte(expected))
+		t.Logf("bytes actual  =%v", []byte(actual))
+	}
+
+	log.Println("-----")
+	log.Println("TestMultiLineConfig(execute > string)")
+	_, notice := config.Monitors[0].Check()
+	if notice == nil {
+		t.Fatalf("Did not receive an alert notice")
+	}
+	expected = "Some string with stuff\n<angle brackets>\n"
+	actual = notice.LastCheckOutput
+	if expected != actual {
+		t.Errorf("TestMultiLineConfig(execute > string) check failed")
+		t.Logf("string expected=`%v`", expected)
+		t.Logf("string actual  =`%v`", actual)
+		t.Logf("bytes expected=%v", []byte(expected))
+		t.Logf("bytes actual  =%v", []byte(actual))
+	}
+
+	log.Println("-----")
+	log.Println("TestMultiLineConfig(parse | string)")
+	expected = "echo 'Some string with stuff'\necho '<angle brackets>'\n"
+	actual = config.Alerts["log_shell"].CommandShell
+	if expected != actual {
+		t.Errorf("TestMultiLineConfig(|) failed")
+		t.Logf("string expected=`%v`", expected)
+		t.Logf("string actual  =`%v`", actual)
+		t.Logf("bytes expected=%v", []byte(expected))
+		t.Logf("bytes actual  =%v", []byte(actual))
+	}
+
+	log.Println("-----")
+	log.Println("TestMultiLineConfig(execute | string)")
+	actual, err = config.Alerts["log_shell"].Send(AlertNotice{})
+	if err != nil {
+		t.Errorf("Execution of alert failed")
+	}
+	expected = "Some string with stuff\n<angle brackets>\n"
+	if expected != actual {
+		t.Errorf("TestMultiLineConfig(execute | string) check failed")
+		t.Logf("string expected=`%v`", expected)
+		t.Logf("string actual  =`%v`", actual)
+		t.Logf("bytes expected=%v", []byte(expected))
+		t.Logf("bytes actual  =%v", []byte(actual))
+	}
+}

manifest.tmpl

@@ -0,0 +1,25 @@
+image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}
+{{#if build.tags}}
+tags:
+{{#each build.tags}}
+  - {{this}}
+{{/each}}
+{{/if}}
+manifests:
+  -
+    image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64
+    platform:
+      architecture: amd64
+      os: linux
+  -
+    image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64
+    platform:
+      architecture: arm64
+      os: linux
+      variant: v8
+  -
+    image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm
+    platform:
+      architecture: arm
+      os: linux
+      variant: v7

monitor.go

@@ -155,18 +155,16 @@ func (monitor Monitor) getAlertAfter() int16 {
 	// Zero is one!
 	if monitor.AlertAfter == 0 {
 		return 1
-	} else {
-		return monitor.AlertAfter
 	}
+	return monitor.AlertAfter
 }
 
 // GetAlertNames gives a list of alert names for a given monitor status
 func (monitor Monitor) GetAlertNames(up bool) []string {
 	if up {
 		return monitor.AlertUp
-	} else {
-		return monitor.AlertDown
 	}
+	return monitor.AlertDown
 }
 
 func (monitor Monitor) createAlertNotice(isUp bool) *AlertNotice {

sample-config.yml

@@ -1,29 +1,41 @@
-check_interval: 30
+---
+check_interval: 5
 
 monitors:
-  - name: My Website
+  - name: Fake Website
     command: ['curl', '-s', '-o', '/dev/null', 'https://minitor.mon']
-    alert_down: [ log, mailgun_down, sms_down ]
-    alert_up: [ log, email_up ]
-    check_interval: 30 # Must be at minimum the global `check_interval`
+    alert_down: [log_down, mailgun_down, sms_down]
+    alert_up: [log_up, email_up]
+    check_interval: 10  # Must be at minimum the global `check_interval`
     alert_after: 3
     alert_every: -1  # Defaults to -1 for exponential backoff. 0 to disable repeating
+  - name: Real Website
+    command: ['curl', '-s', '-o', '/dev/null', 'https://google.com']
+    alert_down: [log_down, mailgun_down, sms_down]
+    alert_up: [log_up, email_up]
+    check_interval: 5
+    alert_after: 3
+    alert_every: -1
 
 alerts:
+  log_down:
+    command: ["echo", "Minitor failure for {{.MonitorName}}"]
+  log_up:
+    command: ["echo", "Minitor recovery for {{.MonitorName}}"]
   email_up:
     command: [sendmail, "me@minitor.mon", "Recovered: {monitor_name}", "We're back!"]
   mailgun_down:
-    command: >
+    command_shell: >
       curl -s -X POST
-      -F subject="Alert! {monitor_name} failed"
+      -F subject="Alert! {{.MonitorName}} failed"
       -F from="Minitor <minitor@minitor.mon>"
       -F to=me@minitor.mon
       -F text="Our monitor failed"
       https://api.mailgun.net/v3/minitor.mon/messages
       -u "api:${MAILGUN_API_KEY}"
   sms_down:
-    command: >
-      curl -s -X POST -F "Body=Failure! {monitor_name} has failed"
+    command_shell: >
+      curl -s -X POST -F "Body=Failure! {{.MonitorName}} has failed"
       -F "From=${AVAILABLE_NUMBER}" -F "To=${MY_PHONE}"
       "https://api.twilio.com/2010-04-01/Accounts/${ACCOUNT_SID}/Messages"
       -u "${ACCOUNT_SID}:${AUTH_TOKEN}"

scripts/README.md

@@ -0,0 +1,5 @@
+# Minitor Scripts
+
+A collection of some handy scripts to use with Minitor
+
+These are not included with the Python package, but they are included in the Docker image in `/app/scripts`.

scripts/docker_check.sh

@@ -0,0 +1,51 @@
+#! /bin/bash
+set -e
+
+#################
+# docker_check.sh
+#
+# Checks the most recent state exit code of a Docker container
+#################
+
+# Docker host will default to a socket
+# To override, export DOCKER_HOST to a new hostname
+DOCKER_HOST="${DOCKER_HOST:=socket}"
+container_name="$1"
+
+# Curls Docker either using a socket or URL
+function curl_docker {
+    local path="$1"
+    if [ "$DOCKER_HOST" == "socket" ]; then
+        curl --unix-socket /var/run/docker.sock "http://localhost/$path" 2>/dev/null
+    else
+        curl "http://${DOCKER_HOST}/$path" 2>/dev/null
+    fi
+}
+
+# Returns caintainer ID for a given container name
+function get_container_id {
+    local container_name="$1"
+    curl_docker 'containers/json?all=1' \
+        | jq -r ".[] | {Id, Name: .Names[]} | select(.Name == \"/${container_name}\") | .Id"
+}
+
+# Returns container JSON
+function inspect_container {
+    local container_id=$1
+    curl_docker "containers/$container_id/json"
+}
+
+if [ -z "$container_name" ]; then
+    echo "Usage: $0 container_name"
+    echo "Will exit with the last status code of continer with provided name"
+    exit 1
+fi
+
+container_id=$(get_container_id $container_name)
+if [ -z "$container_id" ]; then
+    echo "ERROR: Could not find container with name: $container_name"
+    exit 1
+fi
+exit_code=$(inspect_container "$container_id" | jq -r .State.ExitCode)
+
+exit "$exit_code"

scripts/docker_healthcheck.sh

@@ -0,0 +1,61 @@
+#! /bin/bash
+set -e
+
+#################
+# docker_healthcheck.sh
+#
+# Returns the results of a Docker Healthcheck for a container
+#################
+
+# Docker host will default to a socket
+# To override, export DOCKER_HOST to a new hostname
+DOCKER_HOST="${DOCKER_HOST:=socket}"
+container_name="$1"
+
+# Curls Docker either using a socket or URL
+function curl_docker {
+    local path="$1"
+    if [ "$DOCKER_HOST" == "socket" ]; then
+        curl --unix-socket /var/run/docker.sock "http://localhost/$path" 2>/dev/null
+    else
+        curl "http://${DOCKER_HOST}/$path" 2>/dev/null
+    fi
+}
+
+# Returns caintainer ID for a given container name
+function get_container_id {
+    local container_name="$1"
+    curl_docker 'containers/json?all=1' \
+        | jq -r ".[] | {Id, Name: .Names[]} | select(.Name == \"/${container_name}\") | .Id"
+}
+
+# Returns container JSON
+function inspect_container {
+    local container_id="$1"
+    curl_docker "containers/$container_id/json"
+}
+
+if [ -z "$container_name" ]; then
+    echo "Usage: $0 container_name"
+    echo "Will return results of healthcheck for continer with provided name"
+    exit 1
+fi
+
+container_id=$(get_container_id "$container_name")
+if [ -z "$container_id" ]; then
+    echo "ERROR: Could not find container with name: $container_name"
+    exit 1
+fi
+health=$(inspect_container "$container_id" | jq -r '.State.Health.Status')
+
+case "$health" in
+    null)
+        echo "No healthcheck results"
+        ;;
+    starting|healthy)
+        echo "Status: '$health'"
+        ;;
+    *)
+        echo "Status: '$health'"
+        exit 1
+esac

test/valid-config.yml

@@ -1,3 +1,4 @@
+---
 check_interval: 1
 
 monitors:

test/valid-verify-multi-line.yml

@@ -0,0 +1,18 @@
+---
+check_interval: 1
+
+monitors:
+  - name: Shell
+    command_shell: >
+      echo 'Some string with stuff';
+      echo "<angle brackets>";
+      exit 1
+    alert_down: ['log_shell']
+    alert_after: 1
+    alert_every: 0
+
+alerts:
+  log_shell:
+    command_shell: |
+      echo 'Some string with stuff'
+      echo '<angle brackets>'

util.go

@@ -5,19 +5,9 @@ import (
 	"strings"
 )
 
-// escapeCommandShell accepts a command to be executed by a shell and escapes it
-func escapeCommandShell(command string) string {
-	// Remove extra spaces and newlines from ends
-	command = strings.TrimSpace(command)
-	// TODO: Not sure if this part is actually needed. Should verify
-	// Escape double quotes since this will be passed in as an argument
-	command = strings.Replace(command, `"`, `\"`, -1)
-	return command
-}
-
 // ShellCommand takes a string and executes it as a command using `sh`
 func ShellCommand(command string) *exec.Cmd {
-	shellCommand := []string{"sh", "-c", escapeCommandShell(command)}
+	shellCommand := []string{"sh", "-c", strings.TrimSpace(command)}
 	//log.Printf("Shell command: %v", shellCommand)
 	return exec.Command(shellCommand[0], shellCommand[1:]...)
 }