Make Python compatability a flag

This will make transition easier for an interim period. Will remove at
version 1.0

.drone.yml

@@ -1,13 +1,114 @@
+---
 kind: pipeline
 name: test
 
 steps:
-  - name: build
-    image: golang:1.12
-    commands:
-      - make build
 
   - name: test
     image: golang:1.12
     commands:
+      - make build
       - make test
+
+  - name: check
+    image: python:3
+    commands:
+      - pip install pre-commit==1.20.0
+      - make check
+
+  - name: notify
+    image: drillster/drone-email
+    settings:
+      host:
+        from_secret: SMTP_HOST
+      username:
+        from_secret: SMTP_USER
+      password:
+        from_secret: SMTP_PASS
+      from: drone@iamthefij.com
+    when:
+      status: [changed, failure]
+
+---
+kind: pipeline
+name: publish
+
+depends_on:
+  - test
+
+trigger:
+  event:
+    - push
+    - tag
+  refs:
+    - refs/heads/master
+    - refs/tags/v*
+
+steps:
+  - name: build all binaries
+    image: golang:1.12
+    commands:
+      - make all
+
+  - name: push image - arm
+    image: plugins/docker
+    settings:
+      repo: iamthefij/minitor-go
+      auto_tag: true
+      auto_tag_suffix: linux-arm
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      build_args:
+        - ARCH=arm
+        - REPO=arm32v7
+
+  - name: push image - arm64
+    image: plugins/docker
+    settings:
+      repo: iamthefij/minitor-go
+      auto_tag: true
+      auto_tag_suffix: linux-arm64
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      build_args:
+        - ARCH=arm64
+        - REPO=arm64v8
+
+  - name: push image - amd64
+    image: plugins/docker
+    settings:
+      repo: iamthefij/minitor-go
+      auto_tag: true
+      auto_tag_suffix: linux-amd64
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+
+  - name: publish manifest
+    image: plugins/manifest
+    settings:
+      spec: manifest.tmpl
+      auto_tag: true
+      ignore_missing: true
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+
+  - name: notify
+    image: drillster/drone-email
+    settings:
+      host:
+        from_secret: SMTP_HOST
+      username:
+        from_secret: SMTP_USER
+      password:
+        from_secret: SMTP_PASS
+      from: drone@iamthefij.com
+    when:
+      status: [changed, failure]

.gitignore

@@ -16,4 +16,6 @@
 config.yml
 
 # Output binary
-minitor-go
+minitor
+minitor-linux-*
+minitor-darwin-amd64

.pre-commit-config.yaml

@@ -0,0 +1,19 @@
+---
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.4.0
+    hooks:
+      - id: check-added-large-files
+      - id: check-yaml
+        args:
+          - --allow-multiple-documents
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-merge-conflict
+  - repo: git://github.com/dnephin/pre-commit-golang
+    rev: v0.3.5
+    hooks:
+      - id: go-fmt
+      - id: go-imports
+      # - id: gometalinter
+      # - id: golangci-lint

Dockerfile

@@ -1,8 +1,28 @@
 ARG REPO=library
-FROM ${REPO}/busybox:latest
+FROM multiarch/qemu-user-static:4.2.0-2 as qemu-user-static
-WORKDIR /root/
+FROM ${REPO}/alpine:3.10
 
+# Copying all qemu files because amd64 doesn't exist and cannot condional copy
+COPY --from=qemu-user-static /usr/bin/qemu-* /usr/bin/
+
+RUN mkdir /app
+WORKDIR /app/
+
+# Add common checking tools
+RUN apk --no-cache add bash=~5.0 curl=~7.66 jq=~1.6
+
+# Add minitor user for running as non-root
+RUN addgroup -S minitor && adduser -S minitor -G minitor
+
+# Copy scripts
+COPY ./scripts /app/scripts
+RUN chmod -R 755 /app/scripts
+
+# Copy minitor in
 ARG ARCH=amd64
-COPY ./minitor-go ./minitor
+COPY ./minitor-linux-${ARCH} ./minitor
+
+# Drop to non-root user
+USER minitor
 
 ENTRYPOINT [ "./minitor" ]

Dockerfile.multi-stage

@@ -1,7 +1,7 @@
 ARG REPO=library
 FROM golang:1.12-alpine AS builder
 
-RUN apk add --no-cache git
+RUN apk add --no-cache git=~2
 
 RUN mkdir /app
 WORKDIR /app
@@ -16,8 +16,26 @@ ARG VERSION=dev
 ENV CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH}
 RUN go build -ldflags "-X main.version=${VERSION}" -a -installsuffix nocgo -o minitor .
 
-FROM ${REPO}/busybox:latest
+FROM ${REPO}/alpine:3.10
-WORKDIR /root/
+RUN mkdir /app
+WORKDIR /app/
+
+# Copy minitor in
 COPY --from=builder /app/minitor .
 
+# Add common checking tools
+RUN apk --no-cache add bash=~5.0 curl=~7.66 jq=~1.6
+
+# Add minitor user for running as non-root
+RUN addgroup -S minitor && adduser -S minitor -G minitor
+
+# Copy scripts
+COPY ./scripts /app/scripts
+RUN chmod -R 755 /app/scripts
+
+# Drop to non-root user
+USER minitor
+
 ENTRYPOINT [ "./minitor" ]
+
+# vim: set filetype=dockerfile:

Makefile

@@ -1,18 +1,28 @@
 DOCKER_TAG ?= minitor-go-${USER}
+GIT_TAG_NAME := $(shell git tag -l --contains HEAD)
+GIT_SHA := $(shell git rev-parse HEAD)
+VERSION := $(if $(GIT_TAG_NAME),$(GIT_TAG_NAME),$(GIT_SHA))
 
-.PHONY: test
+.PHONY: all
+all: minitor-linux-amd64 minitor-linux-arm minitor-linux-arm64
+
+.PHONY: default
 default: test
 
 .PHONY: build
-build:
+build: minitor
-	go build
 
-minitor-go:
+minitor:
-	go build
+	@echo Version: $(VERSION)
+	go build -ldflags '-X "main.version=${VERSION}"' -o minitor
 
 .PHONY: run
-run: minitor-go build
+run: minitor build
-	./minitor-go -debug
+	./minitor -debug
+
+.PHONY: run-metrics
+run-metrics: minitor build
+	./minitor -debug -metrics
 
 .PHONY: test
 test:
@@ -24,16 +34,68 @@ test:
 	@go tool cover -func=coverage.out | awk -v target=80.0% \
 		'/^total:/ { print "Total coverage: " $$3 " Minimum coverage: " target; if ($$3+0.0 >= target+0.0) print "ok"; else { print "fail"; exit 1; } }'
 
+# Installs pre-commit hooks
+.PHONY: install-hooks
+install-hooks:
+	pre-commit install --install-hooks
+
+# Checks files for encryption
+.PHONY: check
+check:
+	pre-commit run --all-files
 
 .PHONY: clean
 clean:
-	rm -f ./minitor-go
+	rm -f ./minitor
+	rm -f ./minitor-linux-*
+	rm -f ./minitor-darwin-amd64
 	rm -f ./coverage.out
 
 .PHONY: docker-build
 docker-build:
-	docker build -f ./Dockerfile.multi-stage -t $(DOCKER_TAG) .
+	docker build -f ./Dockerfile.multi-stage -t $(DOCKER_TAG)-linux-amd64 .
 
 .PHONY: docker-run
 docker-run: docker-build
 	docker run --rm -v $(shell pwd)/config.yml:/root/config.yml $(DOCKER_TAG)
+
+## Multi-arch targets
+
+# Arch specific go build targets
+minitor-darwin-amd64:
+	GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 \
+		   go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
+		   -o minitor-darwin-amd64
+
+minitor-linux-amd64:
+	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \
+		   go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
+		   -o minitor-linux-amd64
+
+minitor-linux-arm:
+	GOOS=linux GOARCH=arm CGO_ENABLED=0 \
+		   go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
+		   -o minitor-linux-arm
+
+minitor-linux-arm64:
+	GOOS=linux GOARCH=arm64 CGO_ENABLED=0 \
+		   go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
+		   -o minitor-linux-arm64
+
+# Arch specific docker build targets
+.PHONY: docker-build-arm
+docker-build-arm: minitor-linux-arm
+	docker build --build-arg REPO=arm32v7 --build-arg ARCH=arm . -t ${DOCKER_TAG}-linux-arm
+
+.PHONY: docker-build-arm
+docker-build-arm64: minitor-linux-arm64
+	docker build --build-arg REPO=arm64v8 --build-arg ARCH=arm64 . -t ${DOCKER_TAG}-linux-arm64
+
+# Cross run on host architechture
+.PHONY: docker-run-arm
+docker-run-arm: docker-build-arm
+	docker run --rm -v /var/run/docker.sock:/var/run/docker.sock --name $(DOCKER_TAG)-run ${DOCKER_TAG}-linux-arm
+
+.PHONY: docker-run-arm64
+docker-run-arm64: docker-build-arm64
+	docker run --rm -v /var/run/docker.sock:/var/run/docker.sock --name $(DOCKER_TAG)-run ${DOCKER_TAG}-linux-arm64

README.md

@@ -1,36 +1,15 @@
 # minitor-go
 
-A reimplementation of [Minitor](https://git.iamthefij/iamthefij/minitor) in Go
+A reimplementation of [Minitor](https://git.iamthefij.com/iamthefij/minitor) in Go
 
-Minitor is already a very minimal monitoring tool. Python 3 was a quick way to get something live, but Python itself comes with a very large footprint.Thus Go feels like a better fit for the project, longer term.
+Minitor is already a minimal monitoring tool. Python 3 was a quick way to get something live, but Python itself comes with a large footprint. Thus Go feels like a better fit for the project, longer term.
 
 Initial target is meant to be roughly compatible requiring only minor changes to configuration. Future iterations may diverge to take advantage of Go specific features.
 
 ## Differences from Python version
 
-There are a few key differences between the Python version and the v0.x Go version.
 
-First, configuration keys cannot have multiple types in Go, so a different key must be used when specifying a Shell command as a string rather than a list of args. Instead of `command`, you must use `command_shell`. Eg:
+Templating for Alert messages has been updated. In the Python version, `str.format(...)` was used with certain keys passed in that could be used to format messages. In the Go version, we use a struct, `AlertNotice` defined in `alert.go` and the built in Go templating format. Eg.
-
-minitor-py:
-```yaml
-monitors:
-  - name: Exec command
-    command: ['echo', 'test']
-  - name: Shell command
-    command: echo 'test'
-```
-
-minitor-go:
-```yaml
-monitors:
-  - name: Exec command
-    command: ['echo', 'test']
-  - name: Shell command
-    command_shell: echo 'test'
-```
-
-Second, templating for Alert messages has been updated. In the Python version, `str.format(...)` was used with certain keys passed in that could be used to format messages. In the Go version, we use a struct containing Alert info and the built in Go templating format. Eg.
 
 minitor-py:
 ```yaml
@@ -38,7 +17,7 @@ alerts:
   log_command:
     command: ['echo', '{monitor_name}']
   log_shell:
-    command_shell: "echo {monitor_name}"
+    command: 'echo {monitor_name}'
 ```
 
 minitor-go:
@@ -47,7 +26,7 @@ alerts:
   log_command:
     command: ['echo', '{{.MonitorName}}']
   log_shell:
-    command_shell: "echo {{.MonitorName}}"
+    command: 'echo {{.MonitorName}}'
 ```
 
 Finally, newlines in a shell command don't terminate a particular command. Semicolons must be used and continuations should not.
@@ -56,7 +35,7 @@ minitor-py:
 ```yaml
 alerts:
   log_shell:
-    command_shell: >
+    command: >
       echo "line 1"
       echo "line 2"
       echo "continued" \
@@ -67,7 +46,7 @@ minitor-go:
 ```yaml
 alerts:
   log_shell:
-    command_shell: >
+    command: >
       echo "line 1";
       echo "line 2";
       echo "continued"
@@ -84,10 +63,12 @@ Pairity:
   - [x] Run alert commands
   - [x] Run alert commands in a shell
   - [x] Allow templating of alert commands
-  - [ ] Implement Prometheus client to export metrics
+  - [x] Implement Prometheus client to export metrics
-  - [ ] Test coverage
+  - [x] Test coverage
+  - [x] Integration testing (manual or otherwise)
+  - [x] Allow commands and shell commands in the same config key
 
-Improvement:
+Improvement (potentially breaking):
 
   - [ ] Implement leveled logging (maybe glog or logrus)
   - [ ] Consider switching from YAML to TOML
@@ -95,3 +76,4 @@ Improvement:
   - [ ] Consider dropping `alert_up` and `alert_down` in favor of using Go templates that offer more control of messaging
   - [ ] Async checking
   - [ ] Use durations rather than seconds checked in event loop
+  - [ ] Revisit metrics and see if they all make sense

alert.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"log"
 	"os/exec"
+	"strings"
 	"text/template"
 	"time"
 )
@@ -12,8 +13,7 @@ import (
 // Alert is a config driven mechanism for sending a notice
 type Alert struct {
 	Name                 string
-	Command              []string
+	Command              CommandOrShell
-	CommandShell         string `yaml:"command_shell"`
 	commandTemplate      []*template.Template
 	commandShellTemplate *template.Template
 }
@@ -31,26 +31,40 @@ type AlertNotice struct {
 // IsValid returns a boolean indicating if the Alert has been correctly
 // configured
 func (alert Alert) IsValid() bool {
-	atLeastOneCommand := (alert.CommandShell != "" || alert.Command != nil)
+	return !alert.Command.Empty()
-	atMostOneCommand := (alert.CommandShell == "" || alert.Command == nil)
-	return atLeastOneCommand && atMostOneCommand
 }
 
 // BuildTemplates compiles command templates for the Alert
 func (alert *Alert) BuildTemplates() error {
+	// TODO: Remove legacy template support later after 1.0
+	legacy := strings.NewReplacer(
+		"{alert_count}", "{{.AlertCount}}",
+		"{alert_message}", "{{.MonitorName}} check has failed {{.FailureCount}} times",
+		"{failure_count}", "{{.FailureCount}}",
+		"{last_output}", "{{.LastCheckOutput}}",
+		"{last_success}", "{{.LastSuccess}}",
+		"{monitor_name}", "{{.MonitorName}}",
+	)
 	if LogDebug {
 		log.Printf("DEBUG: Building template for alert %s", alert.Name)
 	}
-	if alert.commandTemplate == nil && alert.Command != nil {
+	if alert.commandTemplate == nil && alert.Command.Command != nil {
 		alert.commandTemplate = []*template.Template{}
-		for i, cmdPart := range alert.Command {
+		for i, cmdPart := range alert.Command.Command {
+			if PyCompat {
+				cmdPart = legacy.Replace(cmdPart)
+			}
 			alert.commandTemplate = append(alert.commandTemplate, template.Must(
 				template.New(alert.Name+string(i)).Parse(cmdPart),
 			))
 		}
-	} else if alert.commandShellTemplate == nil && alert.CommandShell != "" {
+	} else if alert.commandShellTemplate == nil && alert.Command.ShellCommand != "" {
+		shellCmd := alert.Command.ShellCommand
+		if PyCompat {
+			shellCmd = legacy.Replace(shellCmd)
+		}
 		alert.commandShellTemplate = template.Must(
-			template.New(alert.Name).Parse(alert.CommandShell),
+			template.New(alert.Name).Parse(shellCmd),
 		)
 	} else {
 		return fmt.Errorf("No template provided for alert %s", alert.Name)
@@ -60,7 +74,7 @@ func (alert *Alert) BuildTemplates() error {
 }
 
 // Send will send an alert notice by executing the command template
-func (alert Alert) Send(notice AlertNotice) (output_str string, err error) {
+func (alert Alert) Send(notice AlertNotice) (outputStr string, err error) {
 	log.Printf("INFO: Sending alert %s for %s", alert.Name, notice.MonitorName)
 	var cmd *exec.Cmd
 	if alert.commandTemplate != nil {
@@ -95,10 +109,23 @@ func (alert Alert) Send(notice AlertNotice) (output_str string, err error) {
 
 	var output []byte
 	output, err = cmd.CombinedOutput()
-	output_str = string(output)
+	outputStr = string(output)
 	if LogDebug {
-		log.Printf("DEBUG: Alert output for: %s\n---\n%s\n---", alert.Name, output_str)
+		log.Printf("DEBUG: Alert output for: %s\n---\n%s\n---", alert.Name, outputStr)
 	}
 
-	return output_str, err
+	return outputStr, err
+}
+
+// NewLogAlert creates an alert that does basic logging using echo
+func NewLogAlert() *Alert {
+	return &Alert{
+		Name: "log",
+		Command: CommandOrShell{
+			Command: []string{
+				"echo",
+				"{{.MonitorName}} check has failed {{.FailureCount}} times",
+			},
+		},
+	}
 }

alert_test.go

@@ -11,14 +11,9 @@ func TestAlertIsValid(t *testing.T) {
 		expected bool
 		name     string
 	}{
-		{Alert{Command: []string{"echo", "test"}}, true, "Command only"},
+		{Alert{Command: CommandOrShell{Command: []string{"echo", "test"}}}, true, "Command only"},
-		{Alert{CommandShell: "echo test"}, true, "CommandShell only"},
+		{Alert{Command: CommandOrShell{ShellCommand: "echo test"}}, true, "CommandShell only"},
 		{Alert{}, false, "No commands"},
-		{
-			Alert{Command: []string{"echo", "test"}, CommandShell: "echo test"},
-			false,
-			"Both commands",
-		},
 	}
 
 	for _, c := range cases {
@@ -39,39 +34,54 @@ func TestAlertSend(t *testing.T) {
 		expectedOutput string
 		expectErr      bool
 		name           string
+		pyCompat       bool
 	}{
 		{
-			Alert{Command: []string{"echo", "{{.MonitorName}}"}},
+			Alert{Command: CommandOrShell{Command: []string{"echo", "{{.MonitorName}}"}}},
 			AlertNotice{MonitorName: "test"},
 			"test\n",
 			false,
 			"Command with template",
+			false,
 		},
 		{
-			Alert{CommandShell: "echo {{.MonitorName}}"},
+			Alert{Command: CommandOrShell{ShellCommand: "echo {{.MonitorName}}"}},
 			AlertNotice{MonitorName: "test"},
 			"test\n",
 			false,
 			"Command shell with template",
+			false,
 		},
 		{
-			Alert{Command: []string{"echo", "{{.Bad}}"}},
+			Alert{Command: CommandOrShell{Command: []string{"echo", "{{.Bad}}"}}},
 			AlertNotice{MonitorName: "test"},
 			"",
 			true,
 			"Command with bad template",
+			false,
 		},
 		{
-			Alert{CommandShell: "echo {{.Bad}}"},
+			Alert{Command: CommandOrShell{ShellCommand: "echo {{.Bad}}"}},
 			AlertNotice{MonitorName: "test"},
 			"",
 			true,
 			"Command shell with bad template",
+			false,
+		},
+		{
+			Alert{Command: CommandOrShell{ShellCommand: "echo {alert_message}"}},
+			AlertNotice{MonitorName: "test", FailureCount: 1},
+			"test check has failed 1 times\n",
+			false,
+			"Command shell with legacy template",
+			true,
 		},
 	}
 
 	for _, c := range cases {
 		log.Printf("Testing case %s", c.name)
+		// Set PyCompat to value of compat flag
+		PyCompat = c.pyCompat
 		c.alert.BuildTemplates()
 		output, err := c.alert.Send(c.notice)
 		hasErr := (err != nil)
@@ -83,6 +93,8 @@ func TestAlertSend(t *testing.T) {
 			t.Errorf("Send(%v err), expected=%v actual=%v", c.name, "Err", err)
 			log.Printf("Case failed: %s", c.name)
 		}
+		// Set PyCompat back to default value
+		PyCompat = false
 		log.Println("-----")
 	}
 }
@@ -103,8 +115,8 @@ func TestAlertBuildTemplate(t *testing.T) {
 		expectErr bool
 		name      string
 	}{
-		{Alert{Command: []string{"echo", "test"}}, false, "Command only"},
+		{Alert{Command: CommandOrShell{Command: []string{"echo", "test"}}}, false, "Command only"},
-		{Alert{CommandShell: "echo test"}, false, "CommandShell only"},
+		{Alert{Command: CommandOrShell{ShellCommand: "echo test"}}, false, "CommandShell only"},
 		{Alert{}, true, "No commands"},
 	}
 

config.go

@@ -2,10 +2,10 @@ package main
 
 import (
 	"errors"
-	"gopkg.in/yaml.v2"
 	"io/ioutil"
 	"log"
-	"os"
+
+	"gopkg.in/yaml.v2"
 )
 
 // Config type is contains all provided user configuration
@@ -15,10 +15,54 @@ type Config struct {
 	Alerts        map[string]*Alert
 }
 
+// CommandOrShell type wraps a string or list of strings
+// for executing a command directly or in a shell
+type CommandOrShell struct {
+	ShellCommand string
+	Command      []string
+}
+
+// Empty checks if the Command has a value
+func (cos CommandOrShell) Empty() bool {
+	return (cos.ShellCommand == "" && cos.Command == nil)
+}
+
+// UnmarshalYAML allows unmarshalling either a string or slice of strings
+// and parsing them as either a command or a shell command.
+func (cos *CommandOrShell) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var cmd []string
+	err := unmarshal(&cmd)
+	// Error indicates this is shell command
+	if err != nil {
+		var shellCmd string
+		err := unmarshal(&shellCmd)
+		if err != nil {
+			return err
+		}
+		cos.ShellCommand = shellCmd
+	} else {
+		cos.Command = cmd
+	}
+	return nil
+}
+
 // IsValid checks config validity and returns true if valid
 func (config Config) IsValid() (isValid bool) {
 	isValid = true
 
+	// Validate alerts
+	if config.Alerts == nil || len(config.Alerts) == 0 {
+		// This should never happen because there is a default alert named 'log' for now
+		log.Printf("ERROR: Invalid alert configuration: Must provide at least one alert")
+		isValid = false
+	}
+	for _, alert := range config.Alerts {
+		if !alert.IsValid() {
+			log.Printf("ERROR: Invalid alert configuration: %s", alert.Name)
+			isValid = false
+		}
+	}
+
 	// Validate monitors
 	if config.Monitors == nil || len(config.Monitors) == 0 {
 		log.Printf("ERROR: Invalid monitor configuration: Must provide at least one monitor")
@@ -43,18 +87,6 @@ func (config Config) IsValid() (isValid bool) {
 		}
 	}
 
-	// Validate alerts
-	if config.Alerts == nil || len(config.Alerts) == 0 {
-		log.Printf("ERROR: Invalid alert configuration: Must provide at least one alert")
-		isValid = false
-	}
-	for _, alert := range config.Alerts {
-		if !alert.IsValid() {
-			log.Printf("ERROR: Invalid alert configuration: %s", alert.Name)
-			isValid = false
-		}
-	}
-
 	return
 }
 
@@ -77,14 +109,25 @@ func LoadConfig(filePath string) (config Config, err error) {
 		return
 	}
 
-	// TODO: Decide if this is better expanded here, or only when executing
+	err = yaml.Unmarshal(data, &config)
-	envExpanded := os.ExpandEnv(string(data))
-	err = yaml.Unmarshal([]byte(envExpanded), &config)
 	if err != nil {
 		return
 	}
 
-	log.Printf("config:\n%v\n", config)
+	if LogDebug {
+		log.Printf("DEBUG: Config values:\n%v\n", config)
+	}
+
+	// Add log alert if not present
+	if PyCompat {
+		// Intialize alerts list if not present
+		if config.Alerts == nil {
+			config.Alerts = map[string]*Alert{}
+		}
+		if _, ok := config.Alerts["log"]; !ok {
+			config.Alerts["log"] = NewLogAlert()
+		}
+	}
 
 	if !config.IsValid() {
 		err = errors.New("Invalid configuration")

config_test.go

@@ -10,22 +10,94 @@ func TestLoadConfig(t *testing.T) {
 		configPath string
 		expectErr  bool
 		name       string
+		pyCompat   bool
 	}{
-		{"./test/valid-config.yml", false, "Valid config file"},
+		{"./test/valid-config.yml", false, "Valid config file", false},
-		{"./test/does-not-exist", true, "Invalid config path"},
+		{"./test/valid-default-log-alert.yml", false, "Valid config file with default log alert PyCompat", true},
-		{"./test/invalid-config-type.yml", true, "Invalid config type for key"},
+		{"./test/valid-default-log-alert.yml", true, "Invalid config file no log alert", false},
-		{"./test/invalid-config-missing-alerts.yml", true, "Invalid config missing alerts"},
+		{"./test/does-not-exist", true, "Invalid config path", false},
-		{"./test/invalid-config-unknown-alert.yml", true, "Invalid config unknown alert"},
+		{"./test/invalid-config-type.yml", true, "Invalid config type for key", false},
+		{"./test/invalid-config-missing-alerts.yml", true, "Invalid config missing alerts", false},
+		{"./test/invalid-config-unknown-alert.yml", true, "Invalid config unknown alert", false},
 	}
 
 	for _, c := range cases {
 		log.Printf("Testing case %s", c.name)
+		// Set PyCompat based on compatibility mode
+		PyCompat = c.pyCompat
 		_, err := LoadConfig(c.configPath)
 		hasErr := (err != nil)
 		if hasErr != c.expectErr {
-			t.Errorf("LoadConfig(%v), expected=%v actual=%v", c.name, "Err", err)
+			t.Errorf("LoadConfig(%v), expected_error=%v actual=%v", c.name, c.expectErr, err)
 			log.Printf("Case failed: %s", c.name)
 		}
+		// Set PyCompat to default value
+		PyCompat = false
 		log.Println("-----")
 	}
 }
+
+// TestMultiLineConfig is a more complicated test stepping through the parsing
+// and execution of mutli-line strings presented in YAML
+func TestMultiLineConfig(t *testing.T) {
+	log.Println("Testing multi-line string config")
+	config, err := LoadConfig("./test/valid-verify-multi-line.yml")
+	if err != nil {
+		t.Fatalf("TestMultiLineConfig(load), expected=no_error actual=%v", err)
+	}
+
+	log.Println("-----")
+	log.Println("TestMultiLineConfig(parse > string)")
+	expected := "echo 'Some string with stuff'; echo \"<angle brackets>\"; exit 1\n"
+	actual := config.Monitors[0].Command.ShellCommand
+	if expected != actual {
+		t.Errorf("TestMultiLineConfig(>) failed")
+		t.Logf("string expected=`%v`", expected)
+		t.Logf("string actual  =`%v`", actual)
+		t.Logf("bytes expected=%v", []byte(expected))
+		t.Logf("bytes actual  =%v", []byte(actual))
+	}
+
+	log.Println("-----")
+	log.Println("TestMultiLineConfig(execute > string)")
+	_, notice := config.Monitors[0].Check()
+	if notice == nil {
+		t.Fatalf("Did not receive an alert notice")
+	}
+	expected = "Some string with stuff\n<angle brackets>\n"
+	actual = notice.LastCheckOutput
+	if expected != actual {
+		t.Errorf("TestMultiLineConfig(execute > string) check failed")
+		t.Logf("string expected=`%v`", expected)
+		t.Logf("string actual  =`%v`", actual)
+		t.Logf("bytes expected=%v", []byte(expected))
+		t.Logf("bytes actual  =%v", []byte(actual))
+	}
+
+	log.Println("-----")
+	log.Println("TestMultiLineConfig(parse | string)")
+	expected = "echo 'Some string with stuff'\necho '<angle brackets>'\n"
+	actual = config.Alerts["log_shell"].Command.ShellCommand
+	if expected != actual {
+		t.Errorf("TestMultiLineConfig(|) failed")
+		t.Logf("string expected=`%v`", expected)
+		t.Logf("string actual  =`%v`", actual)
+		t.Logf("bytes expected=%v", []byte(expected))
+		t.Logf("bytes actual  =%v", []byte(actual))
+	}
+
+	log.Println("-----")
+	log.Println("TestMultiLineConfig(execute | string)")
+	actual, err = config.Alerts["log_shell"].Send(AlertNotice{})
+	if err != nil {
+		t.Errorf("Execution of alert failed")
+	}
+	expected = "Some string with stuff\n<angle brackets>\n"
+	if expected != actual {
+		t.Errorf("TestMultiLineConfig(execute | string) check failed")
+		t.Logf("string expected=`%v`", expected)
+		t.Logf("string actual  =`%v`", actual)
+		t.Logf("bytes expected=%v", []byte(expected))
+		t.Logf("bytes actual  =%v", []byte(actual))
+	}
+}

go.mod

@@ -2,4 +2,7 @@ module git.iamthefij.com/iamthefij/minitor-go
 
 go 1.12
 
-require gopkg.in/yaml.v2 v2.2.4
+require (
+	github.com/prometheus/client_golang v1.2.1
+	gopkg.in/yaml.v2 v2.2.4
+)

go.sum

@@ -1,3 +1,78 @@
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA=
+github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI=
+github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
+github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
+github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

main.go

@@ -11,6 +11,16 @@ var (
 	// LogDebug will control whether debug messsages should be logged
 	LogDebug = false
 
+	// ExportMetrics will track whether or not we want to export metrics to prometheus
+	ExportMetrics = false
+	// MetricsPort is the port to expose metrics on
+	MetricsPort = 8080
+	// Metrics contains all active metrics
+	Metrics = NewMetrics()
+
+	// PyCompat enables support for legacy Python templates
+	PyCompat = false
+
 	// version of minitor being run
 	version = "dev"
 )
@@ -18,7 +28,13 @@ var (
 func checkMonitors(config *Config) error {
 	for _, monitor := range config.Monitors {
 		if monitor.ShouldCheck() {
-			_, alertNotice := monitor.Check()
+			success, alertNotice := monitor.Check()
+
+			hasAlert := alertNotice != nil
+
+			// Track status metrics
+			Metrics.SetMonitorStatus(monitor.Name, success)
+			Metrics.CountCheck(monitor.Name, success, hasAlert)
 
 			// Should probably consider refactoring everything below here
 			if alertNotice != nil {
@@ -50,6 +66,9 @@ func checkMonitors(config *Config) error {
 								err,
 							)
 						}
+
+						// Count alert metrics
+						Metrics.CountAlert(monitor.Name, alert.Name)
 					} else {
 						// This case should never actually happen since we validate against it
 						log.Printf("ERROR: Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
@@ -66,21 +85,30 @@ func checkMonitors(config *Config) error {
 func main() {
 	// Get debug flag
 	flag.BoolVar(&LogDebug, "debug", false, "Enables debug logs (default: false)")
+	flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)")
+	flag.BoolVar(&PyCompat, "py-compat", false, "Enables support for legacy Python Minitor config. Will eventually be removed. (default: false)")
 	var showVersion = flag.Bool("version", false, "Display the version of minitor and exit")
+	var configPath = flag.String("config", "config.yml", "Alternate configuration path (default: config.yml)")
 	flag.Parse()
 
 	// Print version if flag is provided
 	if *showVersion {
-		fmt.Println("Minitor version:", version)
+		log.Println("Minitor version:", version)
 		return
 	}
 
 	// Load configuration
-	config, err := LoadConfig("config.yml")
+	config, err := LoadConfig(*configPath)
 	if err != nil {
 		log.Fatalf("Error loading config: %v", err)
 	}
 
+	// Serve metrics exporter, if specified
+	if ExportMetrics {
+		log.Println("INFO: Exporting metrics to Prometheus")
+		go ServeMetrics()
+	}
+
 	// Start main loop
 	for {
 		err = checkMonitors(&config)

main_test.go

@@ -18,7 +18,7 @@ func TestCheckMonitors(t *testing.T) {
 				Monitors: []*Monitor{
 					&Monitor{
 						Name:    "Success",
-						Command: []string{"true"},
+						Command: CommandOrShell{Command: []string{"true"}},
 					},
 				},
 			},
@@ -30,12 +30,12 @@ func TestCheckMonitors(t *testing.T) {
 				Monitors: []*Monitor{
 					&Monitor{
 						Name:       "Failure",
-						Command:    []string{"false"},
+						Command:    CommandOrShell{Command: []string{"false"}},
 						AlertAfter: 1,
 					},
 					&Monitor{
 						Name:       "Failure",
-						Command:    []string{"false"},
+						Command:    CommandOrShell{Command: []string{"false"}},
 						AlertDown:  []string{"unknown"},
 						AlertAfter: 1,
 					},
@@ -49,12 +49,12 @@ func TestCheckMonitors(t *testing.T) {
 				Monitors: []*Monitor{
 					&Monitor{
 						Name:       "Success",
-						Command:    []string{"ls"},
+						Command:    CommandOrShell{Command: []string{"ls"}},
 						alertCount: 1,
 					},
 					&Monitor{
 						Name:       "Success",
-						Command:    []string{"true"},
+						Command:    CommandOrShell{Command: []string{"true"}},
 						AlertUp:    []string{"unknown"},
 						alertCount: 1,
 					},
@@ -68,14 +68,14 @@ func TestCheckMonitors(t *testing.T) {
 				Monitors: []*Monitor{
 					&Monitor{
 						Name:       "Failure",
-						Command:    []string{"false"},
+						Command:    CommandOrShell{Command: []string{"false"}},
 						AlertDown:  []string{"good"},
 						AlertAfter: 1,
 					},
 				},
 				Alerts: map[string]*Alert{
 					"good": &Alert{
-						Command: []string{"true"},
+						Command: CommandOrShell{Command: []string{"true"}},
 					},
 				},
 			},
@@ -87,7 +87,7 @@ func TestCheckMonitors(t *testing.T) {
 				Monitors: []*Monitor{
 					&Monitor{
 						Name:       "Failure",
-						Command:    []string{"false"},
+						Command:    CommandOrShell{Command: []string{"false"}},
 						AlertDown:  []string{"bad"},
 						AlertAfter: 1,
 					},
@@ -95,7 +95,7 @@ func TestCheckMonitors(t *testing.T) {
 				Alerts: map[string]*Alert{
 					"bad": &Alert{
 						Name:    "bad",
-						Command: []string{"false"},
+						Command: CommandOrShell{Command: []string{"false"}},
 					},
 				},
 			},

manifest.tmpl

@@ -0,0 +1,25 @@
+image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}
+{{#if build.tags}}
+tags:
+{{#each build.tags}}
+  - {{this}}
+{{/each}}
+{{/if}}
+manifests:
+  -
+    image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64
+    platform:
+      architecture: amd64
+      os: linux
+  -
+    image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64
+    platform:
+      architecture: arm64
+      os: linux
+      variant: v8
+  -
+    image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm
+    platform:
+      architecture: arm
+      os: linux
+      variant: v7

metrics.go

@@ -0,0 +1,101 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+// TODO: Not sure if this is the best way to handle. A global instance for
+// metrics isn't bad, but it might be nice to curry versions of the metrics
+// for each monitor. Especially since every monitor has it's own. Perhaps
+// another new function that essentially curries each metric for a given
+// monitor name would do. This could be run when validating monitors and
+// initializing alert templates.
+
+// MinitorMetrics contains all counters and metrics that Minitor will need to access
+type MinitorMetrics struct {
+	alertCount    *prometheus.CounterVec
+	checkCount    *prometheus.CounterVec
+	monitorStatus *prometheus.GaugeVec
+}
+
+// NewMetrics creates and initializes all metrics
+func NewMetrics() *MinitorMetrics {
+	// Initialize all metrics
+	metrics := &MinitorMetrics{
+		alertCount: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "minitor_alert_total",
+				Help: "Number of Minitor alerts",
+			},
+			[]string{"alert", "monitor"},
+		),
+		checkCount: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "minitor_check_total",
+				Help: "Number of Minitor checks",
+			},
+			[]string{"monitor", "status", "is_alert"},
+		),
+		monitorStatus: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Name: "minitor_monitor_up_count",
+				Help: "Status of currently responsive monitors",
+			},
+			[]string{"monitor"},
+		),
+	}
+
+	// Register newly created metrics
+	prometheus.MustRegister(metrics.alertCount)
+	prometheus.MustRegister(metrics.checkCount)
+	prometheus.MustRegister(metrics.monitorStatus)
+
+	return metrics
+}
+
+// SetMonitorStatus sets the current status of Monitor
+func (metrics *MinitorMetrics) SetMonitorStatus(monitor string, isUp bool) {
+	val := 0.0
+	if isUp {
+		val = 1.0
+	}
+	metrics.monitorStatus.With(prometheus.Labels{"monitor": monitor}).Set(val)
+}
+
+// CountCheck counts the result of a particular Monitor check
+func (metrics *MinitorMetrics) CountCheck(monitor string, isSuccess bool, isAlert bool) {
+	status := "failure"
+	if isSuccess {
+		status = "success"
+	}
+
+	alertVal := "false"
+	if isAlert {
+		alertVal = "true"
+	}
+
+	metrics.checkCount.With(
+		prometheus.Labels{"monitor": monitor, "status": status, "is_alert": alertVal},
+	).Inc()
+}
+
+// CountAlert counts an alert
+func (metrics *MinitorMetrics) CountAlert(monitor string, alert string) {
+	metrics.alertCount.With(
+		prometheus.Labels{
+			"alert":   alert,
+			"monitor": monitor,
+		},
+	).Inc()
+}
+
+// ServeMetrics starts an http server with a Prometheus metrics handler
+func ServeMetrics() {
+	http.Handle("/metrics", promhttp.Handler())
+	host := fmt.Sprintf(":%d", MetricsPort)
+	_ = http.ListenAndServe(host, nil)
+}

monitor.go

@@ -11,8 +11,7 @@ import (
 type Monitor struct {
 	// Config values
 	Name          string
-	Command       []string
+	Command       CommandOrShell
-	CommandShell  string   `yaml:"command_shell"`
 	AlertDown     []string `yaml:"alert_down"`
 	AlertUp       []string `yaml:"alert_up"`
 	CheckInterval float64  `yaml:"check_interval"`
@@ -29,10 +28,7 @@ type Monitor struct {
 // IsValid returns a boolean indicating if the Monitor has been correctly
 // configured
 func (monitor Monitor) IsValid() bool {
-	atLeastOneCommand := (monitor.CommandShell != "" || monitor.Command != nil)
+	return (!monitor.Command.Empty() &&
-	atMostOneCommand := (monitor.CommandShell == "" || monitor.Command == nil)
-	return (atLeastOneCommand &&
-		atMostOneCommand &&
 		monitor.getAlertAfter() > 0 &&
 		monitor.AlertDown != nil)
 }
@@ -52,10 +48,10 @@ func (monitor Monitor) ShouldCheck() bool {
 // and a possible AlertNotice
 func (monitor *Monitor) Check() (bool, *AlertNotice) {
 	var cmd *exec.Cmd
-	if monitor.Command != nil {
+	if monitor.Command.Command != nil {
-		cmd = exec.Command(monitor.Command[0], monitor.Command[1:]...)
+		cmd = exec.Command(monitor.Command.Command[0], monitor.Command.Command[1:]...)
 	} else {
-		cmd = ShellCommand(monitor.CommandShell)
+		cmd = ShellCommand(monitor.Command.ShellCommand)
 	}
 
 	output, err := cmd.CombinedOutput()
@@ -155,18 +151,16 @@ func (monitor Monitor) getAlertAfter() int16 {
 	// Zero is one!
 	if monitor.AlertAfter == 0 {
 		return 1
-	} else {
-		return monitor.AlertAfter
 	}
+	return monitor.AlertAfter
 }
 
 // GetAlertNames gives a list of alert names for a given monitor status
 func (monitor Monitor) GetAlertNames(up bool) []string {
 	if up {
 		return monitor.AlertUp
-	} else {
-		return monitor.AlertDown
 	}
+	return monitor.AlertDown
 }
 
 func (monitor Monitor) createAlertNotice(isUp bool) *AlertNotice {

monitor_test.go

@@ -13,16 +13,11 @@ func TestMonitorIsValid(t *testing.T) {
 		expected bool
 		name     string
 	}{
-		{Monitor{Command: []string{"echo", "test"}, AlertDown: []string{"log"}}, true, "Command only"},
+		{Monitor{Command: CommandOrShell{Command: []string{"echo", "test"}}, AlertDown: []string{"log"}}, true, "Command only"},
-		{Monitor{CommandShell: "echo test", AlertDown: []string{"log"}}, true, "CommandShell only"},
+		{Monitor{Command: CommandOrShell{ShellCommand: "echo test"}, AlertDown: []string{"log"}}, true, "CommandShell only"},
-		{Monitor{Command: []string{"echo", "test"}}, false, "No AlertDown"},
+		{Monitor{Command: CommandOrShell{Command: []string{"echo", "test"}}}, false, "No AlertDown"},
 		{Monitor{AlertDown: []string{"log"}}, false, "No commands"},
-		{
+		{Monitor{Command: CommandOrShell{Command: []string{"echo", "test"}}, AlertDown: []string{"log"}, AlertAfter: -1}, false, "Invalid alert threshold, -1"},
-			Monitor{Command: []string{"echo", "test"}, CommandShell: "echo test", AlertDown: []string{"log"}},
-			false,
-			"Both commands",
-		},
-		{Monitor{Command: []string{"echo", "test"}, AlertDown: []string{"log"}, AlertAfter: -1}, false, "Invalid alert threshold, -1"},
 	}
 
 	for _, c := range cases {
@@ -254,22 +249,22 @@ func TestMonitorCheck(t *testing.T) {
 		name    string
 	}{
 		{
-			Monitor{Command: []string{"echo", "success"}},
+			Monitor{Command: CommandOrShell{Command: []string{"echo", "success"}}},
 			expected{isSuccess: true, hasNotice: false, lastOutput: "success\n"},
 			"Test successful command",
 		},
 		{
-			Monitor{CommandShell: "echo success"},
+			Monitor{Command: CommandOrShell{ShellCommand: "echo success"}},
 			expected{isSuccess: true, hasNotice: false, lastOutput: "success\n"},
 			"Test successful command shell",
 		},
 		{
-			Monitor{Command: []string{"total", "failure"}},
+			Monitor{Command: CommandOrShell{Command: []string{"total", "failure"}}},
 			expected{isSuccess: false, hasNotice: true, lastOutput: ""},
 			"Test failed command",
 		},
 		{
-			Monitor{CommandShell: "false"},
+			Monitor{Command: CommandOrShell{ShellCommand: "false"}},
 			expected{isSuccess: false, hasNotice: true, lastOutput: ""},
 			"Test failed command shell",
 		},

sample-config.yml

@@ -1,21 +1,33 @@
-check_interval: 30
+---
+check_interval: 5
 
 monitors:
-  - name: My Website
+  - name: Fake Website
-    command: [ 'curl', '-s', '-o', '/dev/null', 'https://minitor.mon' ]
+    command: ['curl', '-s', '-o', '/dev/null', 'https://minitor.mon']
-    alert_down: [ log, mailgun_down, sms_down ]
+    alert_down: [log_down, mailgun_down, sms_down]
-    alert_up: [ log, email_up ]
+    alert_up: [log_up, email_up]
-    check_interval: 30 # Must be at minimum the global `check_interval`
+    check_interval: 10  # Must be at minimum the global `check_interval`
     alert_after: 3
     alert_every: -1  # Defaults to -1 for exponential backoff. 0 to disable repeating
+  - name: Real Website
+    command: ['curl', '-s', '-o', '/dev/null', 'https://google.com']
+    alert_down: [log_down, mailgun_down, sms_down]
+    alert_up: [log_up, email_up]
+    check_interval: 5
+    alert_after: 3
+    alert_every: -1
 
 alerts:
+  log_down:
+    command: ["echo", "Minitor failure for {{.MonitorName}}"]
+  log_up:
+    command: ["echo", "Minitor recovery for {{.MonitorName}}"]
   email_up:
-    command: [ sendmail, "me@minitor.mon",  "Recovered: {monitor_name}",  "We're back!" ]
+    command: [sendmail, "me@minitor.mon", "Recovered: {monitor_name}", "We're back!"]
   mailgun_down:
     command: >
       curl -s -X POST
-      -F subject="Alert! {monitor_name} failed"
+      -F subject="Alert! {{.MonitorName}} failed"
       -F from="Minitor <minitor@minitor.mon>"
       -F to=me@minitor.mon
       -F text="Our monitor failed"
@@ -23,7 +35,7 @@ alerts:
       -u "api:${MAILGUN_API_KEY}"
   sms_down:
     command: >
-      curl -s -X POST -F "Body=Failure! {monitor_name} has failed"
+      curl -s -X POST -F "Body=Failure! {{.MonitorName}} has failed"
       -F "From=${AVAILABLE_NUMBER}" -F "To=${MY_PHONE}"
       "https://api.twilio.com/2010-04-01/Accounts/${ACCOUNT_SID}/Messages"
       -u "${ACCOUNT_SID}:${AUTH_TOKEN}"

scripts/README.md

@@ -0,0 +1,5 @@
+# Minitor Scripts
+
+A collection of some handy scripts to use with Minitor
+
+These are not included with the Python package, but they are included in the Docker image in `/app/scripts`.

scripts/docker_check.sh

@@ -0,0 +1,51 @@
+#! /bin/bash
+set -e
+
+#################
+# docker_check.sh
+#
+# Checks the most recent state exit code of a Docker container
+#################
+
+# Docker host will default to a socket
+# To override, export DOCKER_HOST to a new hostname
+DOCKER_HOST="${DOCKER_HOST:=socket}"
+container_name="$1"
+
+# Curls Docker either using a socket or URL
+function curl_docker {
+    local path="$1"
+    if [ "$DOCKER_HOST" == "socket" ]; then
+        curl --unix-socket /var/run/docker.sock "http://localhost/$path" 2>/dev/null
+    else
+        curl "http://${DOCKER_HOST}/$path" 2>/dev/null
+    fi
+}
+
+# Returns caintainer ID for a given container name
+function get_container_id {
+    local container_name="$1"
+    curl_docker 'containers/json?all=1' \
+        | jq -r ".[] | {Id, Name: .Names[]} | select(.Name == \"/${container_name}\") | .Id"
+}
+
+# Returns container JSON
+function inspect_container {
+    local container_id=$1
+    curl_docker "containers/$container_id/json"
+}
+
+if [ -z "$container_name" ]; then
+    echo "Usage: $0 container_name"
+    echo "Will exit with the last status code of continer with provided name"
+    exit 1
+fi
+
+container_id=$(get_container_id $container_name)
+if [ -z "$container_id" ]; then
+    echo "ERROR: Could not find container with name: $container_name"
+    exit 1
+fi
+exit_code=$(inspect_container "$container_id" | jq -r .State.ExitCode)
+
+exit "$exit_code"

scripts/docker_healthcheck.sh

@@ -0,0 +1,61 @@
+#! /bin/bash
+set -e
+
+#################
+# docker_healthcheck.sh
+#
+# Returns the results of a Docker Healthcheck for a container
+#################
+
+# Docker host will default to a socket
+# To override, export DOCKER_HOST to a new hostname
+DOCKER_HOST="${DOCKER_HOST:=socket}"
+container_name="$1"
+
+# Curls Docker either using a socket or URL
+function curl_docker {
+    local path="$1"
+    if [ "$DOCKER_HOST" == "socket" ]; then
+        curl --unix-socket /var/run/docker.sock "http://localhost/$path" 2>/dev/null
+    else
+        curl "http://${DOCKER_HOST}/$path" 2>/dev/null
+    fi
+}
+
+# Returns caintainer ID for a given container name
+function get_container_id {
+    local container_name="$1"
+    curl_docker 'containers/json?all=1' \
+        | jq -r ".[] | {Id, Name: .Names[]} | select(.Name == \"/${container_name}\") | .Id"
+}
+
+# Returns container JSON
+function inspect_container {
+    local container_id="$1"
+    curl_docker "containers/$container_id/json"
+}
+
+if [ -z "$container_name" ]; then
+    echo "Usage: $0 container_name"
+    echo "Will return results of healthcheck for continer with provided name"
+    exit 1
+fi
+
+container_id=$(get_container_id "$container_name")
+if [ -z "$container_id" ]; then
+    echo "ERROR: Could not find container with name: $container_name"
+    exit 1
+fi
+health=$(inspect_container "$container_id" | jq -r '.State.Health.Status')
+
+case "$health" in
+    null)
+        echo "No healthcheck results"
+        ;;
+    starting|healthy)
+        echo "Status: '$health'"
+        ;;
+    *)
+        echo "Status: '$health'"
+        exit 1
+esac

test/invalid-config-missing-alerts.yml

@@ -6,4 +6,3 @@ monitors:
     alert_down: [ 'alert_down', 'log_shell', 'log_command' ]
     # alert_every: -1
     alert_every: 0
-

test/valid-config.yml

@@ -1,22 +1,23 @@
+---
 check_interval: 1
 
 monitors:
   - name: Command
     command: ['echo', '$PATH']
-    alert_down: [ 'log_command', 'log_shell' ]
+    alert_down: ['log_command', 'log_shell']
     alert_every: 0
   - name: Shell
-    command_shell: >
+    command: >
       echo 'Some string with stuff';
       echo 'another line';
       echo $PATH;
       exit 1
-    alert_down: [ 'log_command', 'log_shell' ]
+    alert_down: ['log_command', 'log_shell']
     alert_after: 5
     alert_every: 0
 
 alerts:
   log_command:
-    command: [ 'echo', 'regular', '"command!!!"', "{{.MonitorName}}" ]
+    command: ['echo', 'regular', '"command!!!"', "{{.MonitorName}}"]
   log_shell:
-    command_shell: echo "Failure on {{.MonitorName}} User is $USER"
+    command: echo "Failure on {{.MonitorName}} User is $USER"

test/valid-default-log-alert.yml

@@ -0,0 +1,8 @@
+---
+check_interval: 1
+
+monitors:
+  - name: Command
+    command: ['echo', '$PATH']
+    alert_down: ['log']
+    alert_every: 0

test/valid-verify-multi-line.yml

@@ -0,0 +1,18 @@
+---
+check_interval: 1
+
+monitors:
+  - name: Shell
+    command: >
+      echo 'Some string with stuff';
+      echo "<angle brackets>";
+      exit 1
+    alert_down: ['log_shell']
+    alert_after: 1
+    alert_every: 0
+
+alerts:
+  log_shell:
+    command: |
+      echo 'Some string with stuff'
+      echo '<angle brackets>'

util.go

@@ -5,19 +5,9 @@ import (
 	"strings"
 )
 
-// escapeCommandShell accepts a command to be executed by a shell and escapes it
-func escapeCommandShell(command string) string {
-	// Remove extra spaces and newlines from ends
-	command = strings.TrimSpace(command)
-	// TODO: Not sure if this part is actually needed. Should verify
-	// Escape double quotes since this will be passed in as an argument
-	command = strings.Replace(command, `"`, `\"`, -1)
-	return command
-}
-
 // ShellCommand takes a string and executes it as a command using `sh`
 func ShellCommand(command string) *exec.Cmd {
-	shellCommand := []string{"sh", "-c", escapeCommandShell(command)}
+	shellCommand := []string{"sh", "-c", strings.TrimSpace(command)}
 	//log.Printf("Shell command: %v", shellCommand)
 	return exec.Command(shellCommand[0], shellCommand[1:]...)
 }