Update curl version

Previously this was mistakenly returning after sending an alert. Now
all alerts will be sent unless there is an exception on one of them.

.drone.yml

@@ -3,31 +3,15 @@ kind: pipeline
 name: test
 
 steps:
-
   - name: test
-    image: golang:1.12
+    image: golang:1.17
+    environment:
+      VERSION: ${DRONE_TAG:-${DRONE_COMMIT}}
     commands:
-      - make build
       - make test
 
   - name: check
-    image: python:3
+    image: iamthefij/drone-pre-commit:personal
-    commands:
-      - pip install pre-commit==1.20.0
-      - make check
-
-  - name: notify
-    image: drillster/drone-email
-    settings:
-      host:
-        from_secret: SMTP_HOST
-      username:
-        from_secret: SMTP_USER
-      password:
-        from_secret: SMTP_PASS
-      from: drone@iamthefij.com
-    when:
-      status: [changed, failure]
 
 ---
 kind: pipeline
@@ -46,10 +30,36 @@ trigger:
 
 steps:
   - name: build all binaries
-    image: golang:1.12
+    image: golang:1.17
+    environment:
+      VERSION: ${DRONE_TAG:-${DRONE_COMMIT}}
     commands:
       - make all
 
+  - name: compress binaries for release
+    image: ubuntu
+    commands:
+      - find ./dist -type f -executable -execdir tar -czvf {}.tar.gz {} \;
+    when:
+      event: tag
+
+  - name: upload gitea release
+    image: plugins/gitea-release
+    settings:
+      title: ${DRONE_TAG}
+      files: dist/*.tar.gz
+      checksum:
+        - md5
+        - sha1
+        - sha256
+        - sha512
+      base_url:
+        from_secret: gitea_base_url
+      api_key:
+        from_secret: gitea_token
+    when:
+      event: tag
+
   - name: push image - arm
     image: plugins/docker
     settings:
@@ -100,15 +110,27 @@ steps:
       password:
         from_secret: docker_password
 
+---
+kind: pipeline
+name: notify
+
+depends_on:
+  - test
+  - publish
+
+trigger:
+  status:
+    - failure
+
+steps:
+
   - name: notify
     image: drillster/drone-email
     settings:
       host:
-        from_secret: SMTP_HOST
+        from_secret: SMTP_HOST  # pragma: whitelist secret
       username:
-        from_secret: SMTP_USER
+        from_secret: SMTP_USER  # pragma: whitelist secret
       password:
-        from_secret: SMTP_PASS
+        from_secret: SMTP_PASS  # pragma: whitelist secret
       from: drone@iamthefij.com
-    when:
-      status: [changed, failure]

.gitignore

@@ -17,5 +17,5 @@ config.yml
 
 # Output binary
 minitor
-minitor-linux-*
+minitor-go
-minitor-darwin-amd64
+dist/

.golangci.yml

@@ -0,0 +1,48 @@
+---
+linters:
+  enable:
+    - asciicheck
+    - bodyclose
+    - dogsled
+    - dupl
+    - exhaustive
+    - gochecknoinits
+    - gocognit
+    - gocritic
+    - gocyclo
+    - goerr113
+    - gofumpt
+    - goimports
+    - gomnd
+    - goprintffuncname
+    # - gosec
+    # - ifshort
+    - interfacer
+    - maligned
+    - misspell
+    - nakedret
+    - nestif
+    - nlreturn
+    - noctx
+    - unparam
+    - wsl
+    # - errorlint
+  disable:
+    - gochecknoglobals
+
+linters-settings:
+  gosec:
+    excludes:
+      - G204
+#   gomnd:
+#     settings:
+#       mnd:
+#         ignored-functions: math.*
+
+issues:
+  exclude-rules:
+    - path: _test\.go
+      linters:
+        - errcheck
+        - gosec
+        - maligned

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.4.0
+    rev: v3.4.0
     hooks:
       - id: check-added-large-files
       - id: check-yaml
@@ -10,10 +10,16 @@ repos:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-merge-conflict
+  - repo: https://github.com/golangci/golangci-lint
+    rev: v1.42.1
+    hooks:
+      - id: golangci-lint
   - repo: git://github.com/dnephin/pre-commit-golang
-    rev: v0.3.5
+    rev: v0.4.0
     hooks:
       - id: go-fmt
       - id: go-imports
-      # - id: gometalinter
+  - repo: https://github.com/hadolint/hadolint
-      # - id: golangci-lint
+    rev: v2.4.0
+    hooks:
+      - id: hadolint

Dockerfile

@@ -1,15 +1,11 @@
 ARG REPO=library
-FROM multiarch/qemu-user-static:4.2.0-2 as qemu-user-static
+FROM ${REPO}/alpine:3.12
-FROM ${REPO}/alpine:3.10
-
-# Copying all qemu files because amd64 doesn't exist and cannot condional copy
-COPY --from=qemu-user-static /usr/bin/qemu-* /usr/bin/
 
 RUN mkdir /app
 WORKDIR /app/
 
 # Add common checking tools
-RUN apk --no-cache add bash=~5.0 curl=~7.66 jq=~1.6
+RUN apk --no-cache add bash=~5.0 curl=~7.79 jq=~1.6
 
 # Add minitor user for running as non-root
 RUN addgroup -S minitor && adduser -S minitor -G minitor
@@ -20,7 +16,7 @@ RUN chmod -R 755 /app/scripts
 
 # Copy minitor in
 ARG ARCH=amd64
-COPY ./minitor-linux-${ARCH} ./minitor
+COPY ./dist/minitor-linux-${ARCH} ./minitor
 
 # Drop to non-root user
 USER minitor

Dockerfile.multi-stage

@@ -1,7 +1,5 @@
 ARG REPO=library
-FROM golang:1.12-alpine AS builder
+FROM golang:1.17 AS builder
-
-RUN apk add --no-cache git=~2
 
 RUN mkdir /app
 WORKDIR /app
@@ -16,7 +14,7 @@ ARG VERSION=dev
 ENV CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH}
 RUN go build -ldflags "-X main.version=${VERSION}" -a -installsuffix nocgo -o minitor .
 
-FROM ${REPO}/alpine:3.10
+FROM ${REPO}/alpine:3.12
 RUN mkdir /app
 WORKDIR /app/
 
@@ -24,7 +22,7 @@ WORKDIR /app/
 COPY --from=builder /app/minitor .
 
 # Add common checking tools
-RUN apk --no-cache add bash=~5.0 curl=~7.66 jq=~1.6
+RUN apk --no-cache add bash=~5.0 curl=~7.79 jq=~1.6
 
 # Add minitor user for running as non-root
 RUN addgroup -S minitor && adduser -S minitor -G minitor

Makefile

@@ -1,36 +1,43 @@
 DOCKER_TAG ?= minitor-go-${USER}
-GIT_TAG_NAME := $(shell git tag -l --contains HEAD)
+VERSION ?= $(shell git describe --tags --dirty)
-GIT_SHA := $(shell git rev-parse HEAD)
+GOFILES = *.go
-VERSION := $(if $(GIT_TAG_NAME),$(GIT_TAG_NAME),$(GIT_SHA))
+# Multi-arch targets are generated from this
+TARGET_ALIAS = minitor-linux-amd64 minitor-linux-arm minitor-linux-arm64 minitor-darwin-amd64
+TARGETS = $(addprefix dist/,$(TARGET_ALIAS))
+#
+# Default make target will run tests
+.DEFAULT_GOAL = test
 
+# Build all static Minitor binaries
 .PHONY: all
-all: minitor-linux-amd64 minitor-linux-arm minitor-linux-arm64
+all: $(TARGETS)
 
-.PHONY: default
+# Build all static Linux Minitor binaries. Used in Docker images
-default: test
+.PHONY: all-linux
+all-linux: $(filter dist/minitor-linux-%,$(TARGETS))
+
+# Build minitor for the current machine
+minitor: $(GOFILES)
+	@echo Version: $(VERSION)
+	go build -ldflags '-X "main.version=${VERSION}"' -o minitor
 
 .PHONY: build
 build: minitor
 
-minitor:
+# Run minitor for the current machine
-	@echo Version: $(VERSION)
-	go build -ldflags '-X "main.version=${VERSION}"' -o minitor
-
 .PHONY: run
-run: minitor build
+run: minitor
 	./minitor -debug
 
 .PHONY: run-metrics
-run-metrics: minitor build
+run-metrics: minitor
 	./minitor -debug -metrics
 
+# Run all tests
 .PHONY: test
 test:
 	go test -coverprofile=coverage.out
-	@echo
 	go tool cover -func=coverage.out
-	@echo
-	@# Check min coverage percentage
 	@go tool cover -func=coverage.out | awk -v target=80.0% \
 		'/^total:/ { print "Total coverage: " $$3 " Minimum coverage: " target; if ($$3+0.0 >= target+0.0) print "ok"; else { print "fail"; exit 1; } }'
 
@@ -39,7 +46,7 @@ test:
 install-hooks:
 	pre-commit install --install-hooks
 
-# Checks files for encryption
+# Runs pre-commit checks on files
 .PHONY: check
 check:
 	pre-commit run --all-files
@@ -47,9 +54,8 @@ check:
 .PHONY: clean
 clean:
 	rm -f ./minitor
-	rm -f ./minitor-linux-*
-	rm -f ./minitor-darwin-amd64
 	rm -f ./coverage.out
+	rm -fr ./dist
 
 .PHONY: docker-build
 docker-build:
@@ -60,35 +66,23 @@ docker-run: docker-build
 	docker run --rm -v $(shell pwd)/config.yml:/root/config.yml $(DOCKER_TAG)
 
 ## Multi-arch targets
+$(TARGETS): $(GOFILES)
+	mkdir -p ./dist
+	GOOS=$(word 2, $(subst -, ,$(@))) GOARCH=$(word 3, $(subst -, ,$(@))) CGO_ENABLED=0 \
+		 go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
+		 -o $@
 
-# Arch specific go build targets
+.PHONY: $(TARGET_ALIAS)
-minitor-darwin-amd64:
+$(TARGET_ALIAS):
-	GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 \
+	$(MAKE) $(addprefix dist/,$@)
-		   go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
-		   -o minitor-darwin-amd64
-
-minitor-linux-amd64:
-	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \
-		   go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
-		   -o minitor-linux-amd64
-
-minitor-linux-arm:
-	GOOS=linux GOARCH=arm CGO_ENABLED=0 \
-		   go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
-		   -o minitor-linux-arm
-
-minitor-linux-arm64:
-	GOOS=linux GOARCH=arm64 CGO_ENABLED=0 \
-		   go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
-		   -o minitor-linux-arm64
 
 # Arch specific docker build targets
 .PHONY: docker-build-arm
-docker-build-arm: minitor-linux-arm
+docker-build-arm: dist/minitor-linux-arm
 	docker build --build-arg REPO=arm32v7 --build-arg ARCH=arm . -t ${DOCKER_TAG}-linux-arm
 
-.PHONY: docker-build-arm
+.PHONY: docker-build-arm64
-docker-build-arm64: minitor-linux-arm64
+docker-build-arm64: dist/minitor-linux-arm64
 	docker build --build-arg REPO=arm64v8 --build-arg ARCH=arm64 . -t ${DOCKER_TAG}-linux-arm64
 
 # Cross run on host architechture

README.md

@@ -1,79 +1,167 @@
-# minitor-go
+# [minitor-go](https://git.iamthefij.com/iamthefij/minitor-go)
 
-A reimplementation of [Minitor](https://git.iamthefij.com/iamthefij/minitor) in Go
+A minimal monitoring system
+
+## What does it do?
+
+Minitor accepts a YAML configuration file with a set of commands to run and a set of alerts to execute when those commands fail. It is designed to be as simple as possible and relies on other command line tools to do checks and issue alerts.
+
+## But why?
+
+I'm running a few small services and found Sensu, Consul, Nagios, etc. to all be far too complicated for my usecase.
+
+## So how do I use it?
+
+### Running
+
+Install and execute with:
+
+```bash
+go get github.com/iamthefij/minitor-go
+minitor
+```
+
+If locally developing you can use:
+
+```bash
+make run
+```
+
+It will read the contents of `config.yml` and begin its loop. You could also run it directly and provide a new config file via the `-config` argument.
+
+
+#### Docker
+
+You can pull this repository directly from Docker:
+
+```bash
+docker pull iamthefij/minitor-go:latest
+```
+
+The Docker image uses a default `config.yml` that is copied from `sample-config.yml`. This won't really do anything for you, so when you run the Docker image, you should supply your own `config.yml` file:
+
+```bash
+docker run -v $PWD/config.yml:/app/config.yml iamthefij/minitor-go:latest
+```
+
+Images are provided for `amd64`, `arm`, and `arm64` architechtures.
+
+## Configuring
+
+In this repo, you can explore the `sample-config.yml` file for an example, but the general structure is as follows. It should be noted that environment variable interpolation happens on load of the YAML file.
+
+The global configurations are:
+
+|key|value|
+|---|---|
+|`check_interval`|Maximum frequency to run checks for each monitor as duration, eg. 1m2s.|
+|`monitors`|List of all monitors. Detailed description below|
+|`alerts`|List of all alerts. Detailed description below|
+
+### Monitors
+
+All monitors should be listed under `monitors`.
+
+Each monitor allows the following configuration:
+
+|key|value|
+|---|---|
+|`name`|Name of the monitor running. This will show up in messages and logs.|
+|`command`|Specifies the command that should be executed, either in exec or shell form. This command's exit value will determine whether the check is successful|
+|`alert_down`|A list of Alerts to be triggered when the monitor is in a "down" state|
+|`alert_up`|A list of Alerts to be triggered when the monitor moves to an "up" state|
+|`check_interval`|The interval at which this monitor should be checked. This must be greater than the global `check_interval` value|
+|`alert_after`|Allows specifying the number of failed checks before an alert should be triggered|
+|`alert_every`|Allows specifying how often an alert should be retriggered. There are a few magic numbers here. Defaults to `-1` for an exponential backoff. Setting to `0` disables re-alerting. Positive values will allow retriggering after the specified number of checks|
+
+### Alerts
+
+Alerts exist as objects keyed under `alerts`. Their key should be the name of the Alert. This is used in your monitor setup in `alert_down` and `alert_up`.
+
+Eachy alert allows the following configuration:
+
+|key|value|
+|---|---|
+|`command`|Specifies the command that should be executed, either in exec or shell form. This is the command that will be run when the alert is executed. This can be templated with environment variables or the variables shown in the table below|
+
+Also, when alerts are executed, they will be passed through Go's format function with arguments for some attributes of the Monitor. The following monitor specific variables can be referenced using Go formatting syntax:
+
+|token|value|
+|---|---|
+|`{{.AlertCount}}`|Number of times this monitor has alerted|
+|`{{.FailureCount}}`|The total number of sequential failed checks for this monitor|
+|`{{.LastCheckOutput}}`|The last returned value from the check command to either stderr or stdout|
+|`{{.LastSuccess}}`|The ISO datetime of the last successful check|
+|`{{.MonitorName}}`|The name of the monitor that failed and triggered the alert|
+|`{{.IsUp}}`|Indicates if the monitor that is alerting is up or not. Can be used in a conditional message template|
+
+### Metrics
+
+Minitor supports exporting metrics for [Prometheus](https://prometheus.io/). Prometheus is an open source tool for reading and querying metrics from different sources. Combined with another tool, [Grafana](https://grafana.com/), it allows building of charts and dashboards. You could also opt to just use Minitor to log check results, and instead do your alerting with Grafana.
+
+It is also possible to use the metrics endpoint for monitoring Minitor itself! This allows setting up multiple instances of Minitor on different servers and have them monitor each-other so that you can detect a minitor outage.
+
+To run minitor with metrics, use the `-metrics` flag. The metrics will be served on port `8080` by default, though it can be overriden using `-metrics-port`. They will be accessible on the path `/metrics`. Eg. `localhost:8080/metrics`.
+
+```bash
+minitor -metrics
+# or
+minitor -metrics -metrics-port 3000
+```
+
+## Contributing
+
+Whether you're looking to submit a patch or tell me I broke something, you can contribute through the Github mirror and I can merge PRs back to the source repository.
+
+Primary Repo: https://git.iamthefij.com/iamthefij/minitor.git
+
+Github Mirror: https://github.com/IamTheFij/minitor.git
+
+## Original Minitor
+
+This is a reimplementation of [Minitor](https://git.iamthefij.com/iamthefij/minitor) in Go
 
 Minitor is already a minimal monitoring tool. Python 3 was a quick way to get something live, but Python itself comes with a large footprint. Thus Go feels like a better fit for the project, longer term.
 
 Initial target is meant to be roughly compatible requiring only minor changes to configuration. Future iterations may diverge to take advantage of Go specific features.
 
-## Differences from Python version
+### Differences from Python version
-
 
 Templating for Alert messages has been updated. In the Python version, `str.format(...)` was used with certain keys passed in that could be used to format messages. In the Go version, we use a struct, `AlertNotice` defined in `alert.go` and the built in Go templating format. Eg.
 
 minitor-py:
 ```yaml
 alerts:
-  log_command:
+  log:
-    command: ['echo', '{monitor_name}']
-  log_shell:
     command: 'echo {monitor_name}'
 ```
 
 minitor-go:
 ```yaml
 alerts:
-  log_command:
+  log:
-    command: ['echo', '{{.MonitorName}}']
-  log_shell:
     command: 'echo {{.MonitorName}}'
 ```
 
-Finally, newlines in a shell command don't terminate a particular command. Semicolons must be used and continuations should not.
+Interval durations have changed from being an integer number of seconds to a duration string supported by Go, for example:
 
 minitor-py:
 ```yaml
-alerts:
+check_interval: 90
-  log_shell:
-    command: >
-      echo "line 1"
-      echo "line 2"
-      echo "continued" \
-        "line"
 ```
 
 minitor-go:
 ```yaml
-alerts:
+check_interval: 1m30s
-  log_shell:
-    command: >
-      echo "line 1";
-      echo "line 2";
-      echo "continued"
-        "line"
 ```
 
-## To do
+For the time being, legacy configs for the Python version of Minitor should be compatible if you apply the `-py-compat` flag when running Minitor. Eventually, this flag will go away when later breaking changes are introduced.
-There are two sets of task lists. The first is to get rough parity on key features with the Python version. The second is to make some improvements to the framework.
 
-Pairity:
+## Future
 
-  - [x] Run monitor commands
+Future, potentially breaking changes
-  - [x] Run monitor commands in a shell
-  - [x] Run alert commands
-  - [x] Run alert commands in a shell
-  - [x] Allow templating of alert commands
-  - [x] Implement Prometheus client to export metrics
-  - [x] Test coverage
-  - [x] Integration testing (manual or otherwise)
-  - [x] Allow commands and shell commands in the same config key
 
-Improvement (potentially breaking):
-
-  - [ ] Implement leveled logging (maybe glog or logrus)
-  - [ ] Consider switching from YAML to TOML
   - [ ] Consider value of templating vs injecting values into Env variables
-  - [ ] Consider dropping `alert_up` and `alert_down` in favor of using Go templates that offer more control of messaging
   - [ ] Async checking
-  - [ ] Use durations rather than seconds checked in event loop
   - [ ] Revisit metrics and see if they all make sense
+  - [ ] Consider dropping `alert_up` and `alert_down` in favor of using Go templates that offer more control of messaging (Breaking)

alert.go

@@ -2,11 +2,21 @@ package main
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
-	"log"
 	"os/exec"
+	"strings"
 	"text/template"
 	"time"
+
+	"git.iamthefij.com/iamthefij/slog"
+)
+
+var (
+	errNoTemplate = errors.New("no template")
+
+	// ErrAlertFailed indicates that an alert failed to send
+	ErrAlertFailed = errors.New("alert failed")
 )
 
 // Alert is a config driven mechanism for sending a notice
@@ -19,12 +29,12 @@ type Alert struct {
 
 // AlertNotice captures the context for an alert to be sent
 type AlertNotice struct {
-	MonitorName     string
 	AlertCount      int16
 	FailureCount    int16
-	LastCheckOutput string
-	LastSuccess     time.Time
 	IsUp            bool
+	LastSuccess     time.Time
+	MonitorName     string
+	LastCheckOutput string
 }
 
 // IsValid returns a boolean indicating if the Alert has been correctly
@@ -35,53 +45,83 @@ func (alert Alert) IsValid() bool {
 
 // BuildTemplates compiles command templates for the Alert
 func (alert *Alert) BuildTemplates() error {
-	if LogDebug {
+	// TODO: Remove legacy template support later after 1.0
-		log.Printf("DEBUG: Building template for alert %s", alert.Name)
+	legacy := strings.NewReplacer(
-	}
+		"{alert_count}", "{{.AlertCount}}",
-	if alert.commandTemplate == nil && alert.Command.Command != nil {
+		"{alert_message}", "{{.MonitorName}} check has failed {{.FailureCount}} times",
+		"{failure_count}", "{{.FailureCount}}",
+		"{last_output}", "{{.LastCheckOutput}}",
+		"{last_success}", "{{.LastSuccess}}",
+		"{monitor_name}", "{{.MonitorName}}",
+	)
+
+	slog.Debugf("Building template for alert %s", alert.Name)
+
+	switch {
+	case alert.commandTemplate == nil && alert.Command.Command != nil:
 		alert.commandTemplate = []*template.Template{}
 		for i, cmdPart := range alert.Command.Command {
+			if PyCompat {
+				cmdPart = legacy.Replace(cmdPart)
+			}
+
 			alert.commandTemplate = append(alert.commandTemplate, template.Must(
-				template.New(alert.Name+string(i)).Parse(cmdPart),
+				template.New(alert.Name+fmt.Sprint(i)).Parse(cmdPart),
 			))
 		}
-	} else if alert.commandShellTemplate == nil && alert.Command.ShellCommand != "" {
+	case alert.commandShellTemplate == nil && alert.Command.ShellCommand != "":
+		shellCmd := alert.Command.ShellCommand
+
+		if PyCompat {
+			shellCmd = legacy.Replace(shellCmd)
+		}
+
 		alert.commandShellTemplate = template.Must(
-			template.New(alert.Name).Parse(alert.Command.ShellCommand),
+			template.New(alert.Name).Parse(shellCmd),
 		)
-	} else {
+	default:
-		return fmt.Errorf("No template provided for alert %s", alert.Name)
+		return fmt.Errorf("No template provided for alert %s: %w", alert.Name, errNoTemplate)
 	}
 
 	return nil
 }
 
 // Send will send an alert notice by executing the command template
-func (alert Alert) Send(notice AlertNotice) (output_str string, err error) {
+func (alert Alert) Send(notice AlertNotice) (outputStr string, err error) {
-	log.Printf("INFO: Sending alert %s for %s", alert.Name, notice.MonitorName)
+	slog.Infof("Sending alert %s for %s", alert.Name, notice.MonitorName)
+
 	var cmd *exec.Cmd
-	if alert.commandTemplate != nil {
+
+	switch {
+	case alert.commandTemplate != nil:
 		command := []string{}
+
 		for _, cmdTmp := range alert.commandTemplate {
 			var commandBuffer bytes.Buffer
+
 			err = cmdTmp.Execute(&commandBuffer, notice)
 			if err != nil {
 				return
 			}
+
 			command = append(command, commandBuffer.String())
 		}
+
 		cmd = exec.Command(command[0], command[1:]...)
-	} else if alert.commandShellTemplate != nil {
+	case alert.commandShellTemplate != nil:
 		var commandBuffer bytes.Buffer
+
 		err = alert.commandShellTemplate.Execute(&commandBuffer, notice)
 		if err != nil {
 			return
 		}
+
 		shellCommand := commandBuffer.String()
 
 		cmd = ShellCommand(shellCommand)
-	} else {
+	default:
-		err = fmt.Errorf("No templates compiled for alert %v", alert.Name)
+		err = fmt.Errorf("No templates compiled for alert %s: %w", alert.Name, errNoTemplate)
+
 		return
 	}
 
@@ -92,10 +132,30 @@ func (alert Alert) Send(notice AlertNotice) (output_str string, err error) {
 
 	var output []byte
 	output, err = cmd.CombinedOutput()
-	output_str = string(output)
+	outputStr = string(output)
-	if LogDebug {
+	slog.Debugf("Alert output for: %s\n---\n%s\n---", alert.Name, outputStr)
-		log.Printf("DEBUG: Alert output for: %s\n---\n%s\n---", alert.Name, output_str)
+
+	if err != nil {
+		err = fmt.Errorf(
+			"Alert '%s' failed to send. Returned %v: %w",
+			alert.Name,
+			err,
+			ErrAlertFailed,
+		)
 	}
 
-	return output_str, err
+	return outputStr, err
+}
+
+// NewLogAlert creates an alert that does basic logging using echo
+func NewLogAlert() *Alert {
+	return &Alert{
+		Name: "log",
+		Command: CommandOrShell{
+			Command: []string{
+				"echo",
+				"{{.MonitorName}} {{if .IsUp}}has recovered{{else}}check has failed {{.FailureCount}} times{{end}}",
+			},
+		},
+	}
 }

alert_test.go

@@ -18,11 +18,13 @@ func TestAlertIsValid(t *testing.T) {
 
 	for _, c := range cases {
 		log.Printf("Testing case %s", c.name)
+
 		actual := c.alert.IsValid()
 		if actual != c.expected {
 			t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expected, actual)
 			log.Printf("Case failed: %s", c.name)
 		}
+
 		log.Println("-----")
 	}
 }
@@ -34,6 +36,7 @@ func TestAlertSend(t *testing.T) {
 		expectedOutput string
 		expectErr      bool
 		name           string
+		pyCompat       bool
 	}{
 		{
 			Alert{Command: CommandOrShell{Command: []string{"echo", "{{.MonitorName}}"}}},
@@ -41,6 +44,7 @@ func TestAlertSend(t *testing.T) {
 			"test\n",
 			false,
 			"Command with template",
+			false,
 		},
 		{
 			Alert{Command: CommandOrShell{ShellCommand: "echo {{.MonitorName}}"}},
@@ -48,6 +52,7 @@ func TestAlertSend(t *testing.T) {
 			"test\n",
 			false,
 			"Command shell with template",
+			false,
 		},
 		{
 			Alert{Command: CommandOrShell{Command: []string{"echo", "{{.Bad}}"}}},
@@ -55,6 +60,7 @@ func TestAlertSend(t *testing.T) {
 			"",
 			true,
 			"Command with bad template",
+			false,
 		},
 		{
 			Alert{Command: CommandOrShell{ShellCommand: "echo {{.Bad}}"}},
@@ -62,22 +68,62 @@ func TestAlertSend(t *testing.T) {
 			"",
 			true,
 			"Command shell with bad template",
+			false,
+		},
+		{
+			Alert{Command: CommandOrShell{ShellCommand: "echo {alert_message}"}},
+			AlertNotice{MonitorName: "test", FailureCount: 1},
+			"test check has failed 1 times\n",
+			false,
+			"Command shell with legacy template",
+			true,
+		},
+		// Test default log alert down
+		{
+			*NewLogAlert(),
+			AlertNotice{MonitorName: "Test", FailureCount: 1, IsUp: false},
+			"Test check has failed 1 times\n",
+			false,
+			"Default log alert down",
+			false,
+		},
+		// Test default log alert up
+		{
+			*NewLogAlert(),
+			AlertNotice{MonitorName: "Test", IsUp: true},
+			"Test has recovered\n",
+			false,
+			"Default log alert up",
+			false,
 		},
 	}
 
 	for _, c := range cases {
 		log.Printf("Testing case %s", c.name)
-		c.alert.BuildTemplates()
+		// Set PyCompat to value of compat flag
+		PyCompat = c.pyCompat
+
+		err := c.alert.BuildTemplates()
+		if err != nil {
+			t.Errorf("Send(%v output), error building templates: %v", c.name, err)
+		}
+
 		output, err := c.alert.Send(c.notice)
 		hasErr := (err != nil)
+
 		if output != c.expectedOutput {
 			t.Errorf("Send(%v output), expected=%v actual=%v", c.name, c.expectedOutput, output)
 			log.Printf("Case failed: %s", c.name)
 		}
+
 		if hasErr != c.expectErr {
 			t.Errorf("Send(%v err), expected=%v actual=%v", c.name, "Err", err)
 			log.Printf("Case failed: %s", c.name)
 		}
+
+		// Set PyCompat back to default value
+		PyCompat = false
+
 		log.Println("-----")
 	}
 }
@@ -85,10 +131,12 @@ func TestAlertSend(t *testing.T) {
 func TestAlertSendNoTemplates(t *testing.T) {
 	alert := Alert{}
 	notice := AlertNotice{}
+
 	output, err := alert.Send(notice)
 	if err == nil {
 		t.Errorf("Send(no template), expected=%v actual=%v", "Err", output)
 	}
+
 	log.Println("-----")
 }
 
@@ -107,10 +155,12 @@ func TestAlertBuildTemplate(t *testing.T) {
 		log.Printf("Testing case %s", c.name)
 		err := c.alert.BuildTemplates()
 		hasErr := (err != nil)
+
 		if hasErr != c.expectErr {
 			t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expectErr, err)
 			log.Printf("Case failed: %s", c.name)
 		}
+
 		log.Println("-----")
 	}
 }

config.go

@@ -3,14 +3,17 @@ package main
 import (
 	"errors"
 	"io/ioutil"
-	"log"
+	"time"
 
+	"git.iamthefij.com/iamthefij/slog"
 	"gopkg.in/yaml.v2"
 )
 
+var errInvalidConfig = errors.New("Invalid configuration")
+
 // Config type is contains all provided user configuration
 type Config struct {
-	CheckInterval int64 `yaml:"check_interval"`
+	CheckInterval SecondsOrDuration `yaml:"check_interval"`
 	Monitors      []*Monitor
 	Alerts        map[string]*Alert
 }
@@ -35,58 +38,99 @@ func (cos *CommandOrShell) UnmarshalYAML(unmarshal func(interface{}) error) erro
 	// Error indicates this is shell command
 	if err != nil {
 		var shellCmd string
+
 		err := unmarshal(&shellCmd)
 		if err != nil {
 			return err
 		}
+
 		cos.ShellCommand = shellCmd
 	} else {
 		cos.Command = cmd
 	}
+
 	return nil
 }
 
+// SecondsOrDuration wraps a duration value for parsing a duration or seconds from YAML
+// NOTE: This should be removed in favor of only parsing durations once compatibility is broken
+type SecondsOrDuration struct {
+	value time.Duration
+}
+
+// Value returns a duration value
+func (sod SecondsOrDuration) Value() time.Duration {
+	return sod.value
+}
+
+// UnmarshalYAML allows unmarshalling a duration value or seconds if an int was provided
+func (sod *SecondsOrDuration) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var seconds int64
+	err := unmarshal(&seconds)
+
+	if err == nil {
+		sod.value = time.Second * time.Duration(seconds)
+
+		return nil
+	}
+
+	// Error indicates that we don't have an int
+	err = unmarshal(&sod.value)
+
+	return err
+}
+
 // IsValid checks config validity and returns true if valid
 func (config Config) IsValid() (isValid bool) {
 	isValid = true
 
-	// Validate monitors
+	// Validate alerts
-	if config.Monitors == nil || len(config.Monitors) == 0 {
+	if config.Alerts == nil || len(config.Alerts) == 0 {
-		log.Printf("ERROR: Invalid monitor configuration: Must provide at least one monitor")
+		// This should never happen because there is a default alert named 'log' for now
+		slog.Errorf("Invalid alert configuration: Must provide at least one alert")
+
 		isValid = false
 	}
+
+	for _, alert := range config.Alerts {
+		if !alert.IsValid() {
+			slog.Errorf("Invalid alert configuration: %+v", alert.Name)
+
+			isValid = false
+		} else {
+			slog.Debugf("Loaded alert %s", alert.Name)
+		}
+	}
+
+	// Validate monitors
+	if config.Monitors == nil || len(config.Monitors) == 0 {
+		slog.Errorf("Invalid monitor configuration: Must provide at least one monitor")
+
+		isValid = false
+	}
+
 	for _, monitor := range config.Monitors {
 		if !monitor.IsValid() {
-			log.Printf("ERROR: Invalid monitor configuration: %s", monitor.Name)
+			slog.Errorf("Invalid monitor configuration: %s", monitor.Name)
+
 			isValid = false
 		}
 		// Check that all Monitor alerts actually exist
 		for _, isUp := range []bool{true, false} {
 			for _, alertName := range monitor.GetAlertNames(isUp) {
 				if _, ok := config.Alerts[alertName]; !ok {
-					log.Printf(
+					slog.Errorf(
-						"ERROR: Invalid monitor configuration: %s. Unknown alert %s",
+						"Invalid monitor configuration: %s. Unknown alert %s",
 						monitor.Name, alertName,
 					)
+
 					isValid = false
 				}
 			}
 		}
 	}
 
-	// Validate alerts
+	return isValid
-	if config.Alerts == nil || len(config.Alerts) == 0 {
-		log.Printf("ERROR: Invalid alert configuration: Must provide at least one alert")
-		isValid = false
-	}
-	for _, alert := range config.Alerts {
-		if !alert.IsValid() {
-			log.Printf("ERROR: Invalid alert configuration: %s", alert.Name)
-			isValid = false
-		}
-	}
-
-	return
 }
 
 // Init performs extra initialization on top of loading the config from file
@@ -113,17 +157,30 @@ func LoadConfig(filePath string) (config Config, err error) {
 		return
 	}
 
-	if LogDebug {
+	slog.Debugf("Config values:\n%v\n", config)
-		log.Printf("DEBUG: Config values:\n%v\n", config)
-	}
 
-	if !config.IsValid() {
+	// Add log alert if not present
-		err = errors.New("Invalid configuration")
+	if PyCompat {
-		return
+		// Initialize alerts list if not present
+		if config.Alerts == nil {
+			config.Alerts = map[string]*Alert{}
+		}
+
+		if _, ok := config.Alerts["log"]; !ok {
+			config.Alerts["log"] = NewLogAlert()
+		}
 	}
 
 	// Finish initializing configuration
-	err = config.Init()
+	if err = config.Init(); err != nil {
+		return
+	}
 
-	return
+	if !config.IsValid() {
+		err = errInvalidConfig
+
+		return
+	}
+
+	return config, err
 }

config_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"log"
 	"testing"
+	"time"
 )
 
 func TestLoadConfig(t *testing.T) {
@@ -10,30 +11,67 @@ func TestLoadConfig(t *testing.T) {
 		configPath string
 		expectErr  bool
 		name       string
+		pyCompat   bool
 	}{
-		{"./test/valid-config.yml", false, "Valid config file"},
+		{"./test/valid-config.yml", false, "Valid config file", false},
-		{"./test/does-not-exist", true, "Invalid config path"},
+		{"./test/valid-default-log-alert.yml", false, "Valid config file with default log alert PyCompat", true},
-		{"./test/invalid-config-type.yml", true, "Invalid config type for key"},
+		{"./test/valid-default-log-alert.yml", true, "Invalid config file no log alert", false},
-		{"./test/invalid-config-missing-alerts.yml", true, "Invalid config missing alerts"},
+		{"./test/does-not-exist", true, "Invalid config path", false},
-		{"./test/invalid-config-unknown-alert.yml", true, "Invalid config unknown alert"},
+		{"./test/invalid-config-type.yml", true, "Invalid config type for key", false},
+		{"./test/invalid-config-missing-alerts.yml", true, "Invalid config missing alerts", false},
+		{"./test/invalid-config-unknown-alert.yml", true, "Invalid config unknown alert", false},
 	}
 
 	for _, c := range cases {
 		log.Printf("Testing case %s", c.name)
+		// Set PyCompat based on compatibility mode
+		PyCompat = c.pyCompat
 		_, err := LoadConfig(c.configPath)
 		hasErr := (err != nil)
+
 		if hasErr != c.expectErr {
 			t.Errorf("LoadConfig(%v), expected_error=%v actual=%v", c.name, c.expectErr, err)
 			log.Printf("Case failed: %s", c.name)
 		}
-		log.Println("-----")
+
+		// Set PyCompat to default value
+		PyCompat = false
 	}
 }
 
+func TestIntervalParsing(t *testing.T) {
+	log.Printf("Testing case TestIntervalParsing")
+
+	config, err := LoadConfig("./test/valid-config.yml")
+	if err != nil {
+		t.Errorf("Failed loading config: %v", err)
+	}
+
+	oneSecond := time.Second
+	tenSeconds := 10 * time.Second
+	oneMinute := time.Minute
+
+	// validate top level interval seconds represented as an int
+	if config.CheckInterval.Value() != oneSecond {
+		t.Errorf("Incorrectly parsed int seconds. expected=%v actual=%v", oneSecond, config.CheckInterval)
+	}
+
+	if config.Monitors[0].CheckInterval.Value() != tenSeconds {
+		t.Errorf("Incorrectly parsed seconds duration. expected=%v actual=%v", oneSecond, config.CheckInterval)
+	}
+
+	if config.Monitors[1].CheckInterval.Value() != oneMinute {
+		t.Errorf("Incorrectly parsed seconds duration. expected=%v actual=%v", oneSecond, config.CheckInterval)
+	}
+
+	log.Println("-----")
+}
+
 // TestMultiLineConfig is a more complicated test stepping through the parsing
 // and execution of mutli-line strings presented in YAML
 func TestMultiLineConfig(t *testing.T) {
 	log.Println("Testing multi-line string config")
+
 	config, err := LoadConfig("./test/valid-verify-multi-line.yml")
 	if err != nil {
 		t.Fatalf("TestMultiLineConfig(load), expected=no_error actual=%v", err)
@@ -41,8 +79,10 @@ func TestMultiLineConfig(t *testing.T) {
 
 	log.Println("-----")
 	log.Println("TestMultiLineConfig(parse > string)")
+
 	expected := "echo 'Some string with stuff'; echo \"<angle brackets>\"; exit 1\n"
 	actual := config.Monitors[0].Command.ShellCommand
+
 	if expected != actual {
 		t.Errorf("TestMultiLineConfig(>) failed")
 		t.Logf("string expected=`%v`", expected)
@@ -53,12 +93,15 @@ func TestMultiLineConfig(t *testing.T) {
 
 	log.Println("-----")
 	log.Println("TestMultiLineConfig(execute > string)")
+
 	_, notice := config.Monitors[0].Check()
 	if notice == nil {
 		t.Fatalf("Did not receive an alert notice")
 	}
+
 	expected = "Some string with stuff\n<angle brackets>\n"
 	actual = notice.LastCheckOutput
+
 	if expected != actual {
 		t.Errorf("TestMultiLineConfig(execute > string) check failed")
 		t.Logf("string expected=`%v`", expected)
@@ -69,8 +112,10 @@ func TestMultiLineConfig(t *testing.T) {
 
 	log.Println("-----")
 	log.Println("TestMultiLineConfig(parse | string)")
+
 	expected = "echo 'Some string with stuff'\necho '<angle brackets>'\n"
 	actual = config.Alerts["log_shell"].Command.ShellCommand
+
 	if expected != actual {
 		t.Errorf("TestMultiLineConfig(|) failed")
 		t.Logf("string expected=`%v`", expected)
@@ -81,10 +126,12 @@ func TestMultiLineConfig(t *testing.T) {
 
 	log.Println("-----")
 	log.Println("TestMultiLineConfig(execute | string)")
+
 	actual, err = config.Alerts["log_shell"].Send(AlertNotice{})
 	if err != nil {
 		t.Errorf("Execution of alert failed")
 	}
+
 	expected = "Some string with stuff\n<angle brackets>\n"
 	if expected != actual {
 		t.Errorf("TestMultiLineConfig(execute | string) check failed")

go.mod

@@ -1,8 +1,9 @@
 module git.iamthefij.com/iamthefij/minitor-go
 
-go 1.12
+go 1.15
 
 require (
+	git.iamthefij.com/iamthefij/slog v1.3.0
 	github.com/prometheus/client_golang v1.2.1
 	gopkg.in/yaml.v2 v2.2.4
 )

go.sum

@@ -1,3 +1,5 @@
+git.iamthefij.com/iamthefij/slog v1.3.0 h1:4Hu5PQvDrW5e3FrTS3q2iIXW0iPvhNY/9qJsqDR3K3I=
+git.iamthefij.com/iamthefij/slog v1.3.0/go.mod h1:1RUj4hcCompZkAxXCRfUX786tb3cM/Zpkn97dGfUfbg=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=

main.go

@@ -1,16 +1,15 @@
 package main
 
 import (
+	"errors"
 	"flag"
 	"fmt"
-	"log"
 	"time"
+
+	"git.iamthefij.com/iamthefij/slog"
 )
 
 var (
-	// LogDebug will control whether debug messsages should be logged
-	LogDebug = false
-
 	// ExportMetrics will track whether or not we want to export metrics to prometheus
 	ExportMetrics = false
 	// MetricsPort is the port to expose metrics on
@@ -18,59 +17,72 @@ var (
 	// Metrics contains all active metrics
 	Metrics = NewMetrics()
 
+	// PyCompat enables support for legacy Python templates
+	PyCompat = false
+
 	// version of minitor being run
 	version = "dev"
+
+	errUnknownAlert = errors.New("unknown alert")
 )
 
+func sendAlerts(config *Config, monitor *Monitor, alertNotice *AlertNotice) error {
+	slog.Debugf("Received an alert notice from %s", alertNotice.MonitorName)
+	alertNames := monitor.GetAlertNames(alertNotice.IsUp)
+
+	if alertNames == nil {
+		// This should only happen for a recovery alert. AlertDown is validated not empty
+		slog.Warningf(
+			"Received alert, but no alert mechanisms exist. MonitorName=%s IsUp=%t",
+			alertNotice.MonitorName, alertNotice.IsUp,
+		)
+
+		return nil
+	}
+
+	for _, alertName := range alertNames {
+		if alert, ok := config.Alerts[alertName]; ok {
+			output, err := alert.Send(*alertNotice)
+			if err != nil {
+				slog.Errorf(
+					"Alert '%s' failed. result=%v: output=%s",
+					alert.Name,
+					err,
+					output,
+				)
+
+				return err
+			}
+
+			// Count alert metrics
+			Metrics.CountAlert(monitor.Name, alert.Name)
+		} else {
+			// This case should never actually happen since we validate against it
+			slog.Errorf("Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
+
+			return fmt.Errorf("unknown alert for monitor %s: %s: %w", alertNotice.MonitorName, alertName, errUnknownAlert)
+		}
+	}
+
+	return nil
+}
+
 func checkMonitors(config *Config) error {
+	// TODO: Run this in goroutines and capture exceptions
 	for _, monitor := range config.Monitors {
 		if monitor.ShouldCheck() {
 			success, alertNotice := monitor.Check()
-
 			hasAlert := alertNotice != nil
 
 			// Track status metrics
-			Metrics.SetMonitorStatus(monitor.Name, success)
+			Metrics.SetMonitorStatus(monitor.Name, monitor.IsUp())
-			Metrics.CountCheck(monitor.Name, success, hasAlert)
+			Metrics.CountCheck(monitor.Name, success, monitor.LastCheckMilliseconds(), hasAlert)
 
-			// Should probably consider refactoring everything below here
 			if alertNotice != nil {
-				if LogDebug {
+				err := sendAlerts(config, monitor, alertNotice)
-					log.Printf("DEBUG: Recieved an alert notice from %s", alertNotice.MonitorName)
+				// If there was an error in sending an alert, exit early and bubble it up
-				}
+				if err != nil {
-				alertNames := monitor.GetAlertNames(alertNotice.IsUp)
+					return err
-				if alertNames == nil {
-					// This should only happen for a recovery alert. AlertDown is validated not empty
-					log.Printf(
-						"WARNING: Recieved alert, but no alert mechanisms exist. MonitorName=%s IsUp=%t",
-						alertNotice.MonitorName, alertNotice.IsUp,
-					)
-				}
-				for _, alertName := range alertNames {
-					if alert, ok := config.Alerts[alertName]; ok {
-						output, err := alert.Send(*alertNotice)
-						if err != nil {
-							log.Printf(
-								"ERROR: Alert '%s' failed. result=%v: output=%s",
-								alert.Name,
-								err,
-								output,
-							)
-							return fmt.Errorf(
-								"Unsuccessfully triggered alert '%s'. "+
-									"Crashing to avoid false negatives: %v",
-								alert.Name,
-								err,
-							)
-						}
-
-						// Count alert metrics
-						Metrics.CountAlert(monitor.Name, alert.Name)
-					} else {
-						// This case should never actually happen since we validate against it
-						log.Printf("ERROR: Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
-						return fmt.Errorf("Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
-					}
 				}
 			}
 		}
@@ -80,38 +92,38 @@ func checkMonitors(config *Config) error {
 }
 
 func main() {
-	// Get debug flag
+	showVersion := flag.Bool("version", false, "Display the version of minitor and exit")
-	flag.BoolVar(&LogDebug, "debug", false, "Enables debug logs (default: false)")
+	configPath := flag.String("config", "config.yml", "Alternate configuration path (default: config.yml)")
+
+	flag.BoolVar(&slog.DebugLevel, "debug", false, "Enables debug logs (default: false)")
 	flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)")
-	var showVersion = flag.Bool("version", false, "Display the version of minitor and exit")
+	flag.BoolVar(&PyCompat, "py-compat", false, "Enables support for legacy Python Minitor config. Will eventually be removed. (default: false)")
+	flag.IntVar(&MetricsPort, "metrics-port", MetricsPort, "The port that Prometheus metrics should be exported on, if enabled. (default: 8080)")
 	flag.Parse()
 
 	// Print version if flag is provided
 	if *showVersion {
-		log.Println("Minitor version:", version)
+		fmt.Println("Minitor version:", version)
+
 		return
 	}
 
 	// Load configuration
-	config, err := LoadConfig("config.yml")
+	config, err := LoadConfig(*configPath)
-	if err != nil {
+	slog.OnErrFatalf(err, "Error loading config: %v", err)
-		log.Fatalf("Error loading config: %v", err)
-	}
 
 	// Serve metrics exporter, if specified
 	if ExportMetrics {
-		log.Println("INFO: Exporting metrics to Prometheus")
+		slog.Infof("Exporting metrics to Prometheus on port %d", MetricsPort)
+
 		go ServeMetrics()
 	}
 
 	// Start main loop
 	for {
 		err = checkMonitors(&config)
-		if err != nil {
+		slog.OnErrPanicf(err, "Error checking monitors")
-			panic(err)
-		}
 
-		sleepTime := time.Duration(config.CheckInterval) * time.Second
+		time.Sleep(config.CheckInterval.Value())
-		time.Sleep(sleepTime)
 	}
 }

main_test.go

@@ -16,7 +16,7 @@ func TestCheckMonitors(t *testing.T) {
 		{
 			config: Config{
 				Monitors: []*Monitor{
-					&Monitor{
+					{
 						Name:    "Success",
 						Command: CommandOrShell{Command: []string{"true"}},
 					},
@@ -28,36 +28,24 @@ func TestCheckMonitors(t *testing.T) {
 		{
 			config: Config{
 				Monitors: []*Monitor{
-					&Monitor{
+					{
 						Name:       "Failure",
 						Command:    CommandOrShell{Command: []string{"false"}},
 						AlertAfter: 1,
 					},
-					&Monitor{
-						Name:       "Failure",
-						Command:    CommandOrShell{Command: []string{"false"}},
-						AlertDown:  []string{"unknown"},
-						AlertAfter: 1,
-					},
 				},
 			},
 			expectErr: false,
-			name:      "Monitor failure, no and unknown alerts",
+			name:      "Monitor failure, no alerts",
 		},
 		{
 			config: Config{
 				Monitors: []*Monitor{
-					&Monitor{
+					{
 						Name:       "Success",
 						Command:    CommandOrShell{Command: []string{"ls"}},
 						alertCount: 1,
 					},
-					&Monitor{
-						Name:       "Success",
-						Command:    CommandOrShell{Command: []string{"true"}},
-						AlertUp:    []string{"unknown"},
-						alertCount: 1,
-					},
 				},
 			},
 			expectErr: false,
@@ -66,7 +54,35 @@ func TestCheckMonitors(t *testing.T) {
 		{
 			config: Config{
 				Monitors: []*Monitor{
-					&Monitor{
+					{
+						Name:       "Failure",
+						Command:    CommandOrShell{Command: []string{"false"}},
+						AlertDown:  []string{"unknown"},
+						AlertAfter: 1,
+					},
+				},
+			},
+			expectErr: true,
+			name:      "Monitor failure, unknown alerts",
+		},
+		{
+			config: Config{
+				Monitors: []*Monitor{
+					{
+						Name:       "Success",
+						Command:    CommandOrShell{Command: []string{"true"}},
+						AlertUp:    []string{"unknown"},
+						alertCount: 1,
+					},
+				},
+			},
+			expectErr: true,
+			name:      "Monitor recovery, unknown alerts",
+		},
+		{
+			config: Config{
+				Monitors: []*Monitor{
+					{
 						Name:       "Failure",
 						Command:    CommandOrShell{Command: []string{"false"}},
 						AlertDown:  []string{"good"},
@@ -74,7 +90,7 @@ func TestCheckMonitors(t *testing.T) {
 					},
 				},
 				Alerts: map[string]*Alert{
-					"good": &Alert{
+					"good": {
 						Command: CommandOrShell{Command: []string{"true"}},
 					},
 				},
@@ -85,7 +101,7 @@ func TestCheckMonitors(t *testing.T) {
 		{
 			config: Config{
 				Monitors: []*Monitor{
-					&Monitor{
+					{
 						Name:       "Failure",
 						Command:    CommandOrShell{Command: []string{"false"}},
 						AlertDown:  []string{"bad"},
@@ -93,7 +109,7 @@ func TestCheckMonitors(t *testing.T) {
 					},
 				},
 				Alerts: map[string]*Alert{
-					"bad": &Alert{
+					"bad": {
 						Name:    "bad",
 						Command: CommandOrShell{Command: []string{"false"}},
 					},
@@ -105,10 +121,16 @@ func TestCheckMonitors(t *testing.T) {
 	}
 
 	for _, c := range cases {
-		c.config.Init()
+		err := c.config.Init()
-		err := checkMonitors(&c.config)
+		if err != nil {
+			t.Errorf("checkMonitors(%s): unexpected error reading config: %v", c.name, err)
+		}
+
+		err = checkMonitors(&c.config)
 		if err == nil && c.expectErr {
 			t.Errorf("checkMonitors(%s): Expected panic, the code did not panic", c.name)
+		} else if err != nil && !c.expectErr {
+			t.Errorf("checkMonitors(%s): Did not expect an error, but we got one anyway: %v", c.name, err)
 		}
 	}
 }

metrics.go

@@ -19,6 +19,7 @@ import (
 type MinitorMetrics struct {
 	alertCount    *prometheus.CounterVec
 	checkCount    *prometheus.CounterVec
+	checkTime     *prometheus.GaugeVec
 	monitorStatus *prometheus.GaugeVec
 }
 
@@ -40,6 +41,13 @@ func NewMetrics() *MinitorMetrics {
 			},
 			[]string{"monitor", "status", "is_alert"},
 		),
+		checkTime: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Name: "minitor_check_milliseconds",
+				Help: "Time in miliseconds that a check ran for",
+			},
+			[]string{"monitor", "status"},
+		),
 		monitorStatus: prometheus.NewGaugeVec(
 			prometheus.GaugeOpts{
 				Name: "minitor_monitor_up_count",
@@ -52,6 +60,7 @@ func NewMetrics() *MinitorMetrics {
 	// Register newly created metrics
 	prometheus.MustRegister(metrics.alertCount)
 	prometheus.MustRegister(metrics.checkCount)
+	prometheus.MustRegister(metrics.checkTime)
 	prometheus.MustRegister(metrics.monitorStatus)
 
 	return metrics
@@ -63,11 +72,12 @@ func (metrics *MinitorMetrics) SetMonitorStatus(monitor string, isUp bool) {
 	if isUp {
 		val = 1.0
 	}
+
 	metrics.monitorStatus.With(prometheus.Labels{"monitor": monitor}).Set(val)
 }
 
 // CountCheck counts the result of a particular Monitor check
-func (metrics *MinitorMetrics) CountCheck(monitor string, isSuccess bool, isAlert bool) {
+func (metrics *MinitorMetrics) CountCheck(monitor string, isSuccess bool, ms int64, isAlert bool) {
 	status := "failure"
 	if isSuccess {
 		status = "success"
@@ -81,6 +91,10 @@ func (metrics *MinitorMetrics) CountCheck(monitor string, isSuccess bool, isAler
 	metrics.checkCount.With(
 		prometheus.Labels{"monitor": monitor, "status": status, "is_alert": alertVal},
 	).Inc()
+
+	metrics.checkTime.With(
+		prometheus.Labels{"monitor": monitor, "status": status},
+	).Set(float64(ms))
 }
 
 // CountAlert counts an alert
@@ -96,6 +110,8 @@ func (metrics *MinitorMetrics) CountAlert(monitor string, alert string) {
 // ServeMetrics starts an http server with a Prometheus metrics handler
 func ServeMetrics() {
 	http.Handle("/metrics", promhttp.Handler())
+
 	host := fmt.Sprintf(":%d", MetricsPort)
+
 	_ = http.ListenAndServe(host, nil)
 }

monitor.go

@@ -1,28 +1,31 @@
 package main
 
 import (
-	"log"
 	"math"
 	"os/exec"
 	"time"
+
+	"git.iamthefij.com/iamthefij/slog"
 )
 
 // Monitor represents a particular periodic check of a command
-type Monitor struct {
+type Monitor struct { //nolint:maligned
 	// Config values
+	AlertAfter    int16             `yaml:"alert_after"`
+	AlertEvery    int16             `yaml:"alert_every"`
+	CheckInterval SecondsOrDuration `yaml:"check_interval"`
 	Name          string
-	Command       CommandOrShell
 	AlertDown     []string `yaml:"alert_down"`
 	AlertUp       []string `yaml:"alert_up"`
-	CheckInterval float64  `yaml:"check_interval"`
+	Command       CommandOrShell
-	AlertAfter    int16    `yaml:"alert_after"`
+
-	AlertEvery    int16    `yaml:"alert_every"`
 	// Other values
-	lastCheck    time.Time
+	alertCount        int16
-	lastOutput   string
+	failureCount      int16
-	alertCount   int16
+	lastCheck         time.Time
-	failureCount int16
+	lastSuccess       time.Time
-	lastSuccess  time.Time
+	lastOutput        string
+	lastCheckDuration time.Duration
 }
 
 // IsValid returns a boolean indicating if the Monitor has been correctly
@@ -40,8 +43,9 @@ func (monitor Monitor) ShouldCheck() bool {
 		return true
 	}
 
-	sinceLastCheck := time.Now().Sub(monitor.lastCheck).Seconds()
+	sinceLastCheck := time.Since(monitor.lastCheck)
-	return sinceLastCheck >= monitor.CheckInterval
+
+	return sinceLastCheck >= monitor.CheckInterval.Value()
 }
 
 // Check will run the command configured by the Monitor and return a status
@@ -54,11 +58,14 @@ func (monitor *Monitor) Check() (bool, *AlertNotice) {
 		cmd = ShellCommand(monitor.Command.ShellCommand)
 	}
 
+	checkStartTime := time.Now()
 	output, err := cmd.CombinedOutput()
 	monitor.lastCheck = time.Now()
 	monitor.lastOutput = string(output)
+	monitor.lastCheckDuration = monitor.lastCheck.Sub(checkStartTime)
 
 	var alertNotice *AlertNotice
+
 	isSuccess := (err == nil)
 	if isSuccess {
 		alertNotice = monitor.success()
@@ -66,17 +73,11 @@ func (monitor *Monitor) Check() (bool, *AlertNotice) {
 		alertNotice = monitor.failure()
 	}
 
-	if LogDebug {
+	slog.Debugf("Command output: %s", monitor.lastOutput)
-		log.Printf("DEBUG: Command output: %s", monitor.lastOutput)
+	slog.OnErrWarnf(err, "Command result: %v", err)
-	}
-	if err != nil {
-		if LogDebug {
-			log.Printf("DEBUG: Command result: %v", err)
-		}
-	}
 
-	log.Printf(
+	slog.Infof(
-		"INFO: %s success=%t, alert=%t",
+		"%s success=%t, alert=%t",
 		monitor.Name,
 		isSuccess,
 		alertNotice != nil,
@@ -85,15 +86,22 @@ func (monitor *Monitor) Check() (bool, *AlertNotice) {
 	return isSuccess, alertNotice
 }
 
-func (monitor Monitor) isUp() bool {
+// IsUp returns the status of the current monitor
+func (monitor Monitor) IsUp() bool {
 	return monitor.alertCount == 0
 }
 
+// LastCheckMilliseconds gives number of miliseconds the last check ran for
+func (monitor Monitor) LastCheckMilliseconds() int64 {
+	return monitor.lastCheckDuration.Milliseconds()
+}
+
 func (monitor *Monitor) success() (notice *AlertNotice) {
-	if !monitor.isUp() {
+	if !monitor.IsUp() {
 		// Alert that we have recovered
 		notice = monitor.createAlertNotice(true)
 	}
+
 	monitor.failureCount = 0
 	monitor.alertCount = 0
 	monitor.lastSuccess = time.Now()
@@ -105,15 +113,14 @@ func (monitor *Monitor) failure() (notice *AlertNotice) {
 	monitor.failureCount++
 	// If we haven't hit the minimum failures, we can exit
 	if monitor.failureCount < monitor.getAlertAfter() {
-		if LogDebug {
+		slog.Debugf(
-			log.Printf(
+			"%s failed but did not hit minimum failures. "+
-				"DEBUG: %s failed but did not hit minimum failures. "+
+				"Count: %v alert after: %v",
-					"Count: %v alert after: %v",
+			monitor.Name,
-				monitor.Name,
+			monitor.failureCount,
-				monitor.failureCount,
+			monitor.getAlertAfter(),
-				monitor.getAlertAfter(),
+		)
-			)
+
-		}
 		return
 	}
 
@@ -121,19 +128,20 @@ func (monitor *Monitor) failure() (notice *AlertNotice) {
 	failureCount := (monitor.failureCount - monitor.getAlertAfter())
 
 	// Use alert cadence to determine if we should alert
-	if monitor.AlertEvery > 0 {
+	switch {
+	case monitor.AlertEvery > 0:
 		// Handle integer number of failures before alerting
 		if failureCount%monitor.AlertEvery == 0 {
 			notice = monitor.createAlertNotice(false)
 		}
-	} else if monitor.AlertEvery == 0 {
+	case monitor.AlertEvery == 0:
 		// Handle alerting on first failure only
 		if failureCount == 0 {
 			notice = monitor.createAlertNotice(false)
 		}
-	} else {
+	default:
 		// Handle negative numbers indicating an exponential backoff
-		if failureCount >= int16(math.Pow(2, float64(monitor.alertCount))-1) {
+		if failureCount >= int16(math.Pow(2, float64(monitor.alertCount))-1) { //nolint:gomnd
 			notice = monitor.createAlertNotice(false)
 		}
 	}
@@ -143,7 +151,7 @@ func (monitor *Monitor) failure() (notice *AlertNotice) {
 		monitor.alertCount++
 	}
 
-	return
+	return notice
 }
 
 func (monitor Monitor) getAlertAfter() int16 {
@@ -152,6 +160,7 @@ func (monitor Monitor) getAlertAfter() int16 {
 	if monitor.AlertAfter == 0 {
 		return 1
 	}
+
 	return monitor.AlertAfter
 }
 
@@ -160,6 +169,7 @@ func (monitor Monitor) GetAlertNames(up bool) []string {
 	if up {
 		return monitor.AlertUp
 	}
+
 	return monitor.AlertDown
 }
 

monitor_test.go

@@ -22,11 +22,13 @@ func TestMonitorIsValid(t *testing.T) {
 
 	for _, c := range cases {
 		log.Printf("Testing case %s", c.name)
+
 		actual := c.monitor.IsValid()
 		if actual != c.expected {
 			t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expected, actual)
 			log.Printf("Case failed: %s", c.name)
 		}
+
 		log.Println("-----")
 	}
 }
@@ -43,9 +45,9 @@ func TestMonitorShouldCheck(t *testing.T) {
 		name     string
 	}{
 		{Monitor{}, true, "Empty"},
-		{Monitor{lastCheck: timeNow, CheckInterval: 15}, false, "Just checked"},
+		{Monitor{lastCheck: timeNow, CheckInterval: SecondsOrDuration{time.Second * 15}}, false, "Just checked"},
-		{Monitor{lastCheck: timeTenSecAgo, CheckInterval: 15}, false, "-10s"},
+		{Monitor{lastCheck: timeTenSecAgo, CheckInterval: SecondsOrDuration{time.Second * 15}}, false, "-10s"},
-		{Monitor{lastCheck: timeTwentySecAgo, CheckInterval: 15}, true, "-20s"},
+		{Monitor{lastCheck: timeTwentySecAgo, CheckInterval: SecondsOrDuration{time.Second * 15}}, true, "-20s"},
 	}
 
 	for _, c := range cases {
@@ -56,7 +58,7 @@ func TestMonitorShouldCheck(t *testing.T) {
 	}
 }
 
-// TestMonitorIsUp tests the Monitor.isUp()
+// TestMonitorIsUp tests the Monitor.IsUp()
 func TestMonitorIsUp(t *testing.T) {
 	cases := []struct {
 		monitor  Monitor
@@ -71,11 +73,13 @@ func TestMonitorIsUp(t *testing.T) {
 
 	for _, c := range cases {
 		log.Printf("Testing case %s", c.name)
-		actual := c.monitor.isUp()
+
+		actual := c.monitor.IsUp()
 		if actual != c.expected {
-			t.Errorf("isUp(%v), expected=%t actual=%t", c.name, c.expected, actual)
+			t.Errorf("IsUp(%v), expected=%t actual=%t", c.name, c.expected, actual)
 			log.Printf("Case failed: %s", c.name)
 		}
+
 		log.Println("-----")
 	}
 }
@@ -96,11 +100,13 @@ func TestMonitorGetAlertNames(t *testing.T) {
 
 	for _, c := range cases {
 		log.Printf("Testing case %s", c.name)
+
 		actual := c.monitor.GetAlertNames(c.up)
 		if !EqualSliceString(actual, c.expected) {
 			t.Errorf("GetAlertNames(%v), expected=%v actual=%v", c.name, c.expected, actual)
 			log.Printf("Case failed: %s", c.name)
 		}
+
 		log.Println("-----")
 	}
 }
@@ -119,12 +125,15 @@ func TestMonitorSuccess(t *testing.T) {
 
 	for _, c := range cases {
 		log.Printf("Testing case %s", c.name)
+
 		notice := c.monitor.success()
 		hasNotice := (notice != nil)
+
 		if hasNotice != c.expectNotice {
 			t.Errorf("success(%v), expected=%t actual=%t", c.name, c.expectNotice, hasNotice)
 			log.Printf("Case failed: %s", c.name)
 		}
+
 		log.Println("-----")
 	}
 }
@@ -147,12 +156,15 @@ func TestMonitorFailureAlertAfter(t *testing.T) {
 
 	for _, c := range cases {
 		log.Printf("Testing case %s", c.name)
+
 		notice := c.monitor.failure()
 		hasNotice := (notice != nil)
+
 		if hasNotice != c.expectNotice {
 			t.Errorf("failure(%v), expected=%t actual=%t", c.name, c.expectNotice, hasNotice)
 			log.Printf("Case failed: %s", c.name)
 		}
+
 		log.Println("-----")
 	}
 }
@@ -195,10 +207,12 @@ func TestMonitorFailureAlertEvery(t *testing.T) {
 
 		notice := c.monitor.failure()
 		hasNotice := (notice != nil)
+
 		if hasNotice != c.expectNotice {
 			t.Errorf("failure(%v), expected=%t actual=%t", c.name, c.expectNotice, hasNotice)
 			log.Printf("Case failed: %s", c.name)
 		}
+
 		log.Println("-----")
 	}
 }
@@ -223,15 +237,18 @@ func TestMonitorFailureExponential(t *testing.T) {
 	// Unlike previous tests, this one requires a static Monitor with repeated
 	// calls to the failure method
 	monitor := Monitor{failureCount: 0, AlertAfter: 1, AlertEvery: -1}
+
 	for _, c := range cases {
 		log.Printf("Testing case %s", c.name)
 
 		notice := monitor.failure()
 		hasNotice := (notice != nil)
+
 		if hasNotice != c.expectNotice {
 			t.Errorf("failure(%v), expected=%t actual=%t", c.name, c.expectNotice, hasNotice)
 			log.Printf("Case failed: %s", c.name)
 		}
+
 		log.Println("-----")
 	}
 }
@@ -243,6 +260,7 @@ func TestMonitorCheck(t *testing.T) {
 		hasNotice  bool
 		lastOutput string
 	}
+
 	cases := []struct {
 		monitor Monitor
 		expect  expected
@@ -290,6 +308,7 @@ func TestMonitorCheck(t *testing.T) {
 			t.Errorf("Check(%v) (output), expected=%v actual=%v", c.name, c.expect.lastOutput, lastOutput)
 			log.Printf("Case failed: %s", c.name)
 		}
+
 		log.Println("-----")
 	}
 }

sample-config.yml

@@ -3,14 +3,14 @@ check_interval: 5
 
 monitors:
   - name: Fake Website
-    command: ['curl', '-s', '-o', '/dev/null', 'https://minitor.mon']
+    command: ["curl", "-s", "-o", "/dev/null", "https://minitor.mon"]
     alert_down: [log_down, mailgun_down, sms_down]
     alert_up: [log_up, email_up]
-    check_interval: 10  # Must be at minimum the global `check_interval`
+    check_interval: 10 # Must be at minimum the global `check_interval`
     alert_after: 3
-    alert_every: -1  # Defaults to -1 for exponential backoff. 0 to disable repeating
+    alert_every: -1 # Defaults to -1 for exponential backoff. 0 to disable repeating
   - name: Real Website
-    command: ['curl', '-s', '-o', '/dev/null', 'https://google.com']
+    command: ["curl", "-s", "-o", "/dev/null", "https://google.com"]
     alert_down: [log_down, mailgun_down, sms_down]
     alert_up: [log_up, email_up]
     check_interval: 5
@@ -23,7 +23,8 @@ alerts:
   log_up:
     command: ["echo", "Minitor recovery for {{.MonitorName}}"]
   email_up:
-    command: [sendmail, "me@minitor.mon", "Recovered: {monitor_name}", "We're back!"]
+    command:
+      [sendmail, "me@minitor.mon", "Recovered: {monitor_name}", "We're back!"]
   mailgun_down:
     command: >
       curl -s -X POST

scripts/docker_check.sh

@@ -11,6 +11,7 @@ set -e
 # To override, export DOCKER_HOST to a new hostname
 DOCKER_HOST="${DOCKER_HOST:=socket}"
 container_name="$1"
+num_log_lines="$2"
 
 # Curls Docker either using a socket or URL
 function curl_docker {
@@ -31,21 +32,32 @@ function get_container_id {
 
 # Returns container JSON
 function inspect_container {
-    local container_id=$1
+    local container_id="$1"
     curl_docker "containers/$container_id/json"
 }
 
+# Gets some lines from docker log
+function get_logs {
+    container_id="$1"
+    num_lines="$2"
+    curl_docker "containers/$container_id/logs?stdout=1&stderr=1" | tail -n "$num_lines"
+}
+
 if [ -z "$container_name" ]; then
-    echo "Usage: $0 container_name"
+    echo "Usage: $0 container_name [num_log_lines]"
     echo "Will exit with the last status code of continer with provided name"
     exit 1
 fi
 
-container_id=$(get_container_id $container_name)
+container_id=$(get_container_id "$container_name")
 if [ -z "$container_id" ]; then
     echo "ERROR: Could not find container with name: $container_name"
     exit 1
 fi
 exit_code=$(inspect_container "$container_id" | jq -r .State.ExitCode)
 
+if [ -n "$num_log_lines" ]; then
+    get_logs "$container_id" "$num_log_lines"
+fi
+
 exit "$exit_code"

scripts/docker_healthcheck.sh

@@ -11,6 +11,7 @@ set -e
 # To override, export DOCKER_HOST to a new hostname
 DOCKER_HOST="${DOCKER_HOST:=socket}"
 container_name="$1"
+num_log_lines="$2"
 
 # Curls Docker either using a socket or URL
 function curl_docker {
@@ -35,8 +36,15 @@ function inspect_container {
     curl_docker "containers/$container_id/json"
 }
 
+# Gets some lines from docker log
+function get_logs {
+    container_id="$1"
+    num_lines="$2"
+    curl_docker "containers/$container_id/logs?stdout=1&stderr=1" | tail -n "$num_lines"
+}
+
 if [ -z "$container_name" ]; then
-    echo "Usage: $0 container_name"
+    echo "Usage: $0 container_name [num_log_lines]"
     echo "Will return results of healthcheck for continer with provided name"
     exit 1
 fi
@@ -48,6 +56,10 @@ if [ -z "$container_id" ]; then
 fi
 health=$(inspect_container "$container_id" | jq -r '.State.Health.Status')
 
+if [ -n "$num_log_lines" ]; then
+    get_logs "$container_id" "$num_log_lines"
+fi
+
 case "$health" in
     null)
         echo "No healthcheck results"

test/valid-config.yml

@@ -3,21 +3,23 @@ check_interval: 1
 
 monitors:
   - name: Command
-    command: ['echo', '$PATH']
+    command: ["echo", "$PATH"]
-    alert_down: ['log_command', 'log_shell']
+    alert_down: ["log_command", "log_shell"]
     alert_every: 0
+    check_interval: 10s
   - name: Shell
     command: >
       echo 'Some string with stuff';
       echo 'another line';
       echo $PATH;
       exit 1
-    alert_down: ['log_command', 'log_shell']
+    alert_down: ["log_command", "log_shell"]
     alert_after: 5
     alert_every: 0
+    check_interval: 1m
 
 alerts:
   log_command:
-    command: ['echo', 'regular', '"command!!!"', "{{.MonitorName}}"]
+    command: ["echo", "regular", '"command!!!"', "{{.MonitorName}}"]
   log_shell:
     command: echo "Failure on {{.MonitorName}} User is $USER"

test/valid-default-log-alert.yml

@@ -0,0 +1,8 @@
+---
+check_interval: 1
+
+monitors:
+  - name: Command
+    command: ['echo', '$PATH']
+    alert_down: ['log']
+    alert_every: 0

util.go

@@ -8,7 +8,7 @@ import (
 // ShellCommand takes a string and executes it as a command using `sh`
 func ShellCommand(command string) *exec.Cmd {
 	shellCommand := []string{"sh", "-c", strings.TrimSpace(command)}
-	//log.Printf("Shell command: %v", shellCommand)
+
 	return exec.Command(shellCommand[0], shellCommand[1:]...)
 }
 
@@ -17,10 +17,12 @@ func EqualSliceString(a, b []string) bool {
 	if len(a) != len(b) {
 		return false
 	}
+
 	for i, val := range a {
 		if val != b[i] {
 			return false
 		}
 	}
+
 	return true
 }