WIP: Update logging to improve formatting a bit

Update Dockerfiles to newer (roughly) pinned versions
Update Dockerfiles to make this version runnable
2019-11-22 15:07:18 -08:00 · 2019-11-22 14:44:21 -08:00 · 2019-11-22 12:58:26 -08:00 · 2019-11-21 15:40:59 -08:00 · 2019-11-21 15:32:57 -08:00 · 2019-11-20 17:21:48 -08:00
20 changed files with 548 additions and 106 deletions
@@ -1,13 +1,59 @@
+---
 kind: pipeline
 name: test

 steps:
-  - name: build
-    image: golang:1.12
-    commands:
-      - make build

  - name: test
    image: golang:1.12
    commands:
+      - make build
      - make test
+
+  - name: check
+    image: python:3
+    commands:
+      - pip install pre-commit==1.20.0
+      - make check
+
+  - name: notify
+    image: drillster/drone-email
+    settings:
+      host:
+        from_secret: SMTP_HOST
+      username:
+        from_secret: SMTP_USER
+      password:
+        from_secret: SMTP_PASS
+      from: drone@iamthefij.com
+    when:
+      status: [changed, failure]
+
+---
+kind: pipeline
+name: publish
+
+depends_on:
+  - test
+
+trigger:
+  event:
+    - push
+    - tag
+  refs:
+    - refs/heads/master
+    - refs/tags/v*
+
+steps:
+
+  # Might consider moving this step into the previous pipeline
+  - name: push image
+    image: plugins/docker
+    settings:
+      repo: iamthefij/minitor-go
+      dockerfile: Dockerfile.multi-stage
+      auto_tag: true
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
@@ -0,0 +1,19 @@
+---
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.4.0
+    hooks:
+      - id: check-added-large-files
+      - id: check-yaml
+        args:
+          - --allow-multiple-documents
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-merge-conflict
+  - repo: git://github.com/dnephin/pre-commit-golang
+    rev: v0.3.5
+    hooks:
+      - id: go-fmt
+      - id: go-imports
+      # - id: gometalinter
+      # - id: golangci-lint
@@ -1,8 +1,23 @@
 ARG REPO=library
-FROM ${REPO}/busybox:latest
-WORKDIR /root/
+FROM ${REPO}/alpine:3.10
+RUN mkdir /app
+WORKDIR /app/

+# Copy minitor in
 ARG ARCH=amd64
 COPY ./minitor-go ./minitor

+# Add common checking tools
+RUN apk --no-cache add bash=~5.0 curl=~7.66 jq=~1.6
+
+# Add minitor user for running as non-root
+RUN addgroup -S minitor && adduser -S minitor -G minitor
+
+# Copy scripts
+COPY ./scripts /app/scripts
+RUN chmod -R 755 /app/scripts
+
+# Drop to non-root user
+USER minitor
+
 ENTRYPOINT [ "./minitor" ]
@@ -1,7 +1,7 @@
 ARG REPO=library
 FROM golang:1.12-alpine AS builder

-RUN apk add --no-cache git
+RUN apk add --no-cache git=~2

 RUN mkdir /app
 WORKDIR /app
@@ -16,8 +16,26 @@ ARG VERSION=dev
 ENV CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH}
 RUN go build -ldflags "-X main.version=${VERSION}" -a -installsuffix nocgo -o minitor .

-FROM ${REPO}/busybox:latest
-WORKDIR /root/
+FROM ${REPO}/alpine:3.10
+RUN mkdir /app
+WORKDIR /app/
+
+# Copy minitor in
 COPY --from=builder /app/minitor .

+# Add common checking tools
+RUN apk --no-cache add bash=~5.0 curl=~7.66 jq=~1.6
+
+# Add minitor user for running as non-root
+RUN addgroup -S minitor && adduser -S minitor -G minitor
+
+# Copy scripts
+COPY ./scripts /app/scripts
+RUN chmod -R 755 /app/scripts
+
+# Drop to non-root user
+USER minitor
+
 ENTRYPOINT [ "./minitor" ]
+
+# vim: set filetype=dockerfile:
@@ -1,6 +1,7 @@
+.PHONY: all
 DOCKER_TAG ?= minitor-go-${USER}

-.PHONY: test
+.PHONY: default
 default: test

 .PHONY: build
@@ -14,6 +15,10 @@ minitor-go:
 run: minitor-go build
 	./minitor-go -debug

+.PHONY: run-metrics
+run-metrics: minitor-go build
+	./minitor-go -debug -metrics
+
 .PHONY: test
 test:
 	go test -coverprofile=coverage.out
@@ -24,6 +29,15 @@ test:
 	@go tool cover -func=coverage.out | awk -v target=80.0% \
 		'/^total:/ { print "Total coverage: " $$3 " Minimum coverage: " target; if ($$3+0.0 >= target+0.0) print "ok"; else { print "fail"; exit 1; } }'

+# Installs pre-commit hooks
+.PHONY: install-hooks
+install-hooks:
+	pre-commit install --install-hooks
+
+# Checks files for encryption
+.PHONY: check
+check:
+	pre-commit run --all-files

 .PHONY: clean
 clean:
@@ -2,7 +2,7 @@

 A reimplementation of [Minitor](https://git.iamthefij/iamthefij/minitor) in Go

-Minitor is already a very minimal monitoring tool. Python 3 was a quick way to get something live, but Python itself comes with a very large footprint.Thus Go feels like a better fit for the project, longer term.
+Minitor is already a minimal monitoring tool. Python 3 was a quick way to get something live, but Python itself comes with a large footprint. Thus Go feels like a better fit for the project, longer term.

 Initial target is meant to be roughly compatible requiring only minor changes to configuration. Future iterations may diverge to take advantage of Go specific features.

@@ -30,7 +30,7 @@ monitors:
    command_shell: echo 'test'
 ```

-Second, templating for Alert messages has been updated. In the Python version, `str.format(...)` was used with certain keys passed in that could be used to format messages. In the Go version, we use a struct containing Alert info and the built in Go templating format. Eg.
+Second, templating for Alert messages has been updated. In the Python version, `str.format(...)` was used with certain keys passed in that could be used to format messages. In the Go version, we use a struct, `AlertNotice` defined in `alert.go` and the built in Go templating format. Eg.

 minitor-py:
 ```yaml
@@ -38,7 +38,7 @@ alerts:
  log_command:
    command: ['echo', '{monitor_name}']
  log_shell:
-    command_shell: "echo {monitor_name}"
+    command_shell: 'echo {monitor_name}'
 ```

 minitor-go:
@@ -47,7 +47,7 @@ alerts:
  log_command:
    command: ['echo', '{{.MonitorName}}']
  log_shell:
-    command_shell: "echo {{.MonitorName}}"
+    command_shell: 'echo {{.MonitorName}}'
 ```

 Finally, newlines in a shell command don't terminate a particular command. Semicolons must be used and continuations should not.
@@ -84,10 +84,11 @@ Pairity:
  - [x] Run alert commands
  - [x] Run alert commands in a shell
  - [x] Allow templating of alert commands
-  - [ ] Implement Prometheus client to export metrics
-  - [ ] Test coverage
+  - [x] Implement Prometheus client to export metrics
+  - [x] Test coverage
+  - [ ] Integration testing (manual or otherwise)

-Improvement:
+Improvement (potentially breaking):

  - [ ] Implement leveled logging (maybe glog or logrus)
  - [ ] Consider switching from YAML to TOML
@@ -95,3 +96,4 @@ Improvement:
  - [ ] Consider dropping `alert_up` and `alert_down` in favor of using Go templates that offer more control of messaging
  - [ ] Async checking
  - [ ] Use durations rather than seconds checked in event loop
+  - [ ] Revisit metrics and see if they all make sense
@@ -3,10 +3,11 @@ package main
 import (
 	"bytes"
 	"fmt"
-	"log"
 	"os/exec"
 	"text/template"
 	"time"
+
+	log "github.com/sirupsen/logrus"
 )

 // Alert is a config driven mechanism for sending a notice
@@ -38,9 +39,7 @@ func (alert Alert) IsValid() bool {

 // BuildTemplates compiles command templates for the Alert
 func (alert *Alert) BuildTemplates() error {
-	if LogDebug {
-		log.Printf("DEBUG: Building template for alert %s", alert.Name)
-	}
+	log.Debugf("Building template for alert %s", alert.Name)
 	if alert.commandTemplate == nil && alert.Command != nil {
 		alert.commandTemplate = []*template.Template{}
 		for i, cmdPart := range alert.Command {
@@ -60,8 +59,8 @@ func (alert *Alert) BuildTemplates() error {
 }

 // Send will send an alert notice by executing the command template
-func (alert Alert) Send(notice AlertNotice) (output_str string, err error) {
-	log.Printf("INFO: Sending alert %s for %s", alert.Name, notice.MonitorName)
+func (alert Alert) Send(notice AlertNotice) (outputStr string, err error) {
+	log.Infof("Sending alert %s for %s", alert.Name, notice.MonitorName)
 	var cmd *exec.Cmd
 	if alert.commandTemplate != nil {
 		command := []string{}
@@ -95,10 +94,8 @@ func (alert Alert) Send(notice AlertNotice) (output_str string, err error) {

 	var output []byte
 	output, err = cmd.CombinedOutput()
-	output_str = string(output)
-	if LogDebug {
-		log.Printf("DEBUG: Alert output for: %s\n---\n%s\n---", alert.Name, output_str)
-	}
+	outputStr = string(output)
+	log.Debugf("Alert output for: %s\n---\n%s\n---", alert.Name, outputStr)

-	return output_str, err
+	return outputStr, err
 }
@@ -1,8 +1,9 @@
 package main

 import (
-	"log"
 	"testing"
+
+	log "github.com/sirupsen/logrus"
 )

 func TestAlertIsValid(t *testing.T) {
@@ -22,13 +23,13 @@ func TestAlertIsValid(t *testing.T) {
 	}

 	for _, c := range cases {
-		log.Printf("Testing case %s", c.name)
+		log.Debugf("Testing case %s", c.name)
 		actual := c.alert.IsValid()
 		if actual != c.expected {
 			t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expected, actual)
-			log.Printf("Case failed: %s", c.name)
+			log.Debugf("Case failed: %s", c.name)
 		}
-		log.Println("-----")
+		log.Debugf("-----")
 	}
 }

@@ -71,19 +72,19 @@ func TestAlertSend(t *testing.T) {
 	}

 	for _, c := range cases {
-		log.Printf("Testing case %s", c.name)
+		log.Debugf("Testing case %s", c.name)
 		c.alert.BuildTemplates()
 		output, err := c.alert.Send(c.notice)
 		hasErr := (err != nil)
 		if output != c.expectedOutput {
 			t.Errorf("Send(%v output), expected=%v actual=%v", c.name, c.expectedOutput, output)
-			log.Printf("Case failed: %s", c.name)
+			log.Debugf("Case failed: %s", c.name)
 		}
 		if hasErr != c.expectErr {
 			t.Errorf("Send(%v err), expected=%v actual=%v", c.name, "Err", err)
-			log.Printf("Case failed: %s", c.name)
+			log.Debugf("Case failed: %s", c.name)
 		}
-		log.Println("-----")
+		log.Debugf("-----")
 	}
 }

@@ -94,7 +95,7 @@ func TestAlertSendNoTemplates(t *testing.T) {
 	if err == nil {
 		t.Errorf("Send(no template), expected=%v actual=%v", "Err", output)
 	}
-	log.Println("-----")
+	log.Debugf("-----")
 }

 func TestAlertBuildTemplate(t *testing.T) {
@@ -109,13 +110,13 @@ func TestAlertBuildTemplate(t *testing.T) {
 	}

 	for _, c := range cases {
-		log.Printf("Testing case %s", c.name)
+		log.Debugf("Testing case %s", c.name)
 		err := c.alert.BuildTemplates()
 		hasErr := (err != nil)
 		if hasErr != c.expectErr {
 			t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expectErr, err)
-			log.Printf("Case failed: %s", c.name)
+			log.Debugf("Case failed: %s", c.name)
 		}
-		log.Println("-----")
+		log.Debugf("-----")
 	}
 }
@@ -2,10 +2,11 @@ package main

 import (
 	"errors"
-	"gopkg.in/yaml.v2"
 	"io/ioutil"
-	"log"
 	"os"
+
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
 )

 // Config type is contains all provided user configuration
@@ -21,20 +22,20 @@ func (config Config) IsValid() (isValid bool) {

 	// Validate monitors
 	if config.Monitors == nil || len(config.Monitors) == 0 {
-		log.Printf("ERROR: Invalid monitor configuration: Must provide at least one monitor")
+		log.Errorf("Invalid monitor configuration: Must provide at least one monitor")
 		isValid = false
 	}
 	for _, monitor := range config.Monitors {
 		if !monitor.IsValid() {
-			log.Printf("ERROR: Invalid monitor configuration: %s", monitor.Name)
+			log.Errorf("Invalid monitor configuration: %s", monitor.Name)
 			isValid = false
 		}
 		// Check that all Monitor alerts actually exist
 		for _, isUp := range []bool{true, false} {
 			for _, alertName := range monitor.GetAlertNames(isUp) {
 				if _, ok := config.Alerts[alertName]; !ok {
-					log.Printf(
-						"ERROR: Invalid monitor configuration: %s. Unknown alert %s",
+					log.Errorf(
+						"Invalid monitor configuration: %s. Unknown alert %s",
 						monitor.Name, alertName,
 					)
 					isValid = false
@@ -45,12 +46,12 @@ func (config Config) IsValid() (isValid bool) {

 	// Validate alerts
 	if config.Alerts == nil || len(config.Alerts) == 0 {
-		log.Printf("ERROR: Invalid alert configuration: Must provide at least one alert")
+		log.Errorf("Invalid alert configuration: Must provide at least one alert")
 		isValid = false
 	}
 	for _, alert := range config.Alerts {
 		if !alert.IsValid() {
-			log.Printf("ERROR: Invalid alert configuration: %s", alert.Name)
+			log.Errorf("Invalid alert configuration: %s", alert.Name)
 			isValid = false
 		}
 	}
@@ -84,7 +85,7 @@ func LoadConfig(filePath string) (config Config, err error) {
 		return
 	}

-	log.Printf("config:\n%v\n", config)
+	log.Debugf("Config values:\n%v\n", config)

 	if !config.IsValid() {
 		err = errors.New("Invalid configuration")
@@ -2,4 +2,8 @@ module git.iamthefij.com/iamthefij/minitor-go

 go 1.12

-require gopkg.in/yaml.v2 v2.2.4
+require (
+	github.com/prometheus/client_golang v1.2.1
+	github.com/sirupsen/logrus v1.4.2
+	gopkg.in/yaml.v2 v2.2.4
+)
@@ -1,3 +1,80 @@
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA=
+github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI=
+github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
+github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
+github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -3,13 +3,18 @@ package main
 import (
 	"flag"
 	"fmt"
-	"log"
 	"time"
+
+	log "github.com/sirupsen/logrus"
 )

 var (
-	// LogDebug will control whether debug messsages should be logged
-	LogDebug = false
+	// ExportMetrics will track whether or not we want to export metrics to prometheus
+	ExportMetrics = false
+	// MetricsPort is the port to expose metrics on
+	MetricsPort = 8080
+	// Metrics contains all active metrics
+	Metrics = NewMetrics()

 	// version of minitor being run
 	version = "dev"
@@ -18,13 +23,17 @@ var (
 func checkMonitors(config *Config) error {
 	for _, monitor := range config.Monitors {
 		if monitor.ShouldCheck() {
-			_, alertNotice := monitor.Check()
+			success, alertNotice := monitor.Check()
+
+			hasAlert := alertNotice != nil
+
+			// Track status metrics
+			Metrics.SetMonitorStatus(monitor.Name, success)
+			Metrics.CountCheck(monitor.Name, success, hasAlert)

 			// Should probably consider refactoring everything below here
 			if alertNotice != nil {
-				if LogDebug {
-					log.Printf("DEBUG: Recieved an alert notice from %s", alertNotice.MonitorName)
-				}
+				log.Debugf("Recieved an alert notice from %s", alertNotice.MonitorName)
 				alertNames := monitor.GetAlertNames(alertNotice.IsUp)
 				if alertNames == nil {
 					// This should only happen for a recovery alert. AlertDown is validated not empty
@@ -50,6 +59,9 @@ func checkMonitors(config *Config) error {
 								err,
 							)
 						}
+
+						// Count alert metrics
+						Metrics.CountAlert(monitor.Name, alert.Name)
 					} else {
 						// This case should never actually happen since we validate against it
 						log.Printf("ERROR: Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
@@ -65,13 +77,19 @@ func checkMonitors(config *Config) error {

 func main() {
 	// Get debug flag
-	flag.BoolVar(&LogDebug, "debug", false, "Enables debug logs (default: false)")
+	var debug = flag.Bool("debug", false, "Enables debug logs (default: false)")
+	flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)")
 	var showVersion = flag.Bool("version", false, "Display the version of minitor and exit")
 	flag.Parse()

+	// Set debug if flag is set
+	if *debug {
+		log.SetLevel(log.DebugLevel)
+	}
+
 	// Print version if flag is provided
 	if *showVersion {
-		fmt.Println("Minitor version:", version)
+		log.Println("Minitor version:", version)
 		return
 	}

@@ -81,6 +99,12 @@ func main() {
 		log.Fatalf("Error loading config: %v", err)
 	}

+	// Serve metrics exporter, if specified
+	if ExportMetrics {
+		log.Println("INFO: Exporting metrics to Prometheus")
+		go ServeMetrics()
+	}
+
 	// Start main loop
 	for {
 		err = checkMonitors(&config)
@@ -0,0 +1,101 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+// TODO: Not sure if this is the best way to handle. A global instance for
+// metrics isn't bad, but it might be nice to curry versions of the metrics
+// for each monitor. Especially since every monitor has it's own. Perhaps
+// another new function that essentially curries each metric for a given
+// monitor name would do. This could be run when validating monitors and
+// initializing alert templates.
+
+// MinitorMetrics contains all counters and metrics that Minitor will need to access
+type MinitorMetrics struct {
+	alertCount    *prometheus.CounterVec
+	checkCount    *prometheus.CounterVec
+	monitorStatus *prometheus.GaugeVec
+}
+
+// NewMetrics creates and initializes all metrics
+func NewMetrics() *MinitorMetrics {
+	// Initialize all metrics
+	metrics := &MinitorMetrics{
+		alertCount: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "minitor_alert_total",
+				Help: "Number of Minitor alerts",
+			},
+			[]string{"alert", "monitor"},
+		),
+		checkCount: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "minitor_check_total",
+				Help: "Number of Minitor checks",
+			},
+			[]string{"monitor", "status", "is_alert"},
+		),
+		monitorStatus: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Name: "minitor_monitor_up_count",
+				Help: "Status of currently responsive monitors",
+			},
+			[]string{"monitor"},
+		),
+	}
+
+	// Register newly created metrics
+	prometheus.MustRegister(metrics.alertCount)
+	prometheus.MustRegister(metrics.checkCount)
+	prometheus.MustRegister(metrics.monitorStatus)
+
+	return metrics
+}
+
+// SetMonitorStatus sets the current status of Monitor
+func (metrics *MinitorMetrics) SetMonitorStatus(monitor string, isUp bool) {
+	val := 0.0
+	if isUp {
+		val = 1.0
+	}
+	metrics.monitorStatus.With(prometheus.Labels{"monitor": monitor}).Set(val)
+}
+
+// CountCheck counts the result of a particular Monitor check
+func (metrics *MinitorMetrics) CountCheck(monitor string, isSuccess bool, isAlert bool) {
+	status := "failure"
+	if isSuccess {
+		status = "success"
+	}
+
+	alertVal := "false"
+	if isAlert {
+		alertVal = "true"
+	}
+
+	metrics.checkCount.With(
+		prometheus.Labels{"monitor": monitor, "status": status, "is_alert": alertVal},
+	).Inc()
+}
+
+// CountAlert counts an alert
+func (metrics *MinitorMetrics) CountAlert(monitor string, alert string) {
+	metrics.alertCount.With(
+		prometheus.Labels{
+			"alert":   alert,
+			"monitor": monitor,
+		},
+	).Inc()
+}
+
+// ServeMetrics starts an http server with a Prometheus metrics handler
+func ServeMetrics() {
+	http.Handle("/metrics", promhttp.Handler())
+	host := fmt.Sprintf(":%d", MetricsPort)
+	_ = http.ListenAndServe(host, nil)
+}
@@ -1,10 +1,11 @@
 package main

 import (
-	"log"
 	"math"
 	"os/exec"
 	"time"
+
+	log "github.com/sirupsen/logrus"
 )

 // Monitor represents a particular periodic check of a command
@@ -70,20 +71,18 @@ func (monitor *Monitor) Check() (bool, *AlertNotice) {
 		alertNotice = monitor.failure()
 	}

-	if LogDebug {
-		log.Printf("DEBUG: Command output: %s", monitor.lastOutput)
-	}
+	log.Debugf("Command output: %s", monitor.lastOutput)
 	if err != nil {
-		if LogDebug {
-			log.Printf("DEBUG: Command result: %v", err)
-		}
+		log.Debugf("Command result: %v", err)
 	}

-	log.Printf(
-		"INFO: %s success=%t, alert=%t",
+	log.WithFields(log.Fields{
+		"monitor": monitor.Name,
+		"success": isSuccess,
+		"alert":   alertNotice != nil,
+	}).Infof(
+		"%s checked",
 		monitor.Name,
-		isSuccess,
-		alertNotice != nil,
 	)

 	return isSuccess, alertNotice
@@ -109,15 +108,13 @@ func (monitor *Monitor) failure() (notice *AlertNotice) {
 	monitor.failureCount++
 	// If we haven't hit the minimum failures, we can exit
 	if monitor.failureCount < monitor.getAlertAfter() {
-		if LogDebug {
-			log.Printf(
-				"DEBUG: %s failed but did not hit minimum failures. "+
+		log.Debugf(
+			"%s failed but did not hit minimum failures. "+
 				"Count: %v alert after: %v",
 			monitor.Name,
 			monitor.failureCount,
 			monitor.getAlertAfter(),
 		)
-		}
 		return
 	}

@@ -155,18 +152,16 @@ func (monitor Monitor) getAlertAfter() int16 {
 	// Zero is one!
 	if monitor.AlertAfter == 0 {
 		return 1
-	} else {
-		return monitor.AlertAfter
 	}
+	return monitor.AlertAfter
 }

 // GetAlertNames gives a list of alert names for a given monitor status
 func (monitor Monitor) GetAlertNames(up bool) []string {
 	if up {
 		return monitor.AlertUp
-	} else {
-		return monitor.AlertDown
 	}
+	return monitor.AlertDown
 }

 func (monitor Monitor) createAlertNotice(isUp bool) *AlertNotice {
@@ -1,29 +1,41 @@
-check_interval: 30
+---
+check_interval: 5

 monitors:
-  - name: My Website
+  - name: Fake Website
    command: ['curl', '-s', '-o', '/dev/null', 'https://minitor.mon']
-    alert_down: [ log, mailgun_down, sms_down ]
-    alert_up: [ log, email_up ]
-    check_interval: 30 # Must be at minimum the global `check_interval`
+    alert_down: [log_down, mailgun_down, sms_down]
+    alert_up: [log_up, email_up]
+    check_interval: 10  # Must be at minimum the global `check_interval`
    alert_after: 3
    alert_every: -1  # Defaults to -1 for exponential backoff. 0 to disable repeating
+  - name: Real Website
+    command: ['curl', '-s', '-o', '/dev/null', 'https://google.com']
+    alert_down: [log_down, mailgun_down, sms_down]
+    alert_up: [log_up, email_up]
+    check_interval: 5
+    alert_after: 3
+    alert_every: -1

 alerts:
+  log_down:
+    command: ["echo", "Minitor failure for {{.MonitorName}}"]
+  log_up:
+    command: ["echo", "Minitor recovery for {{.MonitorName}}"]
  email_up:
    command: [sendmail, "me@minitor.mon", "Recovered: {monitor_name}", "We're back!"]
  mailgun_down:
-    command: >
+    command_shell: >
      curl -s -X POST
-      -F subject="Alert! {monitor_name} failed"
+      -F subject="Alert! {{.MonitorName}} failed"
      -F from="Minitor <minitor@minitor.mon>"
      -F to=me@minitor.mon
      -F text="Our monitor failed"
      https://api.mailgun.net/v3/minitor.mon/messages
      -u "api:${MAILGUN_API_KEY}"
  sms_down:
-    command: >
-      curl -s -X POST -F "Body=Failure! {monitor_name} has failed"
+    command_shell: >
+      curl -s -X POST -F "Body=Failure! {{.MonitorName}} has failed"
      -F "From=${AVAILABLE_NUMBER}" -F "To=${MY_PHONE}"
      "https://api.twilio.com/2010-04-01/Accounts/${ACCOUNT_SID}/Messages"
      -u "${ACCOUNT_SID}:${AUTH_TOKEN}"
@@ -0,0 +1,5 @@
+# Minitor Scripts
+
+A collection of some handy scripts to use with Minitor
+
+These are not included with the Python package, but they are included in the Docker image in `/app/scripts`.
@@ -0,0 +1,51 @@
+#! /bin/bash
+set -e
+
+#################
+# docker_check.sh
+#
+# Checks the most recent state exit code of a Docker container
+#################
+
+# Docker host will default to a socket
+# To override, export DOCKER_HOST to a new hostname
+DOCKER_HOST="${DOCKER_HOST:=socket}"
+container_name="$1"
+
+# Curls Docker either using a socket or URL
+function curl_docker {
+    local path="$1"
+    if [ "$DOCKER_HOST" == "socket" ]; then
+        curl --unix-socket /var/run/docker.sock "http://localhost/$path" 2>/dev/null
+    else
+        curl "http://${DOCKER_HOST}/$path" 2>/dev/null
+    fi
+}
+
+# Returns caintainer ID for a given container name
+function get_container_id {
+    local container_name="$1"
+    curl_docker 'containers/json?all=1' \
+        | jq -r ".[] | {Id, Name: .Names[]} | select(.Name == \"/${container_name}\") | .Id"
+}
+
+# Returns container JSON
+function inspect_container {
+    local container_id=$1
+    curl_docker "containers/$container_id/json"
+}
+
+if [ -z "$container_name" ]; then
+    echo "Usage: $0 container_name"
+    echo "Will exit with the last status code of continer with provided name"
+    exit 1
+fi
+
+container_id=$(get_container_id $container_name)
+if [ -z "$container_id" ]; then
+    echo "ERROR: Could not find container with name: $container_name"
+    exit 1
+fi
+exit_code=$(inspect_container "$container_id" | jq -r .State.ExitCode)
+
+exit "$exit_code"
@@ -0,0 +1,61 @@
+#! /bin/bash
+set -e
+
+#################
+# docker_healthcheck.sh
+#
+# Returns the results of a Docker Healthcheck for a container
+#################
+
+# Docker host will default to a socket
+# To override, export DOCKER_HOST to a new hostname
+DOCKER_HOST="${DOCKER_HOST:=socket}"
+container_name="$1"
+
+# Curls Docker either using a socket or URL
+function curl_docker {
+    local path="$1"
+    if [ "$DOCKER_HOST" == "socket" ]; then
+        curl --unix-socket /var/run/docker.sock "http://localhost/$path" 2>/dev/null
+    else
+        curl "http://${DOCKER_HOST}/$path" 2>/dev/null
+    fi
+}
+
+# Returns caintainer ID for a given container name
+function get_container_id {
+    local container_name="$1"
+    curl_docker 'containers/json?all=1' \
+        | jq -r ".[] | {Id, Name: .Names[]} | select(.Name == \"/${container_name}\") | .Id"
+}
+
+# Returns container JSON
+function inspect_container {
+    local container_id="$1"
+    curl_docker "containers/$container_id/json"
+}
+
+if [ -z "$container_name" ]; then
+    echo "Usage: $0 container_name"
+    echo "Will return results of healthcheck for continer with provided name"
+    exit 1
+fi
+
+container_id=$(get_container_id "$container_name")
+if [ -z "$container_id" ]; then
+    echo "ERROR: Could not find container with name: $container_name"
+    exit 1
+fi
+health=$(inspect_container "$container_id" | jq -r '.State.Health.Status')
+
+case "$health" in
+    null)
+        echo "No healthcheck results"
+        ;;
+    starting|healthy)
+        echo "Status: '$health'"
+        ;;
+    *)
+        echo "Status: '$health'"
+        exit 1
+esac
@@ -6,4 +6,3 @@ monitors:
    alert_down: [ 'alert_down', 'log_shell', 'log_command' ]
    # alert_every: -1
    alert_every: 0
-
Author	SHA1	Message	Date
Ian Fijolek	f1451166e6	WIP: Update logging to improve formatting a bit	2019-11-22 15:07:18 -08:00
Ian Fijolek	f6ccd9a3bd	Update Dockerfiles to newer (roughly) pinned versions	2019-11-22 14:44:21 -08:00
Ian Fijolek	f463ef27b7	Update Dockerfiles to make this version runnable Should now have pairity in terms of system utilities and scripts for checking services	2019-11-22 12:58:26 -08:00
Ian Fijolek	76ae8f3a44	Do build and test in one step Speed up build time by moving these two tasks to one step so that a new container doesn't have to be spun up and the cached modules from the build step are reused in the test step.	2019-11-21 15:40:59 -08:00
Ian Fijolek	9b9f803231	Add pre-commit to the repo This adds pre-commit which can be used to enforce consistent style and common errors (like committing large files)	2019-11-21 15:32:57 -08:00
Ian Fijolek	b808df7365	Update README to indicate where to get alert template info	2019-11-20 17:21:48 -08:00
Ian Fijolek	b1422bbec2	Add split out metrics to a new make target In case trying to run without having an available port to serve http on	2019-11-15 17:36:10 -08:00
Ian Fijolek	604c27118a	Add Docker deploy pipeline	2019-11-15 17:30:29 -08:00
Ian Fijolek	b2d9882c91	Update readme with accurate To do status	2019-11-15 17:17:46 -08:00
Ian Fijolek	457e19af9b	Add prometheus metrics exporter This should add metrics parity to the Python version	2019-11-15 17:14:20 -08:00