Compare commits

..

1 Commits

Author SHA1 Message Date
IamTheFij
f66276c24f Fix ownership of app directory
This seems to be causing an error on startup.
2019-12-21 08:26:04 -08:00
27 changed files with 375 additions and 1055 deletions
+23 -100
View File
@@ -3,15 +3,31 @@ kind: pipeline
name: test name: test
steps: steps:
- name: test - name: test
image: golang:1.15 image: golang:1.12
environment:
VERSION: ${DRONE_TAG:-${DRONE_COMMIT}}
commands: commands:
- make build
- make test - make test
- name: check - name: check
image: iamthefij/drone-pre-commit:personal image: python:3
commands:
- pip install pre-commit==1.20.0
- make check
- name: notify
image: drillster/drone-email
settings:
host:
from_secret: SMTP_HOST
username:
from_secret: SMTP_USER
password:
from_secret: SMTP_PASS
from: drone@iamthefij.com
when:
status: [changed, failure]
--- ---
kind: pipeline kind: pipeline
@@ -29,108 +45,15 @@ trigger:
- refs/tags/v* - refs/tags/v*
steps: steps:
- name: build all binaries
image: golang:1.15
environment:
VERSION: ${DRONE_TAG:-${DRONE_COMMIT}}
commands:
- make all
- name: compress binaries for release # Might consider moving this step into the previous pipeline
image: ubuntu - name: push image
commands:
- find ./dist -type f -executable -execdir tar -czvf {}.tar.gz {} \;
when:
event: tag
- name: upload gitea release
image: plugins/gitea-release
settings:
title: ${DRONE_TAG}
files: dist/*.tar.gz
checksum:
- md5
- sha1
- sha256
- sha512
base_url:
from_secret: gitea_base_url
api_key:
from_secret: gitea_token
when:
event: tag
- name: push image - arm
image: plugins/docker image: plugins/docker
settings: settings:
repo: iamthefij/minitor-go repo: iamthefij/minitor-go
dockerfile: Dockerfile.multi-stage
auto_tag: true auto_tag: true
auto_tag_suffix: linux-arm
username: username:
from_secret: docker_username from_secret: docker_username
password: password:
from_secret: docker_password from_secret: docker_password
build_args:
- ARCH=arm
- REPO=arm32v7
- name: push image - arm64
image: plugins/docker
settings:
repo: iamthefij/minitor-go
auto_tag: true
auto_tag_suffix: linux-arm64
username:
from_secret: docker_username
password:
from_secret: docker_password
build_args:
- ARCH=arm64
- REPO=arm64v8
- name: push image - amd64
image: plugins/docker
settings:
repo: iamthefij/minitor-go
auto_tag: true
auto_tag_suffix: linux-amd64
username:
from_secret: docker_username
password:
from_secret: docker_password
- name: publish manifest
image: plugins/manifest
settings:
spec: manifest.tmpl
auto_tag: true
ignore_missing: true
username:
from_secret: docker_username
password:
from_secret: docker_password
---
kind: pipeline
name: notify
depends_on:
- test
- publish
trigger:
status:
- failure
steps:
- name: notify
image: drillster/drone-email
settings:
host:
from_secret: SMTP_HOST # pragma: whitelist secret
username:
from_secret: SMTP_USER # pragma: whitelist secret
password:
from_secret: SMTP_PASS # pragma: whitelist secret
from: drone@iamthefij.com
Vendored
-2
View File
@@ -16,6 +16,4 @@
config.yml config.yml
# Output binary # Output binary
minitor
minitor-go minitor-go
dist/
-48
View File
@@ -1,48 +0,0 @@
---
linters:
enable:
- asciicheck
- bodyclose
- dogsled
- dupl
- exhaustive
- gochecknoinits
- gocognit
- gocritic
- gocyclo
- goerr113
- gofumpt
- goimports
- gomnd
- goprintffuncname
# - gosec
# - ifshort
- interfacer
- maligned
- misspell
- nakedret
- nestif
- nlreturn
- noctx
- unparam
- wsl
# - errorlint
disable:
- gochecknoglobals
linters-settings:
gosec:
excludes:
- G204
# gomnd:
# settings:
# mnd:
# ignored-functions: math.*
issues:
exclude-rules:
- path: _test\.go
linters:
- errcheck
- gosec
- maligned
+4 -7
View File
@@ -1,7 +1,7 @@
--- ---
repos: repos:
- repo: https://github.com/pre-commit/pre-commit-hooks - repo: https://github.com/pre-commit/pre-commit-hooks
rev: v3.4.0 rev: v2.4.0
hooks: hooks:
- id: check-added-large-files - id: check-added-large-files
- id: check-yaml - id: check-yaml
@@ -11,12 +11,9 @@ repos:
- id: end-of-file-fixer - id: end-of-file-fixer
- id: check-merge-conflict - id: check-merge-conflict
- repo: git://github.com/dnephin/pre-commit-golang - repo: git://github.com/dnephin/pre-commit-golang
rev: v0.4.0 rev: v0.3.5
hooks: hooks:
- id: go-fmt - id: go-fmt
- id: go-imports - id: go-imports
- id: golangci-lint # - id: gometalinter
- repo: https://github.com/hadolint/hadolint # - id: golangci-lint
rev: v2.4.0
hooks:
- id: hadolint
+5 -9
View File
@@ -1,13 +1,12 @@
ARG REPO=library ARG REPO=library
FROM multiarch/qemu-user-static:4.2.0-2 as qemu-user-static
FROM ${REPO}/alpine:3.10 FROM ${REPO}/alpine:3.10
# Copying all qemu files because amd64 doesn't exist and cannot condional copy
COPY --from=qemu-user-static /usr/bin/qemu-* /usr/bin/
RUN mkdir /app RUN mkdir /app
WORKDIR /app/ WORKDIR /app/
# Copy minitor in
ARG ARCH=amd64
COPY ./minitor-go ./minitor
# Add common checking tools # Add common checking tools
RUN apk --no-cache add bash=~5.0 curl=~7.66 jq=~1.6 RUN apk --no-cache add bash=~5.0 curl=~7.66 jq=~1.6
@@ -16,12 +15,9 @@ RUN addgroup -S minitor && adduser -S minitor -G minitor
# Copy scripts # Copy scripts
COPY ./scripts /app/scripts COPY ./scripts /app/scripts
RUN chown -R minitor:minitor /app
RUN chmod -R 755 /app/scripts RUN chmod -R 755 /app/scripts
# Copy minitor in
ARG ARCH=amd64
COPY ./dist/minitor-linux-${ARCH} ./minitor
# Drop to non-root user # Drop to non-root user
USER minitor USER minitor
+1
View File
@@ -31,6 +31,7 @@ RUN addgroup -S minitor && adduser -S minitor -G minitor
# Copy scripts # Copy scripts
COPY ./scripts /app/scripts COPY ./scripts /app/scripts
RUN chown -R minitor:minitor /app
RUN chmod -R 755 /app/scripts RUN chmod -R 755 /app/scripts
# Drop to non-root user # Drop to non-root user
+18 -60
View File
@@ -1,43 +1,31 @@
DOCKER_TAG ?= minitor-go-${USER}
VERSION ?= $(shell git describe --tags --dirty)
GOFILES = *.go
# Multi-arch targets are generated from this
TARGET_ALIAS = minitor-linux-amd64 minitor-linux-arm minitor-linux-arm64 minitor-darwin-amd64
TARGETS = $(addprefix dist/,$(TARGET_ALIAS))
#
# Default make target will run tests
.DEFAULT_GOAL = test
# Build all static Minitor binaries
.PHONY: all .PHONY: all
all: $(TARGETS) DOCKER_TAG ?= minitor-go-${USER}
# Build all static Linux Minitor binaries. Used in Docker images .PHONY: default
.PHONY: all-linux default: test
all-linux: $(filter dist/minitor-linux-%,$(TARGETS))
# Build minitor for the current machine
minitor: $(GOFILES)
@echo Version: $(VERSION)
go build -ldflags '-X "main.version=${VERSION}"' -o minitor
.PHONY: build .PHONY: build
build: minitor build:
go build
minitor-go:
go build
# Run minitor for the current machine
.PHONY: run .PHONY: run
run: minitor run: minitor-go build
./minitor -debug ./minitor-go -debug
.PHONY: run-metrics .PHONY: run-metrics
run-metrics: minitor run-metrics: minitor-go build
./minitor -debug -metrics ./minitor-go -debug -metrics
# Run all tests
.PHONY: test .PHONY: test
test: test:
go test -coverprofile=coverage.out go test -coverprofile=coverage.out
@echo
go tool cover -func=coverage.out go tool cover -func=coverage.out
@echo
@# Check min coverage percentage
@go tool cover -func=coverage.out | awk -v target=80.0% \ @go tool cover -func=coverage.out | awk -v target=80.0% \
'/^total:/ { print "Total coverage: " $$3 " Minimum coverage: " target; if ($$3+0.0 >= target+0.0) print "ok"; else { print "fail"; exit 1; } }' '/^total:/ { print "Total coverage: " $$3 " Minimum coverage: " target; if ($$3+0.0 >= target+0.0) print "ok"; else { print "fail"; exit 1; } }'
@@ -46,50 +34,20 @@ test:
install-hooks: install-hooks:
pre-commit install --install-hooks pre-commit install --install-hooks
# Runs pre-commit checks on files # Checks files for encryption
.PHONY: check .PHONY: check
check: check:
pre-commit run --all-files pre-commit run --all-files
.PHONY: clean .PHONY: clean
clean: clean:
rm -f ./minitor rm -f ./minitor-go
rm -f ./coverage.out rm -f ./coverage.out
rm -fr ./dist
.PHONY: docker-build .PHONY: docker-build
docker-build: docker-build:
docker build -f ./Dockerfile.multi-stage -t $(DOCKER_TAG)-linux-amd64 . docker build -f ./Dockerfile.multi-stage -t $(DOCKER_TAG) .
.PHONY: docker-run .PHONY: docker-run
docker-run: docker-build docker-run: docker-build
docker run --rm -v $(shell pwd)/config.yml:/root/config.yml $(DOCKER_TAG) docker run --rm -v $(shell pwd)/config.yml:/root/config.yml $(DOCKER_TAG)
## Multi-arch targets
$(TARGETS): $(GOFILES)
mkdir -p ./dist
GOOS=$(word 2, $(subst -, ,$(@))) GOARCH=$(word 3, $(subst -, ,$(@))) CGO_ENABLED=0 \
go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
-o $@
.PHONY: $(TARGET_ALIAS)
$(TARGET_ALIAS):
$(MAKE) $(addprefix dist/,$@)
# Arch specific docker build targets
.PHONY: docker-build-arm
docker-build-arm: dist/minitor-linux-arm
docker build --build-arg REPO=arm32v7 --build-arg ARCH=arm . -t ${DOCKER_TAG}-linux-arm
.PHONY: docker-build-arm64
docker-build-arm64: dist/minitor-linux-arm64
docker build --build-arg REPO=arm64v8 --build-arg ARCH=arm64 . -t ${DOCKER_TAG}-linux-arm64
# Cross run on host architechture
.PHONY: docker-run-arm
docker-run-arm: docker-build-arm
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock --name $(DOCKER_TAG)-run ${DOCKER_TAG}-linux-arm
.PHONY: docker-run-arm64
docker-run-arm64: docker-build-arm64
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock --name $(DOCKER_TAG)-run ${DOCKER_TAG}-linux-arm64
+66 -134
View File
@@ -1,167 +1,99 @@
# [minitor-go](https://git.iamthefij.com/iamthefij/minitor-go) # minitor-go
A minimal monitoring system A reimplementation of [Minitor](https://git.iamthefij/iamthefij/minitor) in Go
## What does it do?
Minitor accepts a YAML configuration file with a set of commands to run and a set of alerts to execute when those commands fail. It is designed to be as simple as possible and relies on other command line tools to do checks and issue alerts.
## But why?
I'm running a few small services and found Sensu, Consul, Nagios, etc. to all be far too complicated for my usecase.
## So how do I use it?
### Running
Install and execute with:
```bash
go get github.com/iamthefij/minitor-go
minitor
```
If locally developing you can use:
```bash
make run
```
It will read the contents of `config.yml` and begin its loop. You could also run it directly and provide a new config file via the `-config` argument.
#### Docker
You can pull this repository directly from Docker:
```bash
docker pull iamthefij/minitor-go:latest
```
The Docker image uses a default `config.yml` that is copied from `sample-config.yml`. This won't really do anything for you, so when you run the Docker image, you should supply your own `config.yml` file:
```bash
docker run -v $PWD/config.yml:/app/config.yml iamthefij/minitor-go:latest
```
Images are provided for `amd64`, `arm`, and `arm64` architechtures.
## Configuring
In this repo, you can explore the `sample-config.yml` file for an example, but the general structure is as follows. It should be noted that environment variable interpolation happens on load of the YAML file.
The global configurations are:
|key|value|
|---|---|
|`check_interval`|Maximum frequency to run checks for each monitor as duration, eg. 1m2s.|
|`monitors`|List of all monitors. Detailed description below|
|`alerts`|List of all alerts. Detailed description below|
### Monitors
All monitors should be listed under `monitors`.
Each monitor allows the following configuration:
|key|value|
|---|---|
|`name`|Name of the monitor running. This will show up in messages and logs.|
|`command`|Specifies the command that should be executed, either in exec or shell form. This command's exit value will determine whether the check is successful|
|`alert_down`|A list of Alerts to be triggered when the monitor is in a "down" state|
|`alert_up`|A list of Alerts to be triggered when the monitor moves to an "up" state|
|`check_interval`|The interval at which this monitor should be checked. This must be greater than the global `check_interval` value|
|`alert_after`|Allows specifying the number of failed checks before an alert should be triggered|
|`alert_every`|Allows specifying how often an alert should be retriggered. There are a few magic numbers here. Defaults to `-1` for an exponential backoff. Setting to `0` disables re-alerting. Positive values will allow retriggering after the specified number of checks|
### Alerts
Alerts exist as objects keyed under `alerts`. Their key should be the name of the Alert. This is used in your monitor setup in `alert_down` and `alert_up`.
Eachy alert allows the following configuration:
|key|value|
|---|---|
|`command`|Specifies the command that should be executed, either in exec or shell form. This is the command that will be run when the alert is executed. This can be templated with environment variables or the variables shown in the table below|
Also, when alerts are executed, they will be passed through Go's format function with arguments for some attributes of the Monitor. The following monitor specific variables can be referenced using Go formatting syntax:
|token|value|
|---|---|
|`{{.AlertCount}}`|Number of times this monitor has alerted|
|`{{.FailureCount}}`|The total number of sequential failed checks for this monitor|
|`{{.LastCheckOutput}}`|The last returned value from the check command to either stderr or stdout|
|`{{.LastSuccess}}`|The ISO datetime of the last successful check|
|`{{.MonitorName}}`|The name of the monitor that failed and triggered the alert|
|`{{.IsUp}}`|Indicates if the monitor that is alerting is up or not. Can be used in a conditional message template|
### Metrics
Minitor supports exporting metrics for [Prometheus](https://prometheus.io/). Prometheus is an open source tool for reading and querying metrics from different sources. Combined with another tool, [Grafana](https://grafana.com/), it allows building of charts and dashboards. You could also opt to just use Minitor to log check results, and instead do your alerting with Grafana.
It is also possible to use the metrics endpoint for monitoring Minitor itself! This allows setting up multiple instances of Minitor on different servers and have them monitor each-other so that you can detect a minitor outage.
To run minitor with metrics, use the `-metrics` flag. The metrics will be served on port `8080` by default, though it can be overriden using `-metrics-port`. They will be accessible on the path `/metrics`. Eg. `localhost:8080/metrics`.
```bash
minitor -metrics
# or
minitor -metrics -metrics-port 3000
```
## Contributing
Whether you're looking to submit a patch or tell me I broke something, you can contribute through the Github mirror and I can merge PRs back to the source repository.
Primary Repo: https://git.iamthefij.com/iamthefij/minitor.git
Github Mirror: https://github.com/IamTheFij/minitor.git
## Original Minitor
This is a reimplementation of [Minitor](https://git.iamthefij.com/iamthefij/minitor) in Go
Minitor is already a minimal monitoring tool. Python 3 was a quick way to get something live, but Python itself comes with a large footprint. Thus Go feels like a better fit for the project, longer term. Minitor is already a minimal monitoring tool. Python 3 was a quick way to get something live, but Python itself comes with a large footprint. Thus Go feels like a better fit for the project, longer term.
Initial target is meant to be roughly compatible requiring only minor changes to configuration. Future iterations may diverge to take advantage of Go specific features. Initial target is meant to be roughly compatible requiring only minor changes to configuration. Future iterations may diverge to take advantage of Go specific features.
### Differences from Python version ## Differences from Python version
Templating for Alert messages has been updated. In the Python version, `str.format(...)` was used with certain keys passed in that could be used to format messages. In the Go version, we use a struct, `AlertNotice` defined in `alert.go` and the built in Go templating format. Eg. There are a few key differences between the Python version and the v0.x Go version.
First, configuration keys cannot have multiple types in Go, so a different key must be used when specifying a Shell command as a string rather than a list of args. Instead of `command`, you must use `command_shell`. Eg:
minitor-py:
```yaml
monitors:
- name: Exec command
command: ['echo', 'test']
- name: Shell command
command: echo 'test'
```
minitor-go:
```yaml
monitors:
- name: Exec command
command: ['echo', 'test']
- name: Shell command
command_shell: echo 'test'
```
Second, templating for Alert messages has been updated. In the Python version, `str.format(...)` was used with certain keys passed in that could be used to format messages. In the Go version, we use a struct, `AlertNotice` defined in `alert.go` and the built in Go templating format. Eg.
minitor-py: minitor-py:
```yaml ```yaml
alerts: alerts:
log: log_command:
command: 'echo {monitor_name}' command: ['echo', '{monitor_name}']
log_shell:
command_shell: 'echo {monitor_name}'
``` ```
minitor-go: minitor-go:
```yaml ```yaml
alerts: alerts:
log: log_command:
command: 'echo {{.MonitorName}}' command: ['echo', '{{.MonitorName}}']
log_shell:
command_shell: 'echo {{.MonitorName}}'
``` ```
Interval durations have changed from being an integer number of seconds to a duration string supported by Go, for example: Finally, newlines in a shell command don't terminate a particular command. Semicolons must be used and continuations should not.
minitor-py: minitor-py:
```yaml ```yaml
check_interval: 90 alerts:
log_shell:
command_shell: >
echo "line 1"
echo "line 2"
echo "continued" \
"line"
``` ```
minitor-go: minitor-go:
```yaml ```yaml
check_interval: 1m30s alerts:
log_shell:
command_shell: >
echo "line 1";
echo "line 2";
echo "continued"
"line"
``` ```
For the time being, legacy configs for the Python version of Minitor should be compatible if you apply the `-py-compat` flag when running Minitor. Eventually, this flag will go away when later breaking changes are introduced. ## To do
There are two sets of task lists. The first is to get rough parity on key features with the Python version. The second is to make some improvements to the framework.
## Future Pairity:
Future, potentially breaking changes - [x] Run monitor commands
- [x] Run monitor commands in a shell
- [x] Run alert commands
- [x] Run alert commands in a shell
- [x] Allow templating of alert commands
- [x] Implement Prometheus client to export metrics
- [x] Test coverage
- [ ] Integration testing (manual or otherwise)
Improvement (potentially breaking):
- [ ] Implement leveled logging (maybe glog or logrus)
- [ ] Consider switching from YAML to TOML
- [ ] Consider value of templating vs injecting values into Env variables - [ ] Consider value of templating vs injecting values into Env variables
- [ ] Consider dropping `alert_up` and `alert_down` in favor of using Go templates that offer more control of messaging
- [ ] Async checking - [ ] Async checking
- [ ] Use durations rather than seconds checked in event loop
- [ ] Revisit metrics and see if they all make sense - [ ] Revisit metrics and see if they all make sense
- [ ] Consider dropping `alert_up` and `alert_down` in favor of using Go templates that offer more control of messaging (Breaking)
+29 -86
View File
@@ -2,126 +2,89 @@ package main
import ( import (
"bytes" "bytes"
"errors"
"fmt" "fmt"
"log"
"os/exec" "os/exec"
"strings"
"text/template" "text/template"
"time" "time"
"git.iamthefij.com/iamthefij/slog"
)
var (
errNoTemplate = errors.New("no template")
// ErrAlertFailed indicates that an alert failed to send
ErrAlertFailed = errors.New("alert failed")
) )
// Alert is a config driven mechanism for sending a notice // Alert is a config driven mechanism for sending a notice
type Alert struct { type Alert struct {
Name string Name string
Command CommandOrShell Command []string
CommandShell string `yaml:"command_shell"`
commandTemplate []*template.Template commandTemplate []*template.Template
commandShellTemplate *template.Template commandShellTemplate *template.Template
} }
// AlertNotice captures the context for an alert to be sent // AlertNotice captures the context for an alert to be sent
type AlertNotice struct { type AlertNotice struct {
MonitorName string
AlertCount int16 AlertCount int16
FailureCount int16 FailureCount int16
IsUp bool
LastSuccess time.Time
MonitorName string
LastCheckOutput string LastCheckOutput string
LastSuccess time.Time
IsUp bool
} }
// IsValid returns a boolean indicating if the Alert has been correctly // IsValid returns a boolean indicating if the Alert has been correctly
// configured // configured
func (alert Alert) IsValid() bool { func (alert Alert) IsValid() bool {
return !alert.Command.Empty() atLeastOneCommand := (alert.CommandShell != "" || alert.Command != nil)
atMostOneCommand := (alert.CommandShell == "" || alert.Command == nil)
return atLeastOneCommand && atMostOneCommand
} }
// BuildTemplates compiles command templates for the Alert // BuildTemplates compiles command templates for the Alert
func (alert *Alert) BuildTemplates() error { func (alert *Alert) BuildTemplates() error {
// TODO: Remove legacy template support later after 1.0 if LogDebug {
legacy := strings.NewReplacer( log.Printf("DEBUG: Building template for alert %s", alert.Name)
"{alert_count}", "{{.AlertCount}}",
"{alert_message}", "{{.MonitorName}} check has failed {{.FailureCount}} times",
"{failure_count}", "{{.FailureCount}}",
"{last_output}", "{{.LastCheckOutput}}",
"{last_success}", "{{.LastSuccess}}",
"{monitor_name}", "{{.MonitorName}}",
)
slog.Debugf("Building template for alert %s", alert.Name)
switch {
case alert.commandTemplate == nil && alert.Command.Command != nil:
alert.commandTemplate = []*template.Template{}
for i, cmdPart := range alert.Command.Command {
if PyCompat {
cmdPart = legacy.Replace(cmdPart)
} }
if alert.commandTemplate == nil && alert.Command != nil {
alert.commandTemplate = []*template.Template{}
for i, cmdPart := range alert.Command {
alert.commandTemplate = append(alert.commandTemplate, template.Must( alert.commandTemplate = append(alert.commandTemplate, template.Must(
template.New(alert.Name+fmt.Sprint(i)).Parse(cmdPart), template.New(alert.Name+string(i)).Parse(cmdPart),
)) ))
} }
case alert.commandShellTemplate == nil && alert.Command.ShellCommand != "": } else if alert.commandShellTemplate == nil && alert.CommandShell != "" {
shellCmd := alert.Command.ShellCommand
if PyCompat {
shellCmd = legacy.Replace(shellCmd)
}
alert.commandShellTemplate = template.Must( alert.commandShellTemplate = template.Must(
template.New(alert.Name).Parse(shellCmd), template.New(alert.Name).Parse(alert.CommandShell),
) )
default: } else {
return fmt.Errorf("No template provided for alert %s: %w", alert.Name, errNoTemplate) return fmt.Errorf("No template provided for alert %s", alert.Name)
} }
return nil return nil
} }
// Send will send an alert notice by executing the command template // Send will send an alert notice by executing the command template
func (alert Alert) Send(notice AlertNotice) (outputStr string, err error) { func (alert Alert) Send(notice AlertNotice) (output_str string, err error) {
slog.Infof("Sending alert %s for %s", alert.Name, notice.MonitorName) log.Printf("INFO: Sending alert %s for %s", alert.Name, notice.MonitorName)
var cmd *exec.Cmd var cmd *exec.Cmd
if alert.commandTemplate != nil {
switch {
case alert.commandTemplate != nil:
command := []string{} command := []string{}
for _, cmdTmp := range alert.commandTemplate { for _, cmdTmp := range alert.commandTemplate {
var commandBuffer bytes.Buffer var commandBuffer bytes.Buffer
err = cmdTmp.Execute(&commandBuffer, notice) err = cmdTmp.Execute(&commandBuffer, notice)
if err != nil { if err != nil {
return return
} }
command = append(command, commandBuffer.String()) command = append(command, commandBuffer.String())
} }
cmd = exec.Command(command[0], command[1:]...) cmd = exec.Command(command[0], command[1:]...)
case alert.commandShellTemplate != nil: } else if alert.commandShellTemplate != nil {
var commandBuffer bytes.Buffer var commandBuffer bytes.Buffer
err = alert.commandShellTemplate.Execute(&commandBuffer, notice) err = alert.commandShellTemplate.Execute(&commandBuffer, notice)
if err != nil { if err != nil {
return return
} }
shellCommand := commandBuffer.String() shellCommand := commandBuffer.String()
cmd = ShellCommand(shellCommand) cmd = ShellCommand(shellCommand)
default: } else {
err = fmt.Errorf("No templates compiled for alert %s: %w", alert.Name, errNoTemplate) err = fmt.Errorf("No templates compiled for alert %v", alert.Name)
return return
} }
@@ -132,30 +95,10 @@ func (alert Alert) Send(notice AlertNotice) (outputStr string, err error) {
var output []byte var output []byte
output, err = cmd.CombinedOutput() output, err = cmd.CombinedOutput()
outputStr = string(output) output_str = string(output)
slog.Debugf("Alert output for: %s\n---\n%s\n---", alert.Name, outputStr) if LogDebug {
log.Printf("DEBUG: Alert output for: %s\n---\n%s\n---", alert.Name, output_str)
if err != nil {
err = fmt.Errorf(
"Alert '%s' failed to send. Returned %v: %w",
alert.Name,
err,
ErrAlertFailed,
)
} }
return outputStr, err return output_str, err
}
// NewLogAlert creates an alert that does basic logging using echo
func NewLogAlert() *Alert {
return &Alert{
Name: "log",
Command: CommandOrShell{
Command: []string{
"echo",
"{{.MonitorName}} {{if .IsUp}}has recovered{{else}}check has failed {{.FailureCount}} times{{end}}",
},
},
}
} }
+14 -59
View File
@@ -11,20 +11,23 @@ func TestAlertIsValid(t *testing.T) {
expected bool expected bool
name string name string
}{ }{
{Alert{Command: CommandOrShell{Command: []string{"echo", "test"}}}, true, "Command only"}, {Alert{Command: []string{"echo", "test"}}, true, "Command only"},
{Alert{Command: CommandOrShell{ShellCommand: "echo test"}}, true, "CommandShell only"}, {Alert{CommandShell: "echo test"}, true, "CommandShell only"},
{Alert{}, false, "No commands"}, {Alert{}, false, "No commands"},
{
Alert{Command: []string{"echo", "test"}, CommandShell: "echo test"},
false,
"Both commands",
},
} }
for _, c := range cases { for _, c := range cases {
log.Printf("Testing case %s", c.name) log.Printf("Testing case %s", c.name)
actual := c.alert.IsValid() actual := c.alert.IsValid()
if actual != c.expected { if actual != c.expected {
t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expected, actual) t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expected, actual)
log.Printf("Case failed: %s", c.name) log.Printf("Case failed: %s", c.name)
} }
log.Println("-----") log.Println("-----")
} }
} }
@@ -36,94 +39,50 @@ func TestAlertSend(t *testing.T) {
expectedOutput string expectedOutput string
expectErr bool expectErr bool
name string name string
pyCompat bool
}{ }{
{ {
Alert{Command: CommandOrShell{Command: []string{"echo", "{{.MonitorName}}"}}}, Alert{Command: []string{"echo", "{{.MonitorName}}"}},
AlertNotice{MonitorName: "test"}, AlertNotice{MonitorName: "test"},
"test\n", "test\n",
false, false,
"Command with template", "Command with template",
false,
}, },
{ {
Alert{Command: CommandOrShell{ShellCommand: "echo {{.MonitorName}}"}}, Alert{CommandShell: "echo {{.MonitorName}}"},
AlertNotice{MonitorName: "test"}, AlertNotice{MonitorName: "test"},
"test\n", "test\n",
false, false,
"Command shell with template", "Command shell with template",
false,
}, },
{ {
Alert{Command: CommandOrShell{Command: []string{"echo", "{{.Bad}}"}}}, Alert{Command: []string{"echo", "{{.Bad}}"}},
AlertNotice{MonitorName: "test"}, AlertNotice{MonitorName: "test"},
"", "",
true, true,
"Command with bad template", "Command with bad template",
false,
}, },
{ {
Alert{Command: CommandOrShell{ShellCommand: "echo {{.Bad}}"}}, Alert{CommandShell: "echo {{.Bad}}"},
AlertNotice{MonitorName: "test"}, AlertNotice{MonitorName: "test"},
"", "",
true, true,
"Command shell with bad template", "Command shell with bad template",
false,
},
{
Alert{Command: CommandOrShell{ShellCommand: "echo {alert_message}"}},
AlertNotice{MonitorName: "test", FailureCount: 1},
"test check has failed 1 times\n",
false,
"Command shell with legacy template",
true,
},
// Test default log alert down
{
*NewLogAlert(),
AlertNotice{MonitorName: "Test", FailureCount: 1, IsUp: false},
"Test check has failed 1 times\n",
false,
"Default log alert down",
false,
},
// Test default log alert up
{
*NewLogAlert(),
AlertNotice{MonitorName: "Test", IsUp: true},
"Test has recovered\n",
false,
"Default log alert up",
false,
}, },
} }
for _, c := range cases { for _, c := range cases {
log.Printf("Testing case %s", c.name) log.Printf("Testing case %s", c.name)
// Set PyCompat to value of compat flag c.alert.BuildTemplates()
PyCompat = c.pyCompat
err := c.alert.BuildTemplates()
if err != nil {
t.Errorf("Send(%v output), error building templates: %v", c.name, err)
}
output, err := c.alert.Send(c.notice) output, err := c.alert.Send(c.notice)
hasErr := (err != nil) hasErr := (err != nil)
if output != c.expectedOutput { if output != c.expectedOutput {
t.Errorf("Send(%v output), expected=%v actual=%v", c.name, c.expectedOutput, output) t.Errorf("Send(%v output), expected=%v actual=%v", c.name, c.expectedOutput, output)
log.Printf("Case failed: %s", c.name) log.Printf("Case failed: %s", c.name)
} }
if hasErr != c.expectErr { if hasErr != c.expectErr {
t.Errorf("Send(%v err), expected=%v actual=%v", c.name, "Err", err) t.Errorf("Send(%v err), expected=%v actual=%v", c.name, "Err", err)
log.Printf("Case failed: %s", c.name) log.Printf("Case failed: %s", c.name)
} }
// Set PyCompat back to default value
PyCompat = false
log.Println("-----") log.Println("-----")
} }
} }
@@ -131,12 +90,10 @@ func TestAlertSend(t *testing.T) {
func TestAlertSendNoTemplates(t *testing.T) { func TestAlertSendNoTemplates(t *testing.T) {
alert := Alert{} alert := Alert{}
notice := AlertNotice{} notice := AlertNotice{}
output, err := alert.Send(notice) output, err := alert.Send(notice)
if err == nil { if err == nil {
t.Errorf("Send(no template), expected=%v actual=%v", "Err", output) t.Errorf("Send(no template), expected=%v actual=%v", "Err", output)
} }
log.Println("-----") log.Println("-----")
} }
@@ -146,8 +103,8 @@ func TestAlertBuildTemplate(t *testing.T) {
expectErr bool expectErr bool
name string name string
}{ }{
{Alert{Command: CommandOrShell{Command: []string{"echo", "test"}}}, false, "Command only"}, {Alert{Command: []string{"echo", "test"}}, false, "Command only"},
{Alert{Command: CommandOrShell{ShellCommand: "echo test"}}, false, "CommandShell only"}, {Alert{CommandShell: "echo test"}, false, "CommandShell only"},
{Alert{}, true, "No commands"}, {Alert{}, true, "No commands"},
} }
@@ -155,12 +112,10 @@ func TestAlertBuildTemplate(t *testing.T) {
log.Printf("Testing case %s", c.name) log.Printf("Testing case %s", c.name)
err := c.alert.BuildTemplates() err := c.alert.BuildTemplates()
hasErr := (err != nil) hasErr := (err != nil)
if hasErr != c.expectErr { if hasErr != c.expectErr {
t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expectErr, err) t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expectErr, err)
log.Printf("Case failed: %s", c.name) log.Printf("Case failed: %s", c.name)
} }
log.Println("-----") log.Println("-----")
} }
} }
+30 -115
View File
@@ -3,134 +3,60 @@ package main
import ( import (
"errors" "errors"
"io/ioutil" "io/ioutil"
"time" "log"
"os"
"git.iamthefij.com/iamthefij/slog"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
) )
var errInvalidConfig = errors.New("Invalid configuration")
// Config type is contains all provided user configuration // Config type is contains all provided user configuration
type Config struct { type Config struct {
CheckInterval SecondsOrDuration `yaml:"check_interval"` CheckInterval int64 `yaml:"check_interval"`
Monitors []*Monitor Monitors []*Monitor
Alerts map[string]*Alert Alerts map[string]*Alert
} }
// CommandOrShell type wraps a string or list of strings
// for executing a command directly or in a shell
type CommandOrShell struct {
ShellCommand string
Command []string
}
// Empty checks if the Command has a value
func (cos CommandOrShell) Empty() bool {
return (cos.ShellCommand == "" && cos.Command == nil)
}
// UnmarshalYAML allows unmarshalling either a string or slice of strings
// and parsing them as either a command or a shell command.
func (cos *CommandOrShell) UnmarshalYAML(unmarshal func(interface{}) error) error {
var cmd []string
err := unmarshal(&cmd)
// Error indicates this is shell command
if err != nil {
var shellCmd string
err := unmarshal(&shellCmd)
if err != nil {
return err
}
cos.ShellCommand = shellCmd
} else {
cos.Command = cmd
}
return nil
}
// SecondsOrDuration wraps a duration value for parsing a duration or seconds from YAML
// NOTE: This should be removed in favor of only parsing durations once compatibility is broken
type SecondsOrDuration struct {
value time.Duration
}
// Value returns a duration value
func (sod SecondsOrDuration) Value() time.Duration {
return sod.value
}
// UnmarshalYAML allows unmarshalling a duration value or seconds if an int was provided
func (sod *SecondsOrDuration) UnmarshalYAML(unmarshal func(interface{}) error) error {
var seconds int64
err := unmarshal(&seconds)
if err == nil {
sod.value = time.Second * time.Duration(seconds)
return nil
}
// Error indicates that we don't have an int
err = unmarshal(&sod.value)
return err
}
// IsValid checks config validity and returns true if valid // IsValid checks config validity and returns true if valid
func (config Config) IsValid() (isValid bool) { func (config Config) IsValid() (isValid bool) {
isValid = true isValid = true
// Validate alerts
if config.Alerts == nil || len(config.Alerts) == 0 {
// This should never happen because there is a default alert named 'log' for now
slog.Errorf("Invalid alert configuration: Must provide at least one alert")
isValid = false
}
for _, alert := range config.Alerts {
if !alert.IsValid() {
slog.Errorf("Invalid alert configuration: %+v", alert.Name)
isValid = false
} else {
slog.Debugf("Loaded alert %s", alert.Name)
}
}
// Validate monitors // Validate monitors
if config.Monitors == nil || len(config.Monitors) == 0 { if config.Monitors == nil || len(config.Monitors) == 0 {
slog.Errorf("Invalid monitor configuration: Must provide at least one monitor") log.Printf("ERROR: Invalid monitor configuration: Must provide at least one monitor")
isValid = false isValid = false
} }
for _, monitor := range config.Monitors { for _, monitor := range config.Monitors {
if !monitor.IsValid() { if !monitor.IsValid() {
slog.Errorf("Invalid monitor configuration: %s", monitor.Name) log.Printf("ERROR: Invalid monitor configuration: %s", monitor.Name)
isValid = false isValid = false
} }
// Check that all Monitor alerts actually exist // Check that all Monitor alerts actually exist
for _, isUp := range []bool{true, false} { for _, isUp := range []bool{true, false} {
for _, alertName := range monitor.GetAlertNames(isUp) { for _, alertName := range monitor.GetAlertNames(isUp) {
if _, ok := config.Alerts[alertName]; !ok { if _, ok := config.Alerts[alertName]; !ok {
slog.Errorf( log.Printf(
"Invalid monitor configuration: %s. Unknown alert %s", "ERROR: Invalid monitor configuration: %s. Unknown alert %s",
monitor.Name, alertName, monitor.Name, alertName,
) )
isValid = false isValid = false
} }
} }
} }
} }
return isValid // Validate alerts
if config.Alerts == nil || len(config.Alerts) == 0 {
log.Printf("ERROR: Invalid alert configuration: Must provide at least one alert")
isValid = false
}
for _, alert := range config.Alerts {
if !alert.IsValid() {
log.Printf("ERROR: Invalid alert configuration: %s", alert.Name)
isValid = false
}
}
return
} }
// Init performs extra initialization on top of loading the config from file // Init performs extra initialization on top of loading the config from file
@@ -152,35 +78,24 @@ func LoadConfig(filePath string) (config Config, err error) {
return return
} }
err = yaml.Unmarshal(data, &config) // TODO: Decide if this is better expanded here, or only when executing
envExpanded := os.ExpandEnv(string(data))
err = yaml.Unmarshal([]byte(envExpanded), &config)
if err != nil { if err != nil {
return return
} }
slog.Debugf("Config values:\n%v\n", config) if LogDebug {
log.Printf("DEBUG: Config values:\n%v\n", config)
// Add log alert if not present
if PyCompat {
// Initialize alerts list if not present
if config.Alerts == nil {
config.Alerts = map[string]*Alert{}
}
if _, ok := config.Alerts["log"]; !ok {
config.Alerts["log"] = NewLogAlert()
}
}
// Finish initializing configuration
if err = config.Init(); err != nil {
return
} }
if !config.IsValid() { if !config.IsValid() {
err = errInvalidConfig err = errors.New("Invalid configuration")
return return
} }
return config, err // Finish initializing configuration
err = config.Init()
return
} }
+7 -119
View File
@@ -3,7 +3,6 @@ package main
import ( import (
"log" "log"
"testing" "testing"
"time"
) )
func TestLoadConfig(t *testing.T) { func TestLoadConfig(t *testing.T) {
@@ -11,133 +10,22 @@ func TestLoadConfig(t *testing.T) {
configPath string configPath string
expectErr bool expectErr bool
name string name string
pyCompat bool
}{ }{
{"./test/valid-config.yml", false, "Valid config file", false}, {"./test/valid-config.yml", false, "Valid config file"},
{"./test/valid-default-log-alert.yml", false, "Valid config file with default log alert PyCompat", true}, {"./test/does-not-exist", true, "Invalid config path"},
{"./test/valid-default-log-alert.yml", true, "Invalid config file no log alert", false}, {"./test/invalid-config-type.yml", true, "Invalid config type for key"},
{"./test/does-not-exist", true, "Invalid config path", false}, {"./test/invalid-config-missing-alerts.yml", true, "Invalid config missing alerts"},
{"./test/invalid-config-type.yml", true, "Invalid config type for key", false}, {"./test/invalid-config-unknown-alert.yml", true, "Invalid config unknown alert"},
{"./test/invalid-config-missing-alerts.yml", true, "Invalid config missing alerts", false},
{"./test/invalid-config-unknown-alert.yml", true, "Invalid config unknown alert", false},
} }
for _, c := range cases { for _, c := range cases {
log.Printf("Testing case %s", c.name) log.Printf("Testing case %s", c.name)
// Set PyCompat based on compatibility mode
PyCompat = c.pyCompat
_, err := LoadConfig(c.configPath) _, err := LoadConfig(c.configPath)
hasErr := (err != nil) hasErr := (err != nil)
if hasErr != c.expectErr { if hasErr != c.expectErr {
t.Errorf("LoadConfig(%v), expected_error=%v actual=%v", c.name, c.expectErr, err) t.Errorf("LoadConfig(%v), expected=%v actual=%v", c.name, "Err", err)
log.Printf("Case failed: %s", c.name) log.Printf("Case failed: %s", c.name)
} }
log.Println("-----")
// Set PyCompat to default value
PyCompat = false
}
}
func TestIntervalParsing(t *testing.T) {
log.Printf("Testing case TestIntervalParsing")
config, err := LoadConfig("./test/valid-config.yml")
if err != nil {
t.Errorf("Failed loading config: %v", err)
}
oneSecond := time.Second
tenSeconds := 10 * time.Second
oneMinute := time.Minute
// validate top level interval seconds represented as an int
if config.CheckInterval.Value() != oneSecond {
t.Errorf("Incorrectly parsed int seconds. expected=%v actual=%v", oneSecond, config.CheckInterval)
}
if config.Monitors[0].CheckInterval.Value() != tenSeconds {
t.Errorf("Incorrectly parsed seconds duration. expected=%v actual=%v", oneSecond, config.CheckInterval)
}
if config.Monitors[1].CheckInterval.Value() != oneMinute {
t.Errorf("Incorrectly parsed seconds duration. expected=%v actual=%v", oneSecond, config.CheckInterval)
}
log.Println("-----")
}
// TestMultiLineConfig is a more complicated test stepping through the parsing
// and execution of mutli-line strings presented in YAML
func TestMultiLineConfig(t *testing.T) {
log.Println("Testing multi-line string config")
config, err := LoadConfig("./test/valid-verify-multi-line.yml")
if err != nil {
t.Fatalf("TestMultiLineConfig(load), expected=no_error actual=%v", err)
}
log.Println("-----")
log.Println("TestMultiLineConfig(parse > string)")
expected := "echo 'Some string with stuff'; echo \"<angle brackets>\"; exit 1\n"
actual := config.Monitors[0].Command.ShellCommand
if expected != actual {
t.Errorf("TestMultiLineConfig(>) failed")
t.Logf("string expected=`%v`", expected)
t.Logf("string actual =`%v`", actual)
t.Logf("bytes expected=%v", []byte(expected))
t.Logf("bytes actual =%v", []byte(actual))
}
log.Println("-----")
log.Println("TestMultiLineConfig(execute > string)")
_, notice := config.Monitors[0].Check()
if notice == nil {
t.Fatalf("Did not receive an alert notice")
}
expected = "Some string with stuff\n<angle brackets>\n"
actual = notice.LastCheckOutput
if expected != actual {
t.Errorf("TestMultiLineConfig(execute > string) check failed")
t.Logf("string expected=`%v`", expected)
t.Logf("string actual =`%v`", actual)
t.Logf("bytes expected=%v", []byte(expected))
t.Logf("bytes actual =%v", []byte(actual))
}
log.Println("-----")
log.Println("TestMultiLineConfig(parse | string)")
expected = "echo 'Some string with stuff'\necho '<angle brackets>'\n"
actual = config.Alerts["log_shell"].Command.ShellCommand
if expected != actual {
t.Errorf("TestMultiLineConfig(|) failed")
t.Logf("string expected=`%v`", expected)
t.Logf("string actual =`%v`", actual)
t.Logf("bytes expected=%v", []byte(expected))
t.Logf("bytes actual =%v", []byte(actual))
}
log.Println("-----")
log.Println("TestMultiLineConfig(execute | string)")
actual, err = config.Alerts["log_shell"].Send(AlertNotice{})
if err != nil {
t.Errorf("Execution of alert failed")
}
expected = "Some string with stuff\n<angle brackets>\n"
if expected != actual {
t.Errorf("TestMultiLineConfig(execute | string) check failed")
t.Logf("string expected=`%v`", expected)
t.Logf("string actual =`%v`", actual)
t.Logf("bytes expected=%v", []byte(expected))
t.Logf("bytes actual =%v", []byte(actual))
} }
} }
+1 -2
View File
@@ -1,9 +1,8 @@
module git.iamthefij.com/iamthefij/minitor-go module git.iamthefij.com/iamthefij/minitor-go
go 1.15 go 1.12
require ( require (
git.iamthefij.com/iamthefij/slog v1.3.0
github.com/prometheus/client_golang v1.2.1 github.com/prometheus/client_golang v1.2.1
gopkg.in/yaml.v2 v2.2.4 gopkg.in/yaml.v2 v2.2.4
) )
-2
View File
@@ -1,5 +1,3 @@
git.iamthefij.com/iamthefij/slog v1.3.0 h1:4Hu5PQvDrW5e3FrTS3q2iIXW0iPvhNY/9qJsqDR3K3I=
git.iamthefij.com/iamthefij/slog v1.3.0/go.mod h1:1RUj4hcCompZkAxXCRfUX786tb3cM/Zpkn97dGfUfbg=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+55 -63
View File
@@ -1,15 +1,16 @@
package main package main
import ( import (
"errors"
"flag" "flag"
"fmt" "fmt"
"log"
"time" "time"
"git.iamthefij.com/iamthefij/slog"
) )
var ( var (
// LogDebug will control whether debug messsages should be logged
LogDebug = false
// ExportMetrics will track whether or not we want to export metrics to prometheus // ExportMetrics will track whether or not we want to export metrics to prometheus
ExportMetrics = false ExportMetrics = false
// MetricsPort is the port to expose metrics on // MetricsPort is the port to expose metrics on
@@ -17,54 +18,10 @@ var (
// Metrics contains all active metrics // Metrics contains all active metrics
Metrics = NewMetrics() Metrics = NewMetrics()
// PyCompat enables support for legacy Python templates
PyCompat = false
// version of minitor being run // version of minitor being run
version = "dev" version = "dev"
errUnknownAlert = errors.New("unknown alert")
) )
func sendAlerts(config *Config, monitor *Monitor, alertNotice *AlertNotice) error {
slog.Debugf("Received an alert notice from %s", alertNotice.MonitorName)
alertNames := monitor.GetAlertNames(alertNotice.IsUp)
if alertNames == nil {
// This should only happen for a recovery alert. AlertDown is validated not empty
slog.Warningf(
"Received alert, but no alert mechanisms exist. MonitorName=%s IsUp=%t",
alertNotice.MonitorName, alertNotice.IsUp,
)
}
for _, alertName := range alertNames {
if alert, ok := config.Alerts[alertName]; ok {
output, err := alert.Send(*alertNotice)
if err != nil {
slog.Errorf(
"Alert '%s' failed. result=%v: output=%s",
alert.Name,
err,
output,
)
return err
}
// Count alert metrics
Metrics.CountAlert(monitor.Name, alert.Name)
} else {
// This case should never actually happen since we validate against it
slog.Errorf("Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
return fmt.Errorf("unknown alert for monitor %s: %s: %w", alertNotice.MonitorName, alertName, errUnknownAlert)
}
}
return nil
}
func checkMonitors(config *Config) error { func checkMonitors(config *Config) error {
for _, monitor := range config.Monitors { for _, monitor := range config.Monitors {
if monitor.ShouldCheck() { if monitor.ShouldCheck() {
@@ -73,11 +30,48 @@ func checkMonitors(config *Config) error {
hasAlert := alertNotice != nil hasAlert := alertNotice != nil
// Track status metrics // Track status metrics
Metrics.SetMonitorStatus(monitor.Name, monitor.IsUp()) Metrics.SetMonitorStatus(monitor.Name, success)
Metrics.CountCheck(monitor.Name, success, monitor.LastCheckMilliseconds(), hasAlert) Metrics.CountCheck(monitor.Name, success, hasAlert)
// Should probably consider refactoring everything below here
if alertNotice != nil { if alertNotice != nil {
return sendAlerts(config, monitor, alertNotice) if LogDebug {
log.Printf("DEBUG: Recieved an alert notice from %s", alertNotice.MonitorName)
}
alertNames := monitor.GetAlertNames(alertNotice.IsUp)
if alertNames == nil {
// This should only happen for a recovery alert. AlertDown is validated not empty
log.Printf(
"WARNING: Recieved alert, but no alert mechanisms exist. MonitorName=%s IsUp=%t",
alertNotice.MonitorName, alertNotice.IsUp,
)
}
for _, alertName := range alertNames {
if alert, ok := config.Alerts[alertName]; ok {
output, err := alert.Send(*alertNotice)
if err != nil {
log.Printf(
"ERROR: Alert '%s' failed. result=%v: output=%s",
alert.Name,
err,
output,
)
return fmt.Errorf(
"Unsuccessfully triggered alert '%s'. "+
"Crashing to avoid false negatives: %v",
alert.Name,
err,
)
}
// Count alert metrics
Metrics.CountAlert(monitor.Name, alert.Name)
} else {
// This case should never actually happen since we validate against it
log.Printf("ERROR: Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
return fmt.Errorf("Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
}
}
} }
} }
} }
@@ -86,30 +80,27 @@ func checkMonitors(config *Config) error {
} }
func main() { func main() {
showVersion := flag.Bool("version", false, "Display the version of minitor and exit") // Get debug flag
configPath := flag.String("config", "config.yml", "Alternate configuration path (default: config.yml)") flag.BoolVar(&LogDebug, "debug", false, "Enables debug logs (default: false)")
flag.BoolVar(&slog.DebugLevel, "debug", false, "Enables debug logs (default: false)")
flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)") flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)")
flag.BoolVar(&PyCompat, "py-compat", false, "Enables support for legacy Python Minitor config. Will eventually be removed. (default: false)") var showVersion = flag.Bool("version", false, "Display the version of minitor and exit")
flag.IntVar(&MetricsPort, "metrics-port", 8080, "The port that Prometheus metrics should be exported on, if enabled. (default: 8080)")
flag.Parse() flag.Parse()
// Print version if flag is provided // Print version if flag is provided
if *showVersion { if *showVersion {
fmt.Println("Minitor version:", version) log.Println("Minitor version:", version)
return return
} }
// Load configuration // Load configuration
config, err := LoadConfig(*configPath) config, err := LoadConfig("config.yml")
slog.OnErrFatalf(err, "Error loading config: %v", err) if err != nil {
log.Fatalf("Error loading config: %v", err)
}
// Serve metrics exporter, if specified // Serve metrics exporter, if specified
if ExportMetrics { if ExportMetrics {
slog.Infof("Exporting metrics to Prometheus on port %d", MetricsPort) log.Println("INFO: Exporting metrics to Prometheus")
go ServeMetrics() go ServeMetrics()
} }
@@ -120,6 +111,7 @@ func main() {
panic(err) panic(err)
} }
time.Sleep(config.CheckInterval.Value()) sleepTime := time.Duration(config.CheckInterval) * time.Second
time.Sleep(sleepTime)
} }
} }
+22 -44
View File
@@ -18,7 +18,7 @@ func TestCheckMonitors(t *testing.T) {
Monitors: []*Monitor{ Monitors: []*Monitor{
&Monitor{ &Monitor{
Name: "Success", Name: "Success",
Command: CommandOrShell{Command: []string{"true"}}, Command: []string{"true"},
}, },
}, },
}, },
@@ -30,20 +30,32 @@ func TestCheckMonitors(t *testing.T) {
Monitors: []*Monitor{ Monitors: []*Monitor{
&Monitor{ &Monitor{
Name: "Failure", Name: "Failure",
Command: CommandOrShell{Command: []string{"false"}}, Command: []string{"false"},
AlertAfter: 1,
},
&Monitor{
Name: "Failure",
Command: []string{"false"},
AlertDown: []string{"unknown"},
AlertAfter: 1, AlertAfter: 1,
}, },
}, },
}, },
expectErr: false, expectErr: false,
name: "Monitor failure, no alerts", name: "Monitor failure, no and unknown alerts",
}, },
{ {
config: Config{ config: Config{
Monitors: []*Monitor{ Monitors: []*Monitor{
&Monitor{ &Monitor{
Name: "Success", Name: "Success",
Command: CommandOrShell{Command: []string{"ls"}}, Command: []string{"ls"},
alertCount: 1,
},
&Monitor{
Name: "Success",
Command: []string{"true"},
AlertUp: []string{"unknown"},
alertCount: 1, alertCount: 1,
}, },
}, },
@@ -56,42 +68,14 @@ func TestCheckMonitors(t *testing.T) {
Monitors: []*Monitor{ Monitors: []*Monitor{
&Monitor{ &Monitor{
Name: "Failure", Name: "Failure",
Command: CommandOrShell{Command: []string{"false"}}, Command: []string{"false"},
AlertDown: []string{"unknown"},
AlertAfter: 1,
},
},
},
expectErr: true,
name: "Monitor failure, unknown alerts",
},
{
config: Config{
Monitors: []*Monitor{
&Monitor{
Name: "Success",
Command: CommandOrShell{Command: []string{"true"}},
AlertUp: []string{"unknown"},
alertCount: 1,
},
},
},
expectErr: true,
name: "Monitor recovery, unknown alerts",
},
{
config: Config{
Monitors: []*Monitor{
&Monitor{
Name: "Failure",
Command: CommandOrShell{Command: []string{"false"}},
AlertDown: []string{"good"}, AlertDown: []string{"good"},
AlertAfter: 1, AlertAfter: 1,
}, },
}, },
Alerts: map[string]*Alert{ Alerts: map[string]*Alert{
"good": &Alert{ "good": &Alert{
Command: CommandOrShell{Command: []string{"true"}}, Command: []string{"true"},
}, },
}, },
}, },
@@ -103,7 +87,7 @@ func TestCheckMonitors(t *testing.T) {
Monitors: []*Monitor{ Monitors: []*Monitor{
&Monitor{ &Monitor{
Name: "Failure", Name: "Failure",
Command: CommandOrShell{Command: []string{"false"}}, Command: []string{"false"},
AlertDown: []string{"bad"}, AlertDown: []string{"bad"},
AlertAfter: 1, AlertAfter: 1,
}, },
@@ -111,7 +95,7 @@ func TestCheckMonitors(t *testing.T) {
Alerts: map[string]*Alert{ Alerts: map[string]*Alert{
"bad": &Alert{ "bad": &Alert{
Name: "bad", Name: "bad",
Command: CommandOrShell{Command: []string{"false"}}, Command: []string{"false"},
}, },
}, },
}, },
@@ -121,16 +105,10 @@ func TestCheckMonitors(t *testing.T) {
} }
for _, c := range cases { for _, c := range cases {
err := c.config.Init() c.config.Init()
if err != nil { err := checkMonitors(&c.config)
t.Errorf("checkMonitors(%s): unexpected error reading config: %v", c.name, err)
}
err = checkMonitors(&c.config)
if err == nil && c.expectErr { if err == nil && c.expectErr {
t.Errorf("checkMonitors(%s): Expected panic, the code did not panic", c.name) t.Errorf("checkMonitors(%s): Expected panic, the code did not panic", c.name)
} else if err != nil && !c.expectErr {
t.Errorf("checkMonitors(%s): Did not expect an error, but we got one anyway: %v", c.name, err)
} }
} }
} }
-25
View File
@@ -1,25 +0,0 @@
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}
{{#if build.tags}}
tags:
{{#each build.tags}}
- {{this}}
{{/each}}
{{/if}}
manifests:
-
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64
platform:
architecture: amd64
os: linux
-
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64
platform:
architecture: arm64
os: linux
variant: v8
-
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm
platform:
architecture: arm
os: linux
variant: v7
+1 -17
View File
@@ -19,7 +19,6 @@ import (
type MinitorMetrics struct { type MinitorMetrics struct {
alertCount *prometheus.CounterVec alertCount *prometheus.CounterVec
checkCount *prometheus.CounterVec checkCount *prometheus.CounterVec
checkTime *prometheus.GaugeVec
monitorStatus *prometheus.GaugeVec monitorStatus *prometheus.GaugeVec
} }
@@ -41,13 +40,6 @@ func NewMetrics() *MinitorMetrics {
}, },
[]string{"monitor", "status", "is_alert"}, []string{"monitor", "status", "is_alert"},
), ),
checkTime: prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "minitor_check_milliseconds",
Help: "Time in miliseconds that a check ran for",
},
[]string{"monitor", "status"},
),
monitorStatus: prometheus.NewGaugeVec( monitorStatus: prometheus.NewGaugeVec(
prometheus.GaugeOpts{ prometheus.GaugeOpts{
Name: "minitor_monitor_up_count", Name: "minitor_monitor_up_count",
@@ -60,7 +52,6 @@ func NewMetrics() *MinitorMetrics {
// Register newly created metrics // Register newly created metrics
prometheus.MustRegister(metrics.alertCount) prometheus.MustRegister(metrics.alertCount)
prometheus.MustRegister(metrics.checkCount) prometheus.MustRegister(metrics.checkCount)
prometheus.MustRegister(metrics.checkTime)
prometheus.MustRegister(metrics.monitorStatus) prometheus.MustRegister(metrics.monitorStatus)
return metrics return metrics
@@ -72,12 +63,11 @@ func (metrics *MinitorMetrics) SetMonitorStatus(monitor string, isUp bool) {
if isUp { if isUp {
val = 1.0 val = 1.0
} }
metrics.monitorStatus.With(prometheus.Labels{"monitor": monitor}).Set(val) metrics.monitorStatus.With(prometheus.Labels{"monitor": monitor}).Set(val)
} }
// CountCheck counts the result of a particular Monitor check // CountCheck counts the result of a particular Monitor check
func (metrics *MinitorMetrics) CountCheck(monitor string, isSuccess bool, ms int64, isAlert bool) { func (metrics *MinitorMetrics) CountCheck(monitor string, isSuccess bool, isAlert bool) {
status := "failure" status := "failure"
if isSuccess { if isSuccess {
status = "success" status = "success"
@@ -91,10 +81,6 @@ func (metrics *MinitorMetrics) CountCheck(monitor string, isSuccess bool, ms int
metrics.checkCount.With( metrics.checkCount.With(
prometheus.Labels{"monitor": monitor, "status": status, "is_alert": alertVal}, prometheus.Labels{"monitor": monitor, "status": status, "is_alert": alertVal},
).Inc() ).Inc()
metrics.checkTime.With(
prometheus.Labels{"monitor": monitor, "status": status},
).Set(float64(ms))
} }
// CountAlert counts an alert // CountAlert counts an alert
@@ -110,8 +96,6 @@ func (metrics *MinitorMetrics) CountAlert(monitor string, alert string) {
// ServeMetrics starts an http server with a Prometheus metrics handler // ServeMetrics starts an http server with a Prometheus metrics handler
func ServeMetrics() { func ServeMetrics() {
http.Handle("/metrics", promhttp.Handler()) http.Handle("/metrics", promhttp.Handler())
host := fmt.Sprintf(":%d", MetricsPort) host := fmt.Sprintf(":%d", MetricsPort)
_ = http.ListenAndServe(host, nil) _ = http.ListenAndServe(host, nil)
} }
+43 -47
View File
@@ -1,37 +1,38 @@
package main package main
import ( import (
"log"
"math" "math"
"os/exec" "os/exec"
"time" "time"
"git.iamthefij.com/iamthefij/slog"
) )
// Monitor represents a particular periodic check of a command // Monitor represents a particular periodic check of a command
type Monitor struct { //nolint:maligned type Monitor struct {
// Config values // Config values
AlertAfter int16 `yaml:"alert_after"`
AlertEvery int16 `yaml:"alert_every"`
CheckInterval SecondsOrDuration `yaml:"check_interval"`
Name string Name string
Command []string
CommandShell string `yaml:"command_shell"`
AlertDown []string `yaml:"alert_down"` AlertDown []string `yaml:"alert_down"`
AlertUp []string `yaml:"alert_up"` AlertUp []string `yaml:"alert_up"`
Command CommandOrShell CheckInterval float64 `yaml:"check_interval"`
AlertAfter int16 `yaml:"alert_after"`
AlertEvery int16 `yaml:"alert_every"`
// Other values // Other values
lastCheck time.Time
lastOutput string
alertCount int16 alertCount int16
failureCount int16 failureCount int16
lastCheck time.Time
lastSuccess time.Time lastSuccess time.Time
lastOutput string
lastCheckDuration time.Duration
} }
// IsValid returns a boolean indicating if the Monitor has been correctly // IsValid returns a boolean indicating if the Monitor has been correctly
// configured // configured
func (monitor Monitor) IsValid() bool { func (monitor Monitor) IsValid() bool {
return (!monitor.Command.Empty() && atLeastOneCommand := (monitor.CommandShell != "" || monitor.Command != nil)
atMostOneCommand := (monitor.CommandShell == "" || monitor.Command == nil)
return (atLeastOneCommand &&
atMostOneCommand &&
monitor.getAlertAfter() > 0 && monitor.getAlertAfter() > 0 &&
monitor.AlertDown != nil) monitor.AlertDown != nil)
} }
@@ -43,29 +44,25 @@ func (monitor Monitor) ShouldCheck() bool {
return true return true
} }
sinceLastCheck := time.Since(monitor.lastCheck) sinceLastCheck := time.Now().Sub(monitor.lastCheck).Seconds()
return sinceLastCheck >= monitor.CheckInterval
return sinceLastCheck >= monitor.CheckInterval.Value()
} }
// Check will run the command configured by the Monitor and return a status // Check will run the command configured by the Monitor and return a status
// and a possible AlertNotice // and a possible AlertNotice
func (monitor *Monitor) Check() (bool, *AlertNotice) { func (monitor *Monitor) Check() (bool, *AlertNotice) {
var cmd *exec.Cmd var cmd *exec.Cmd
if monitor.Command.Command != nil { if monitor.Command != nil {
cmd = exec.Command(monitor.Command.Command[0], monitor.Command.Command[1:]...) cmd = exec.Command(monitor.Command[0], monitor.Command[1:]...)
} else { } else {
cmd = ShellCommand(monitor.Command.ShellCommand) cmd = ShellCommand(monitor.CommandShell)
} }
checkStartTime := time.Now()
output, err := cmd.CombinedOutput() output, err := cmd.CombinedOutput()
monitor.lastCheck = time.Now() monitor.lastCheck = time.Now()
monitor.lastOutput = string(output) monitor.lastOutput = string(output)
monitor.lastCheckDuration = monitor.lastCheck.Sub(checkStartTime)
var alertNotice *AlertNotice var alertNotice *AlertNotice
isSuccess := (err == nil) isSuccess := (err == nil)
if isSuccess { if isSuccess {
alertNotice = monitor.success() alertNotice = monitor.success()
@@ -73,11 +70,17 @@ func (monitor *Monitor) Check() (bool, *AlertNotice) {
alertNotice = monitor.failure() alertNotice = monitor.failure()
} }
slog.Debugf("Command output: %s", monitor.lastOutput) if LogDebug {
slog.OnErrWarnf(err, "Command result: %v", err) log.Printf("DEBUG: Command output: %s", monitor.lastOutput)
}
if err != nil {
if LogDebug {
log.Printf("DEBUG: Command result: %v", err)
}
}
slog.Infof( log.Printf(
"%s success=%t, alert=%t", "INFO: %s success=%t, alert=%t",
monitor.Name, monitor.Name,
isSuccess, isSuccess,
alertNotice != nil, alertNotice != nil,
@@ -86,22 +89,15 @@ func (monitor *Monitor) Check() (bool, *AlertNotice) {
return isSuccess, alertNotice return isSuccess, alertNotice
} }
// IsUp returns the status of the current monitor func (monitor Monitor) isUp() bool {
func (monitor Monitor) IsUp() bool {
return monitor.alertCount == 0 return monitor.alertCount == 0
} }
// LastCheckMilliseconds gives number of miliseconds the last check ran for
func (monitor Monitor) LastCheckMilliseconds() int64 {
return monitor.lastCheckDuration.Milliseconds()
}
func (monitor *Monitor) success() (notice *AlertNotice) { func (monitor *Monitor) success() (notice *AlertNotice) {
if !monitor.IsUp() { if !monitor.isUp() {
// Alert that we have recovered // Alert that we have recovered
notice = monitor.createAlertNotice(true) notice = monitor.createAlertNotice(true)
} }
monitor.failureCount = 0 monitor.failureCount = 0
monitor.alertCount = 0 monitor.alertCount = 0
monitor.lastSuccess = time.Now() monitor.lastSuccess = time.Now()
@@ -113,14 +109,15 @@ func (monitor *Monitor) failure() (notice *AlertNotice) {
monitor.failureCount++ monitor.failureCount++
// If we haven't hit the minimum failures, we can exit // If we haven't hit the minimum failures, we can exit
if monitor.failureCount < monitor.getAlertAfter() { if monitor.failureCount < monitor.getAlertAfter() {
slog.Debugf( if LogDebug {
"%s failed but did not hit minimum failures. "+ log.Printf(
"DEBUG: %s failed but did not hit minimum failures. "+
"Count: %v alert after: %v", "Count: %v alert after: %v",
monitor.Name, monitor.Name,
monitor.failureCount, monitor.failureCount,
monitor.getAlertAfter(), monitor.getAlertAfter(),
) )
}
return return
} }
@@ -128,20 +125,19 @@ func (monitor *Monitor) failure() (notice *AlertNotice) {
failureCount := (monitor.failureCount - monitor.getAlertAfter()) failureCount := (monitor.failureCount - monitor.getAlertAfter())
// Use alert cadence to determine if we should alert // Use alert cadence to determine if we should alert
switch { if monitor.AlertEvery > 0 {
case monitor.AlertEvery > 0:
// Handle integer number of failures before alerting // Handle integer number of failures before alerting
if failureCount%monitor.AlertEvery == 0 { if failureCount%monitor.AlertEvery == 0 {
notice = monitor.createAlertNotice(false) notice = monitor.createAlertNotice(false)
} }
case monitor.AlertEvery == 0: } else if monitor.AlertEvery == 0 {
// Handle alerting on first failure only // Handle alerting on first failure only
if failureCount == 0 { if failureCount == 0 {
notice = monitor.createAlertNotice(false) notice = monitor.createAlertNotice(false)
} }
default: } else {
// Handle negative numbers indicating an exponential backoff // Handle negative numbers indicating an exponential backoff
if failureCount >= int16(math.Pow(2, float64(monitor.alertCount))-1) { //nolint:gomnd if failureCount >= int16(math.Pow(2, float64(monitor.alertCount))-1) {
notice = monitor.createAlertNotice(false) notice = monitor.createAlertNotice(false)
} }
} }
@@ -151,7 +147,7 @@ func (monitor *Monitor) failure() (notice *AlertNotice) {
monitor.alertCount++ monitor.alertCount++
} }
return notice return
} }
func (monitor Monitor) getAlertAfter() int16 { func (monitor Monitor) getAlertAfter() int16 {
@@ -159,18 +155,18 @@ func (monitor Monitor) getAlertAfter() int16 {
// Zero is one! // Zero is one!
if monitor.AlertAfter == 0 { if monitor.AlertAfter == 0 {
return 1 return 1
} } else {
return monitor.AlertAfter return monitor.AlertAfter
}
} }
// GetAlertNames gives a list of alert names for a given monitor status // GetAlertNames gives a list of alert names for a given monitor status
func (monitor Monitor) GetAlertNames(up bool) []string { func (monitor Monitor) GetAlertNames(up bool) []string {
if up { if up {
return monitor.AlertUp return monitor.AlertUp
} } else {
return monitor.AlertDown return monitor.AlertDown
}
} }
func (monitor Monitor) createAlertNotice(isUp bool) *AlertNotice { func (monitor Monitor) createAlertNotice(isUp bool) *AlertNotice {
+19 -33
View File
@@ -13,22 +13,25 @@ func TestMonitorIsValid(t *testing.T) {
expected bool expected bool
name string name string
}{ }{
{Monitor{Command: CommandOrShell{Command: []string{"echo", "test"}}, AlertDown: []string{"log"}}, true, "Command only"}, {Monitor{Command: []string{"echo", "test"}, AlertDown: []string{"log"}}, true, "Command only"},
{Monitor{Command: CommandOrShell{ShellCommand: "echo test"}, AlertDown: []string{"log"}}, true, "CommandShell only"}, {Monitor{CommandShell: "echo test", AlertDown: []string{"log"}}, true, "CommandShell only"},
{Monitor{Command: CommandOrShell{Command: []string{"echo", "test"}}}, false, "No AlertDown"}, {Monitor{Command: []string{"echo", "test"}}, false, "No AlertDown"},
{Monitor{AlertDown: []string{"log"}}, false, "No commands"}, {Monitor{AlertDown: []string{"log"}}, false, "No commands"},
{Monitor{Command: CommandOrShell{Command: []string{"echo", "test"}}, AlertDown: []string{"log"}, AlertAfter: -1}, false, "Invalid alert threshold, -1"}, {
Monitor{Command: []string{"echo", "test"}, CommandShell: "echo test", AlertDown: []string{"log"}},
false,
"Both commands",
},
{Monitor{Command: []string{"echo", "test"}, AlertDown: []string{"log"}, AlertAfter: -1}, false, "Invalid alert threshold, -1"},
} }
for _, c := range cases { for _, c := range cases {
log.Printf("Testing case %s", c.name) log.Printf("Testing case %s", c.name)
actual := c.monitor.IsValid() actual := c.monitor.IsValid()
if actual != c.expected { if actual != c.expected {
t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expected, actual) t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expected, actual)
log.Printf("Case failed: %s", c.name) log.Printf("Case failed: %s", c.name)
} }
log.Println("-----") log.Println("-----")
} }
} }
@@ -45,9 +48,9 @@ func TestMonitorShouldCheck(t *testing.T) {
name string name string
}{ }{
{Monitor{}, true, "Empty"}, {Monitor{}, true, "Empty"},
{Monitor{lastCheck: timeNow, CheckInterval: SecondsOrDuration{time.Second * 15}}, false, "Just checked"}, {Monitor{lastCheck: timeNow, CheckInterval: 15}, false, "Just checked"},
{Monitor{lastCheck: timeTenSecAgo, CheckInterval: SecondsOrDuration{time.Second * 15}}, false, "-10s"}, {Monitor{lastCheck: timeTenSecAgo, CheckInterval: 15}, false, "-10s"},
{Monitor{lastCheck: timeTwentySecAgo, CheckInterval: SecondsOrDuration{time.Second * 15}}, true, "-20s"}, {Monitor{lastCheck: timeTwentySecAgo, CheckInterval: 15}, true, "-20s"},
} }
for _, c := range cases { for _, c := range cases {
@@ -58,7 +61,7 @@ func TestMonitorShouldCheck(t *testing.T) {
} }
} }
// TestMonitorIsUp tests the Monitor.IsUp() // TestMonitorIsUp tests the Monitor.isUp()
func TestMonitorIsUp(t *testing.T) { func TestMonitorIsUp(t *testing.T) {
cases := []struct { cases := []struct {
monitor Monitor monitor Monitor
@@ -73,13 +76,11 @@ func TestMonitorIsUp(t *testing.T) {
for _, c := range cases { for _, c := range cases {
log.Printf("Testing case %s", c.name) log.Printf("Testing case %s", c.name)
actual := c.monitor.isUp()
actual := c.monitor.IsUp()
if actual != c.expected { if actual != c.expected {
t.Errorf("IsUp(%v), expected=%t actual=%t", c.name, c.expected, actual) t.Errorf("isUp(%v), expected=%t actual=%t", c.name, c.expected, actual)
log.Printf("Case failed: %s", c.name) log.Printf("Case failed: %s", c.name)
} }
log.Println("-----") log.Println("-----")
} }
} }
@@ -100,13 +101,11 @@ func TestMonitorGetAlertNames(t *testing.T) {
for _, c := range cases { for _, c := range cases {
log.Printf("Testing case %s", c.name) log.Printf("Testing case %s", c.name)
actual := c.monitor.GetAlertNames(c.up) actual := c.monitor.GetAlertNames(c.up)
if !EqualSliceString(actual, c.expected) { if !EqualSliceString(actual, c.expected) {
t.Errorf("GetAlertNames(%v), expected=%v actual=%v", c.name, c.expected, actual) t.Errorf("GetAlertNames(%v), expected=%v actual=%v", c.name, c.expected, actual)
log.Printf("Case failed: %s", c.name) log.Printf("Case failed: %s", c.name)
} }
log.Println("-----") log.Println("-----")
} }
} }
@@ -125,15 +124,12 @@ func TestMonitorSuccess(t *testing.T) {
for _, c := range cases { for _, c := range cases {
log.Printf("Testing case %s", c.name) log.Printf("Testing case %s", c.name)
notice := c.monitor.success() notice := c.monitor.success()
hasNotice := (notice != nil) hasNotice := (notice != nil)
if hasNotice != c.expectNotice { if hasNotice != c.expectNotice {
t.Errorf("success(%v), expected=%t actual=%t", c.name, c.expectNotice, hasNotice) t.Errorf("success(%v), expected=%t actual=%t", c.name, c.expectNotice, hasNotice)
log.Printf("Case failed: %s", c.name) log.Printf("Case failed: %s", c.name)
} }
log.Println("-----") log.Println("-----")
} }
} }
@@ -156,15 +152,12 @@ func TestMonitorFailureAlertAfter(t *testing.T) {
for _, c := range cases { for _, c := range cases {
log.Printf("Testing case %s", c.name) log.Printf("Testing case %s", c.name)
notice := c.monitor.failure() notice := c.monitor.failure()
hasNotice := (notice != nil) hasNotice := (notice != nil)
if hasNotice != c.expectNotice { if hasNotice != c.expectNotice {
t.Errorf("failure(%v), expected=%t actual=%t", c.name, c.expectNotice, hasNotice) t.Errorf("failure(%v), expected=%t actual=%t", c.name, c.expectNotice, hasNotice)
log.Printf("Case failed: %s", c.name) log.Printf("Case failed: %s", c.name)
} }
log.Println("-----") log.Println("-----")
} }
} }
@@ -207,12 +200,10 @@ func TestMonitorFailureAlertEvery(t *testing.T) {
notice := c.monitor.failure() notice := c.monitor.failure()
hasNotice := (notice != nil) hasNotice := (notice != nil)
if hasNotice != c.expectNotice { if hasNotice != c.expectNotice {
t.Errorf("failure(%v), expected=%t actual=%t", c.name, c.expectNotice, hasNotice) t.Errorf("failure(%v), expected=%t actual=%t", c.name, c.expectNotice, hasNotice)
log.Printf("Case failed: %s", c.name) log.Printf("Case failed: %s", c.name)
} }
log.Println("-----") log.Println("-----")
} }
} }
@@ -237,18 +228,15 @@ func TestMonitorFailureExponential(t *testing.T) {
// Unlike previous tests, this one requires a static Monitor with repeated // Unlike previous tests, this one requires a static Monitor with repeated
// calls to the failure method // calls to the failure method
monitor := Monitor{failureCount: 0, AlertAfter: 1, AlertEvery: -1} monitor := Monitor{failureCount: 0, AlertAfter: 1, AlertEvery: -1}
for _, c := range cases { for _, c := range cases {
log.Printf("Testing case %s", c.name) log.Printf("Testing case %s", c.name)
notice := monitor.failure() notice := monitor.failure()
hasNotice := (notice != nil) hasNotice := (notice != nil)
if hasNotice != c.expectNotice { if hasNotice != c.expectNotice {
t.Errorf("failure(%v), expected=%t actual=%t", c.name, c.expectNotice, hasNotice) t.Errorf("failure(%v), expected=%t actual=%t", c.name, c.expectNotice, hasNotice)
log.Printf("Case failed: %s", c.name) log.Printf("Case failed: %s", c.name)
} }
log.Println("-----") log.Println("-----")
} }
} }
@@ -260,29 +248,28 @@ func TestMonitorCheck(t *testing.T) {
hasNotice bool hasNotice bool
lastOutput string lastOutput string
} }
cases := []struct { cases := []struct {
monitor Monitor monitor Monitor
expect expected expect expected
name string name string
}{ }{
{ {
Monitor{Command: CommandOrShell{Command: []string{"echo", "success"}}}, Monitor{Command: []string{"echo", "success"}},
expected{isSuccess: true, hasNotice: false, lastOutput: "success\n"}, expected{isSuccess: true, hasNotice: false, lastOutput: "success\n"},
"Test successful command", "Test successful command",
}, },
{ {
Monitor{Command: CommandOrShell{ShellCommand: "echo success"}}, Monitor{CommandShell: "echo success"},
expected{isSuccess: true, hasNotice: false, lastOutput: "success\n"}, expected{isSuccess: true, hasNotice: false, lastOutput: "success\n"},
"Test successful command shell", "Test successful command shell",
}, },
{ {
Monitor{Command: CommandOrShell{Command: []string{"total", "failure"}}}, Monitor{Command: []string{"total", "failure"}},
expected{isSuccess: false, hasNotice: true, lastOutput: ""}, expected{isSuccess: false, hasNotice: true, lastOutput: ""},
"Test failed command", "Test failed command",
}, },
{ {
Monitor{Command: CommandOrShell{ShellCommand: "false"}}, Monitor{CommandShell: "false"},
expected{isSuccess: false, hasNotice: true, lastOutput: ""}, expected{isSuccess: false, hasNotice: true, lastOutput: ""},
"Test failed command shell", "Test failed command shell",
}, },
@@ -308,7 +295,6 @@ func TestMonitorCheck(t *testing.T) {
t.Errorf("Check(%v) (output), expected=%v actual=%v", c.name, c.expect.lastOutput, lastOutput) t.Errorf("Check(%v) (output), expected=%v actual=%v", c.name, c.expect.lastOutput, lastOutput)
log.Printf("Case failed: %s", c.name) log.Printf("Case failed: %s", c.name)
} }
log.Println("-----") log.Println("-----")
} }
} }
+5 -6
View File
@@ -3,14 +3,14 @@ check_interval: 5
monitors: monitors:
- name: Fake Website - name: Fake Website
command: ["curl", "-s", "-o", "/dev/null", "https://minitor.mon"] command: ['curl', '-s', '-o', '/dev/null', 'https://minitor.mon']
alert_down: [log_down, mailgun_down, sms_down] alert_down: [log_down, mailgun_down, sms_down]
alert_up: [log_up, email_up] alert_up: [log_up, email_up]
check_interval: 10 # Must be at minimum the global `check_interval` check_interval: 10 # Must be at minimum the global `check_interval`
alert_after: 3 alert_after: 3
alert_every: -1 # Defaults to -1 for exponential backoff. 0 to disable repeating alert_every: -1 # Defaults to -1 for exponential backoff. 0 to disable repeating
- name: Real Website - name: Real Website
command: ["curl", "-s", "-o", "/dev/null", "https://google.com"] command: ['curl', '-s', '-o', '/dev/null', 'https://google.com']
alert_down: [log_down, mailgun_down, sms_down] alert_down: [log_down, mailgun_down, sms_down]
alert_up: [log_up, email_up] alert_up: [log_up, email_up]
check_interval: 5 check_interval: 5
@@ -23,10 +23,9 @@ alerts:
log_up: log_up:
command: ["echo", "Minitor recovery for {{.MonitorName}}"] command: ["echo", "Minitor recovery for {{.MonitorName}}"]
email_up: email_up:
command: command: [sendmail, "me@minitor.mon", "Recovered: {monitor_name}", "We're back!"]
[sendmail, "me@minitor.mon", "Recovered: {monitor_name}", "We're back!"]
mailgun_down: mailgun_down:
command: > command_shell: >
curl -s -X POST curl -s -X POST
-F subject="Alert! {{.MonitorName}} failed" -F subject="Alert! {{.MonitorName}} failed"
-F from="Minitor <minitor@minitor.mon>" -F from="Minitor <minitor@minitor.mon>"
@@ -35,7 +34,7 @@ alerts:
https://api.mailgun.net/v3/minitor.mon/messages https://api.mailgun.net/v3/minitor.mon/messages
-u "api:${MAILGUN_API_KEY}" -u "api:${MAILGUN_API_KEY}"
sms_down: sms_down:
command: > command_shell: >
curl -s -X POST -F "Body=Failure! {{.MonitorName}} has failed" curl -s -X POST -F "Body=Failure! {{.MonitorName}} has failed"
-F "From=${AVAILABLE_NUMBER}" -F "To=${MY_PHONE}" -F "From=${AVAILABLE_NUMBER}" -F "To=${MY_PHONE}"
"https://api.twilio.com/2010-04-01/Accounts/${ACCOUNT_SID}/Messages" "https://api.twilio.com/2010-04-01/Accounts/${ACCOUNT_SID}/Messages"
+3 -15
View File
@@ -11,7 +11,6 @@ set -e
# To override, export DOCKER_HOST to a new hostname # To override, export DOCKER_HOST to a new hostname
DOCKER_HOST="${DOCKER_HOST:=socket}" DOCKER_HOST="${DOCKER_HOST:=socket}"
container_name="$1" container_name="$1"
num_log_lines="$2"
# Curls Docker either using a socket or URL # Curls Docker either using a socket or URL
function curl_docker { function curl_docker {
@@ -32,32 +31,21 @@ function get_container_id {
# Returns container JSON # Returns container JSON
function inspect_container { function inspect_container {
local container_id="$1" local container_id=$1
curl_docker "containers/$container_id/json" curl_docker "containers/$container_id/json"
} }
# Gets some lines from docker log
function get_logs {
container_id="$1"
num_lines="$2"
curl_docker "containers/$container_id/logs?stdout=1&stderr=1" | tail -n "$num_lines"
}
if [ -z "$container_name" ]; then if [ -z "$container_name" ]; then
echo "Usage: $0 container_name [num_log_lines]" echo "Usage: $0 container_name"
echo "Will exit with the last status code of continer with provided name" echo "Will exit with the last status code of continer with provided name"
exit 1 exit 1
fi fi
container_id=$(get_container_id "$container_name") container_id=$(get_container_id $container_name)
if [ -z "$container_id" ]; then if [ -z "$container_id" ]; then
echo "ERROR: Could not find container with name: $container_name" echo "ERROR: Could not find container with name: $container_name"
exit 1 exit 1
fi fi
exit_code=$(inspect_container "$container_id" | jq -r .State.ExitCode) exit_code=$(inspect_container "$container_id" | jq -r .State.ExitCode)
if [ -n "$num_log_lines" ]; then
get_logs "$container_id" "$num_log_lines"
fi
exit "$exit_code" exit "$exit_code"
+1 -13
View File
@@ -11,7 +11,6 @@ set -e
# To override, export DOCKER_HOST to a new hostname # To override, export DOCKER_HOST to a new hostname
DOCKER_HOST="${DOCKER_HOST:=socket}" DOCKER_HOST="${DOCKER_HOST:=socket}"
container_name="$1" container_name="$1"
num_log_lines="$2"
# Curls Docker either using a socket or URL # Curls Docker either using a socket or URL
function curl_docker { function curl_docker {
@@ -36,15 +35,8 @@ function inspect_container {
curl_docker "containers/$container_id/json" curl_docker "containers/$container_id/json"
} }
# Gets some lines from docker log
function get_logs {
container_id="$1"
num_lines="$2"
curl_docker "containers/$container_id/logs?stdout=1&stderr=1" | tail -n "$num_lines"
}
if [ -z "$container_name" ]; then if [ -z "$container_name" ]; then
echo "Usage: $0 container_name [num_log_lines]" echo "Usage: $0 container_name"
echo "Will return results of healthcheck for continer with provided name" echo "Will return results of healthcheck for continer with provided name"
exit 1 exit 1
fi fi
@@ -56,10 +48,6 @@ if [ -z "$container_id" ]; then
fi fi
health=$(inspect_container "$container_id" | jq -r '.State.Health.Status') health=$(inspect_container "$container_id" | jq -r '.State.Health.Status')
if [ -n "$num_log_lines" ]; then
get_logs "$container_id" "$num_log_lines"
fi
case "$health" in case "$health" in
null) null)
echo "No healthcheck results" echo "No healthcheck results"
+6 -9
View File
@@ -1,25 +1,22 @@
---
check_interval: 1 check_interval: 1
monitors: monitors:
- name: Command - name: Command
command: ["echo", "$PATH"] command: ['echo', '$PATH']
alert_down: ["log_command", "log_shell"] alert_down: [ 'log_command', 'log_shell' ]
alert_every: 0 alert_every: 0
check_interval: 10s
- name: Shell - name: Shell
command: > command_shell: >
echo 'Some string with stuff'; echo 'Some string with stuff';
echo 'another line'; echo 'another line';
echo $PATH; echo $PATH;
exit 1 exit 1
alert_down: ["log_command", "log_shell"] alert_down: [ 'log_command', 'log_shell' ]
alert_after: 5 alert_after: 5
alert_every: 0 alert_every: 0
check_interval: 1m
alerts: alerts:
log_command: log_command:
command: ["echo", "regular", '"command!!!"', "{{.MonitorName}}"] command: [ 'echo', 'regular', '"command!!!"', "{{.MonitorName}}" ]
log_shell: log_shell:
command: echo "Failure on {{.MonitorName}} User is $USER" command_shell: echo "Failure on {{.MonitorName}} User is $USER"
-8
View File
@@ -1,8 +0,0 @@
---
check_interval: 1
monitors:
- name: Command
command: ['echo', '$PATH']
alert_down: ['log']
alert_every: 0
-18
View File
@@ -1,18 +0,0 @@
---
check_interval: 1
monitors:
- name: Shell
command: >
echo 'Some string with stuff';
echo "<angle brackets>";
exit 1
alert_down: ['log_shell']
alert_after: 1
alert_every: 0
alerts:
log_shell:
command: |
echo 'Some string with stuff'
echo '<angle brackets>'
+12 -4
View File
@@ -5,10 +5,20 @@ import (
"strings" "strings"
) )
// escapeCommandShell accepts a command to be executed by a shell and escapes it
func escapeCommandShell(command string) string {
// Remove extra spaces and newlines from ends
command = strings.TrimSpace(command)
// TODO: Not sure if this part is actually needed. Should verify
// Escape double quotes since this will be passed in as an argument
command = strings.Replace(command, `"`, `\"`, -1)
return command
}
// ShellCommand takes a string and executes it as a command using `sh` // ShellCommand takes a string and executes it as a command using `sh`
func ShellCommand(command string) *exec.Cmd { func ShellCommand(command string) *exec.Cmd {
shellCommand := []string{"sh", "-c", strings.TrimSpace(command)} shellCommand := []string{"sh", "-c", escapeCommandShell(command)}
//log.Printf("Shell command: %v", shellCommand)
return exec.Command(shellCommand[0], shellCommand[1:]...) return exec.Command(shellCommand[0], shellCommand[1:]...)
} }
@@ -17,12 +27,10 @@ func EqualSliceString(a, b []string) bool {
if len(a) != len(b) { if len(a) != len(b) {
return false return false
} }
for i, val := range a { for i, val := range a {
if val != b[i] { if val != b[i] {
return false return false
} }
} }
return true return true
} }