Compare commits
1 Commits
v0.2.1
..
prometheus
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6c7c0a470f |
+4
-105
@@ -1,114 +1,13 @@
|
|||||||
---
|
|
||||||
kind: pipeline
|
kind: pipeline
|
||||||
name: test
|
name: test
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
|
- name: build
|
||||||
|
image: golang:1.12
|
||||||
|
commands:
|
||||||
|
- make build
|
||||||
|
|
||||||
- name: test
|
- name: test
|
||||||
image: golang:1.12
|
image: golang:1.12
|
||||||
commands:
|
commands:
|
||||||
- make build
|
|
||||||
- make test
|
- make test
|
||||||
|
|
||||||
- name: check
|
|
||||||
image: python:3
|
|
||||||
commands:
|
|
||||||
- pip install pre-commit==1.20.0
|
|
||||||
- make check
|
|
||||||
|
|
||||||
- name: notify
|
|
||||||
image: drillster/drone-email
|
|
||||||
settings:
|
|
||||||
host:
|
|
||||||
from_secret: SMTP_HOST
|
|
||||||
username:
|
|
||||||
from_secret: SMTP_USER
|
|
||||||
password:
|
|
||||||
from_secret: SMTP_PASS
|
|
||||||
from: drone@iamthefij.com
|
|
||||||
when:
|
|
||||||
status: [changed, failure]
|
|
||||||
|
|
||||||
---
|
|
||||||
kind: pipeline
|
|
||||||
name: publish
|
|
||||||
|
|
||||||
depends_on:
|
|
||||||
- test
|
|
||||||
|
|
||||||
trigger:
|
|
||||||
event:
|
|
||||||
- push
|
|
||||||
- tag
|
|
||||||
refs:
|
|
||||||
- refs/heads/master
|
|
||||||
- refs/tags/v*
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: build all binaries
|
|
||||||
image: golang:1.12
|
|
||||||
commands:
|
|
||||||
- make all
|
|
||||||
|
|
||||||
- name: push image - arm
|
|
||||||
image: plugins/docker
|
|
||||||
settings:
|
|
||||||
repo: iamthefij/minitor-go
|
|
||||||
auto_tag: true
|
|
||||||
auto_tag_suffix: linux-arm
|
|
||||||
username:
|
|
||||||
from_secret: docker_username
|
|
||||||
password:
|
|
||||||
from_secret: docker_password
|
|
||||||
build_args:
|
|
||||||
- ARCH=arm
|
|
||||||
- REPO=arm32v7
|
|
||||||
|
|
||||||
- name: push image - arm64
|
|
||||||
image: plugins/docker
|
|
||||||
settings:
|
|
||||||
repo: iamthefij/minitor-go
|
|
||||||
auto_tag: true
|
|
||||||
auto_tag_suffix: linux-arm64
|
|
||||||
username:
|
|
||||||
from_secret: docker_username
|
|
||||||
password:
|
|
||||||
from_secret: docker_password
|
|
||||||
build_args:
|
|
||||||
- ARCH=arm64
|
|
||||||
- REPO=arm64v8
|
|
||||||
|
|
||||||
- name: push image - amd64
|
|
||||||
image: plugins/docker
|
|
||||||
settings:
|
|
||||||
repo: iamthefij/minitor-go
|
|
||||||
auto_tag: true
|
|
||||||
auto_tag_suffix: linux-amd64
|
|
||||||
username:
|
|
||||||
from_secret: docker_username
|
|
||||||
password:
|
|
||||||
from_secret: docker_password
|
|
||||||
|
|
||||||
- name: publish manifest
|
|
||||||
image: plugins/manifest
|
|
||||||
settings:
|
|
||||||
spec: manifest.tmpl
|
|
||||||
auto_tag: true
|
|
||||||
ignore_missing: true
|
|
||||||
username:
|
|
||||||
from_secret: docker_username
|
|
||||||
password:
|
|
||||||
from_secret: docker_password
|
|
||||||
|
|
||||||
- name: notify
|
|
||||||
image: drillster/drone-email
|
|
||||||
settings:
|
|
||||||
host:
|
|
||||||
from_secret: SMTP_HOST
|
|
||||||
username:
|
|
||||||
from_secret: SMTP_USER
|
|
||||||
password:
|
|
||||||
from_secret: SMTP_PASS
|
|
||||||
from: drone@iamthefij.com
|
|
||||||
when:
|
|
||||||
status: [changed, failure]
|
|
||||||
|
|||||||
Vendored
+1
-3
@@ -16,6 +16,4 @@
|
|||||||
config.yml
|
config.yml
|
||||||
|
|
||||||
# Output binary
|
# Output binary
|
||||||
minitor
|
minitor-go
|
||||||
minitor-linux-*
|
|
||||||
minitor-darwin-amd64
|
|
||||||
|
|||||||
@@ -1,19 +0,0 @@
|
|||||||
---
|
|
||||||
repos:
|
|
||||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
||||||
rev: v2.4.0
|
|
||||||
hooks:
|
|
||||||
- id: check-added-large-files
|
|
||||||
- id: check-yaml
|
|
||||||
args:
|
|
||||||
- --allow-multiple-documents
|
|
||||||
- id: trailing-whitespace
|
|
||||||
- id: end-of-file-fixer
|
|
||||||
- id: check-merge-conflict
|
|
||||||
- repo: git://github.com/dnephin/pre-commit-golang
|
|
||||||
rev: v0.3.5
|
|
||||||
hooks:
|
|
||||||
- id: go-fmt
|
|
||||||
- id: go-imports
|
|
||||||
# - id: gometalinter
|
|
||||||
# - id: golangci-lint
|
|
||||||
+3
-23
@@ -1,28 +1,8 @@
|
|||||||
ARG REPO=library
|
ARG REPO=library
|
||||||
FROM multiarch/qemu-user-static:4.2.0-2 as qemu-user-static
|
FROM ${REPO}/busybox:latest
|
||||||
FROM ${REPO}/alpine:3.10
|
WORKDIR /root/
|
||||||
|
|
||||||
# Copying all qemu files because amd64 doesn't exist and cannot condional copy
|
|
||||||
COPY --from=qemu-user-static /usr/bin/qemu-* /usr/bin/
|
|
||||||
|
|
||||||
RUN mkdir /app
|
|
||||||
WORKDIR /app/
|
|
||||||
|
|
||||||
# Add common checking tools
|
|
||||||
RUN apk --no-cache add bash=~5.0 curl=~7.66 jq=~1.6
|
|
||||||
|
|
||||||
# Add minitor user for running as non-root
|
|
||||||
RUN addgroup -S minitor && adduser -S minitor -G minitor
|
|
||||||
|
|
||||||
# Copy scripts
|
|
||||||
COPY ./scripts /app/scripts
|
|
||||||
RUN chmod -R 755 /app/scripts
|
|
||||||
|
|
||||||
# Copy minitor in
|
|
||||||
ARG ARCH=amd64
|
ARG ARCH=amd64
|
||||||
COPY ./minitor-linux-${ARCH} ./minitor
|
COPY ./minitor-go ./minitor
|
||||||
|
|
||||||
# Drop to non-root user
|
|
||||||
USER minitor
|
|
||||||
|
|
||||||
ENTRYPOINT [ "./minitor" ]
|
ENTRYPOINT [ "./minitor" ]
|
||||||
|
|||||||
+3
-21
@@ -1,7 +1,7 @@
|
|||||||
ARG REPO=library
|
ARG REPO=library
|
||||||
FROM golang:1.12-alpine AS builder
|
FROM golang:1.12-alpine AS builder
|
||||||
|
|
||||||
RUN apk add --no-cache git=~2
|
RUN apk add --no-cache git
|
||||||
|
|
||||||
RUN mkdir /app
|
RUN mkdir /app
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
@@ -16,26 +16,8 @@ ARG VERSION=dev
|
|||||||
ENV CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH}
|
ENV CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH}
|
||||||
RUN go build -ldflags "-X main.version=${VERSION}" -a -installsuffix nocgo -o minitor .
|
RUN go build -ldflags "-X main.version=${VERSION}" -a -installsuffix nocgo -o minitor .
|
||||||
|
|
||||||
FROM ${REPO}/alpine:3.10
|
FROM ${REPO}/busybox:latest
|
||||||
RUN mkdir /app
|
WORKDIR /root/
|
||||||
WORKDIR /app/
|
|
||||||
|
|
||||||
# Copy minitor in
|
|
||||||
COPY --from=builder /app/minitor .
|
COPY --from=builder /app/minitor .
|
||||||
|
|
||||||
# Add common checking tools
|
|
||||||
RUN apk --no-cache add bash=~5.0 curl=~7.66 jq=~1.6
|
|
||||||
|
|
||||||
# Add minitor user for running as non-root
|
|
||||||
RUN addgroup -S minitor && adduser -S minitor -G minitor
|
|
||||||
|
|
||||||
# Copy scripts
|
|
||||||
COPY ./scripts /app/scripts
|
|
||||||
RUN chmod -R 755 /app/scripts
|
|
||||||
|
|
||||||
# Drop to non-root user
|
|
||||||
USER minitor
|
|
||||||
|
|
||||||
ENTRYPOINT [ "./minitor" ]
|
ENTRYPOINT [ "./minitor" ]
|
||||||
|
|
||||||
# vim: set filetype=dockerfile:
|
|
||||||
|
|||||||
@@ -1,28 +1,18 @@
|
|||||||
DOCKER_TAG ?= minitor-go-${USER}
|
DOCKER_TAG ?= minitor-go-${USER}
|
||||||
GIT_TAG_NAME := $(shell git tag -l --contains HEAD)
|
|
||||||
GIT_SHA := $(shell git rev-parse HEAD)
|
|
||||||
VERSION := $(if $(GIT_TAG_NAME),$(GIT_TAG_NAME),$(GIT_SHA))
|
|
||||||
|
|
||||||
.PHONY: all
|
.PHONY: test
|
||||||
all: minitor-linux-amd64 minitor-linux-arm minitor-linux-arm64
|
|
||||||
|
|
||||||
.PHONY: default
|
|
||||||
default: test
|
default: test
|
||||||
|
|
||||||
.PHONY: build
|
.PHONY: build
|
||||||
build: minitor
|
build:
|
||||||
|
go build
|
||||||
|
|
||||||
minitor:
|
minitor-go:
|
||||||
@echo Version: $(VERSION)
|
go build
|
||||||
go build -ldflags '-X "main.version=${VERSION}"' -o minitor
|
|
||||||
|
|
||||||
.PHONY: run
|
.PHONY: run
|
||||||
run: minitor build
|
run: minitor-go build
|
||||||
./minitor -debug
|
./minitor-go -debug
|
||||||
|
|
||||||
.PHONY: run-metrics
|
|
||||||
run-metrics: minitor build
|
|
||||||
./minitor -debug -metrics
|
|
||||||
|
|
||||||
.PHONY: test
|
.PHONY: test
|
||||||
test:
|
test:
|
||||||
@@ -34,68 +24,16 @@ test:
|
|||||||
@go tool cover -func=coverage.out | awk -v target=80.0% \
|
@go tool cover -func=coverage.out | awk -v target=80.0% \
|
||||||
'/^total:/ { print "Total coverage: " $$3 " Minimum coverage: " target; if ($$3+0.0 >= target+0.0) print "ok"; else { print "fail"; exit 1; } }'
|
'/^total:/ { print "Total coverage: " $$3 " Minimum coverage: " target; if ($$3+0.0 >= target+0.0) print "ok"; else { print "fail"; exit 1; } }'
|
||||||
|
|
||||||
# Installs pre-commit hooks
|
|
||||||
.PHONY: install-hooks
|
|
||||||
install-hooks:
|
|
||||||
pre-commit install --install-hooks
|
|
||||||
|
|
||||||
# Checks files for encryption
|
|
||||||
.PHONY: check
|
|
||||||
check:
|
|
||||||
pre-commit run --all-files
|
|
||||||
|
|
||||||
.PHONY: clean
|
.PHONY: clean
|
||||||
clean:
|
clean:
|
||||||
rm -f ./minitor
|
rm -f ./minitor-go
|
||||||
rm -f ./minitor-linux-*
|
|
||||||
rm -f ./minitor-darwin-amd64
|
|
||||||
rm -f ./coverage.out
|
rm -f ./coverage.out
|
||||||
|
|
||||||
.PHONY: docker-build
|
.PHONY: docker-build
|
||||||
docker-build:
|
docker-build:
|
||||||
docker build -f ./Dockerfile.multi-stage -t $(DOCKER_TAG)-linux-amd64 .
|
docker build -f ./Dockerfile.multi-stage -t $(DOCKER_TAG) .
|
||||||
|
|
||||||
.PHONY: docker-run
|
.PHONY: docker-run
|
||||||
docker-run: docker-build
|
docker-run: docker-build
|
||||||
docker run --rm -v $(shell pwd)/config.yml:/root/config.yml $(DOCKER_TAG)
|
docker run --rm -v $(shell pwd)/config.yml:/root/config.yml $(DOCKER_TAG)
|
||||||
|
|
||||||
## Multi-arch targets
|
|
||||||
|
|
||||||
# Arch specific go build targets
|
|
||||||
minitor-darwin-amd64:
|
|
||||||
GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 \
|
|
||||||
go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
|
|
||||||
-o minitor-darwin-amd64
|
|
||||||
|
|
||||||
minitor-linux-amd64:
|
|
||||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \
|
|
||||||
go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
|
|
||||||
-o minitor-linux-amd64
|
|
||||||
|
|
||||||
minitor-linux-arm:
|
|
||||||
GOOS=linux GOARCH=arm CGO_ENABLED=0 \
|
|
||||||
go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
|
|
||||||
-o minitor-linux-arm
|
|
||||||
|
|
||||||
minitor-linux-arm64:
|
|
||||||
GOOS=linux GOARCH=arm64 CGO_ENABLED=0 \
|
|
||||||
go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
|
|
||||||
-o minitor-linux-arm64
|
|
||||||
|
|
||||||
# Arch specific docker build targets
|
|
||||||
.PHONY: docker-build-arm
|
|
||||||
docker-build-arm: minitor-linux-arm
|
|
||||||
docker build --build-arg REPO=arm32v7 --build-arg ARCH=arm . -t ${DOCKER_TAG}-linux-arm
|
|
||||||
|
|
||||||
.PHONY: docker-build-arm
|
|
||||||
docker-build-arm64: minitor-linux-arm64
|
|
||||||
docker build --build-arg REPO=arm64v8 --build-arg ARCH=arm64 . -t ${DOCKER_TAG}-linux-arm64
|
|
||||||
|
|
||||||
# Cross run on host architechture
|
|
||||||
.PHONY: docker-run-arm
|
|
||||||
docker-run-arm: docker-build-arm
|
|
||||||
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock --name $(DOCKER_TAG)-run ${DOCKER_TAG}-linux-arm
|
|
||||||
|
|
||||||
.PHONY: docker-run-arm64
|
|
||||||
docker-run-arm64: docker-build-arm64
|
|
||||||
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock --name $(DOCKER_TAG)-run ${DOCKER_TAG}-linux-arm64
|
|
||||||
|
|||||||
@@ -1,156 +1,97 @@
|
|||||||
# [minitor-go](https://git.iamthefij.com/iamthefij/minitor-go)
|
# minitor-go
|
||||||
|
|
||||||
A minimal monitoring system
|
A reimplementation of [Minitor](https://git.iamthefij/iamthefij/minitor) in Go
|
||||||
|
|
||||||
## What does it do?
|
Minitor is already a very minimal monitoring tool. Python 3 was a quick way to get something live, but Python itself comes with a very large footprint.Thus Go feels like a better fit for the project, longer term.
|
||||||
|
|
||||||
Minitor accepts a YAML configuration file with a set of commands to run and a set of alerts to execute when those commands fail. It is designed to be as simple as possible and relies on other command line tools to do checks and issue alerts.
|
|
||||||
|
|
||||||
## But why?
|
|
||||||
|
|
||||||
I'm running a few small services and found Sensu, Consul, Nagios, etc. to all be far too complicated for my usecase.
|
|
||||||
|
|
||||||
## So how do I use it?
|
|
||||||
|
|
||||||
### Running
|
|
||||||
|
|
||||||
Install and execute with:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go get github.com/iamthefij/minitor-go
|
|
||||||
minitor
|
|
||||||
```
|
|
||||||
|
|
||||||
If locally developing you can use:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
make run
|
|
||||||
```
|
|
||||||
|
|
||||||
It will read the contents of `config.yml` and begin its loop. You could also run it directly and provide a new config file via the `-config` argument.
|
|
||||||
|
|
||||||
|
|
||||||
#### Docker
|
|
||||||
|
|
||||||
You can pull this repository directly from Docker:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker pull iamthefij/minitor-go:latest
|
|
||||||
```
|
|
||||||
|
|
||||||
The Docker image uses a default `config.yml` that is copied from `sample-config.yml`. This won't really do anything for you, so when you run the Docker image, you should supply your own `config.yml` file:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker run -v $PWD/config.yml:/app/config.yml iamthefij/minitor-go:latest
|
|
||||||
```
|
|
||||||
|
|
||||||
Images are provided for `amd64`, `arm`, and `arm64` architechtures.
|
|
||||||
|
|
||||||
## Configuring
|
|
||||||
|
|
||||||
In this repo, you can explore the `sample-config.yml` file for an example, but the general structure is as follows. It should be noted that environment variable interpolation happens on load of the YAML file.
|
|
||||||
|
|
||||||
The global configurations are:
|
|
||||||
|
|
||||||
|key|value|
|
|
||||||
|---|---|
|
|
||||||
|`check_interval`|Maximum frequency to run checks for each monitor|
|
|
||||||
|`monitors`|List of all monitors. Detailed description below|
|
|
||||||
|`alerts`|List of all alerts. Detailed description below|
|
|
||||||
|
|
||||||
### Monitors
|
|
||||||
|
|
||||||
All monitors should be listed under `monitors`.
|
|
||||||
|
|
||||||
Each monitor allows the following configuration:
|
|
||||||
|
|
||||||
|key|value|
|
|
||||||
|---|---|
|
|
||||||
|`name`|Name of the monitor running. This will show up in messages and logs.|
|
|
||||||
|`command`|Specifies the command that should be executed, either in exec or shell form. This command's exit value will determine whether the check is successful|
|
|
||||||
|`alert_down`|A list of Alerts to be triggered when the monitor is in a "down" state|
|
|
||||||
|`alert_up`|A list of Alerts to be triggered when the monitor moves to an "up" state|
|
|
||||||
|`check_interval`|The interval at which this monitor should be checked. This must be greater than the global `check_interval` value|
|
|
||||||
|`alert_after`|Allows specifying the number of failed checks before an alert should be triggered|
|
|
||||||
|`alert_every`|Allows specifying how often an alert should be retriggered. There are a few magic numbers here. Defaults to `-1` for an exponential backoff. Setting to `0` disables re-alerting. Positive values will allow retriggering after the specified number of checks|
|
|
||||||
|
|
||||||
### Alerts
|
|
||||||
|
|
||||||
Alerts exist as objects keyed under `alerts`. Their key should be the name of the Alert. This is used in your monitor setup in `alert_down` and `alert_up`.
|
|
||||||
|
|
||||||
Eachy alert allows the following configuration:
|
|
||||||
|
|
||||||
|key|value|
|
|
||||||
|---|---|
|
|
||||||
|`command`|Specifies the command that should be executed, either in exec or shell form. This is the command that will be run when the alert is executed. This can be templated with environment variables or the variables shown in the table below|
|
|
||||||
|
|
||||||
Also, when alerts are executed, they will be passed through Go's format function with arguments for some attributes of the Monitor. The following monitor specific variables can be referenced using Go formatting syntax:
|
|
||||||
|
|
||||||
|token|value|
|
|
||||||
|---|---|
|
|
||||||
|`{{.AlertCount}}`|Number of times this monitor has alerted|
|
|
||||||
|`{{.FailureCount}}`|The total number of sequential failed checks for this monitor|
|
|
||||||
|`{{.LastCheckOutput}}`|The last returned value from the check command to either stderr or stdout|
|
|
||||||
|`{{.LastSuccess}}`|The ISO datetime of the last successful check|
|
|
||||||
|`{{.MonitorName}}`|The name of the monitor that failed and triggered the alert|
|
|
||||||
|
|
||||||
### Metrics
|
|
||||||
|
|
||||||
Minitor supports exporting metrics for [Prometheus](https://prometheus.io/). Prometheus is an open source tool for reading and querying metrics from different sources. Combined with another tool, [Grafana](https://grafana.com/), it allows building of charts and dashboards. You could also opt to just use Minitor to log check results, and instead do your alerting with Grafana.
|
|
||||||
|
|
||||||
It is also possible to use the metrics endpoint for monitoring Minitor itself! This allows setting up multiple instances of Minitor on different servers and have them monitor each-other so that you can detect a minitor outage.
|
|
||||||
|
|
||||||
To run minitor with metrics, use the `-metrics` flag. The metrics will be served on port `8080` by default, though it can be overriden using `-metrics-port`. They will be accessible on the path `/metrics`. Eg. `localhost:8080/metrics`.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
minitor -metrics
|
|
||||||
# or
|
|
||||||
minitor -metrics -metrics-port 3000
|
|
||||||
```
|
|
||||||
|
|
||||||
## Contributing
|
|
||||||
|
|
||||||
Whether you're looking to submit a patch or just tell me I broke something, you can contribute through the Github mirror and I can merge PRs back to the source repository.
|
|
||||||
|
|
||||||
Primary Repo: https://git.iamthefij.com/iamthefij/minitor.git
|
|
||||||
|
|
||||||
Github Mirror: https://github.com/IamTheFij/minitor.git
|
|
||||||
|
|
||||||
## Original Minitor
|
|
||||||
|
|
||||||
This is a reimplementation of [Minitor](https://git.iamthefij.com/iamthefij/minitor) in Go
|
|
||||||
|
|
||||||
Minitor is already a minimal monitoring tool. Python 3 was a quick way to get something live, but Python itself comes with a large footprint. Thus Go feels like a better fit for the project, longer term.
|
|
||||||
|
|
||||||
Initial target is meant to be roughly compatible requiring only minor changes to configuration. Future iterations may diverge to take advantage of Go specific features.
|
Initial target is meant to be roughly compatible requiring only minor changes to configuration. Future iterations may diverge to take advantage of Go specific features.
|
||||||
|
|
||||||
### Differences from Python version
|
## Differences from Python version
|
||||||
|
|
||||||
Templating for Alert messages has been updated. In the Python version, `str.format(...)` was used with certain keys passed in that could be used to format messages. In the Go version, we use a struct, `AlertNotice` defined in `alert.go` and the built in Go templating format. Eg.
|
There are a few key differences between the Python version and the v0.x Go version.
|
||||||
|
|
||||||
|
First, configuration keys cannot have multiple types in Go, so a different key must be used when specifying a Shell command as a string rather than a list of args. Instead of `command`, you must use `command_shell`. Eg:
|
||||||
|
|
||||||
|
minitor-py:
|
||||||
|
```yaml
|
||||||
|
monitors:
|
||||||
|
- name: Exec command
|
||||||
|
command: ['echo', 'test']
|
||||||
|
- name: Shell command
|
||||||
|
command: echo 'test'
|
||||||
|
```
|
||||||
|
|
||||||
|
minitor-go:
|
||||||
|
```yaml
|
||||||
|
monitors:
|
||||||
|
- name: Exec command
|
||||||
|
command: ['echo', 'test']
|
||||||
|
- name: Shell command
|
||||||
|
command_shell: echo 'test'
|
||||||
|
```
|
||||||
|
|
||||||
|
Second, templating for Alert messages has been updated. In the Python version, `str.format(...)` was used with certain keys passed in that could be used to format messages. In the Go version, we use a struct containing Alert info and the built in Go templating format. Eg.
|
||||||
|
|
||||||
minitor-py:
|
minitor-py:
|
||||||
```yaml
|
```yaml
|
||||||
alerts:
|
alerts:
|
||||||
log:
|
log_command:
|
||||||
command: 'echo {monitor_name}'
|
command: ['echo', '{monitor_name}']
|
||||||
|
log_shell:
|
||||||
|
command_shell: "echo {monitor_name}"
|
||||||
```
|
```
|
||||||
|
|
||||||
minitor-go:
|
minitor-go:
|
||||||
```yaml
|
```yaml
|
||||||
alerts:
|
alerts:
|
||||||
log:
|
log_command:
|
||||||
command: 'echo {{.MonitorName}}'
|
command: ['echo', '{{.MonitorName}}']
|
||||||
|
log_shell:
|
||||||
|
command_shell: "echo {{.MonitorName}}"
|
||||||
```
|
```
|
||||||
|
|
||||||
For the time being, legacy configs for the Python version of Minitor should be compatible if you apply the `-py-compat` flag when running Minitor. Eventually, this flag will go away when later breaking changes are introduced.
|
Finally, newlines in a shell command don't terminate a particular command. Semicolons must be used and continuations should not.
|
||||||
|
|
||||||
## Future
|
minitor-py:
|
||||||
|
```yaml
|
||||||
|
alerts:
|
||||||
|
log_shell:
|
||||||
|
command_shell: >
|
||||||
|
echo "line 1"
|
||||||
|
echo "line 2"
|
||||||
|
echo "continued" \
|
||||||
|
"line"
|
||||||
|
```
|
||||||
|
|
||||||
Future, potentially breaking changes
|
minitor-go:
|
||||||
|
```yaml
|
||||||
|
alerts:
|
||||||
|
log_shell:
|
||||||
|
command_shell: >
|
||||||
|
echo "line 1";
|
||||||
|
echo "line 2";
|
||||||
|
echo "continued"
|
||||||
|
"line"
|
||||||
|
```
|
||||||
|
|
||||||
|
## To do
|
||||||
|
There are two sets of task lists. The first is to get rough parity on key features with the Python version. The second is to make some improvements to the framework.
|
||||||
|
|
||||||
|
Pairity:
|
||||||
|
|
||||||
|
- [x] Run monitor commands
|
||||||
|
- [x] Run monitor commands in a shell
|
||||||
|
- [x] Run alert commands
|
||||||
|
- [x] Run alert commands in a shell
|
||||||
|
- [x] Allow templating of alert commands
|
||||||
|
- [ ] Implement Prometheus client to export metrics
|
||||||
|
- [ ] Test coverage
|
||||||
|
|
||||||
|
Improvement:
|
||||||
|
|
||||||
- [ ] Implement leveled logging (maybe glog or logrus)
|
- [ ] Implement leveled logging (maybe glog or logrus)
|
||||||
|
- [ ] Consider switching from YAML to TOML
|
||||||
- [ ] Consider value of templating vs injecting values into Env variables
|
- [ ] Consider value of templating vs injecting values into Env variables
|
||||||
|
- [ ] Consider dropping `alert_up` and `alert_down` in favor of using Go templates that offer more control of messaging
|
||||||
- [ ] Async checking
|
- [ ] Async checking
|
||||||
- [ ] Revisit metrics and see if they all make sense
|
- [ ] Use durations rather than seconds checked in event loop
|
||||||
- [ ] Consider dropping `alert_up` and `alert_down` in favor of using Go templates that offer more control of messaging (Breaking)
|
|
||||||
- [ ] Use durations rather than seconds checked in event loop (Potentially breaking)
|
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"strings"
|
|
||||||
"text/template"
|
"text/template"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -13,7 +12,8 @@ import (
|
|||||||
// Alert is a config driven mechanism for sending a notice
|
// Alert is a config driven mechanism for sending a notice
|
||||||
type Alert struct {
|
type Alert struct {
|
||||||
Name string
|
Name string
|
||||||
Command CommandOrShell
|
Command []string
|
||||||
|
CommandShell string `yaml:"command_shell"`
|
||||||
commandTemplate []*template.Template
|
commandTemplate []*template.Template
|
||||||
commandShellTemplate *template.Template
|
commandShellTemplate *template.Template
|
||||||
}
|
}
|
||||||
@@ -31,40 +31,26 @@ type AlertNotice struct {
|
|||||||
// IsValid returns a boolean indicating if the Alert has been correctly
|
// IsValid returns a boolean indicating if the Alert has been correctly
|
||||||
// configured
|
// configured
|
||||||
func (alert Alert) IsValid() bool {
|
func (alert Alert) IsValid() bool {
|
||||||
return !alert.Command.Empty()
|
atLeastOneCommand := (alert.CommandShell != "" || alert.Command != nil)
|
||||||
|
atMostOneCommand := (alert.CommandShell == "" || alert.Command == nil)
|
||||||
|
return atLeastOneCommand && atMostOneCommand
|
||||||
}
|
}
|
||||||
|
|
||||||
// BuildTemplates compiles command templates for the Alert
|
// BuildTemplates compiles command templates for the Alert
|
||||||
func (alert *Alert) BuildTemplates() error {
|
func (alert *Alert) BuildTemplates() error {
|
||||||
// TODO: Remove legacy template support later after 1.0
|
|
||||||
legacy := strings.NewReplacer(
|
|
||||||
"{alert_count}", "{{.AlertCount}}",
|
|
||||||
"{alert_message}", "{{.MonitorName}} check has failed {{.FailureCount}} times",
|
|
||||||
"{failure_count}", "{{.FailureCount}}",
|
|
||||||
"{last_output}", "{{.LastCheckOutput}}",
|
|
||||||
"{last_success}", "{{.LastSuccess}}",
|
|
||||||
"{monitor_name}", "{{.MonitorName}}",
|
|
||||||
)
|
|
||||||
if LogDebug {
|
if LogDebug {
|
||||||
log.Printf("DEBUG: Building template for alert %s", alert.Name)
|
log.Printf("DEBUG: Building template for alert %s", alert.Name)
|
||||||
}
|
}
|
||||||
if alert.commandTemplate == nil && alert.Command.Command != nil {
|
if alert.commandTemplate == nil && alert.Command != nil {
|
||||||
alert.commandTemplate = []*template.Template{}
|
alert.commandTemplate = []*template.Template{}
|
||||||
for i, cmdPart := range alert.Command.Command {
|
for i, cmdPart := range alert.Command {
|
||||||
if PyCompat {
|
|
||||||
cmdPart = legacy.Replace(cmdPart)
|
|
||||||
}
|
|
||||||
alert.commandTemplate = append(alert.commandTemplate, template.Must(
|
alert.commandTemplate = append(alert.commandTemplate, template.Must(
|
||||||
template.New(alert.Name+string(i)).Parse(cmdPart),
|
template.New(alert.Name+string(i)).Parse(cmdPart),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
} else if alert.commandShellTemplate == nil && alert.Command.ShellCommand != "" {
|
} else if alert.commandShellTemplate == nil && alert.CommandShell != "" {
|
||||||
shellCmd := alert.Command.ShellCommand
|
|
||||||
if PyCompat {
|
|
||||||
shellCmd = legacy.Replace(shellCmd)
|
|
||||||
}
|
|
||||||
alert.commandShellTemplate = template.Must(
|
alert.commandShellTemplate = template.Must(
|
||||||
template.New(alert.Name).Parse(shellCmd),
|
template.New(alert.Name).Parse(alert.CommandShell),
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
return fmt.Errorf("No template provided for alert %s", alert.Name)
|
return fmt.Errorf("No template provided for alert %s", alert.Name)
|
||||||
@@ -74,7 +60,7 @@ func (alert *Alert) BuildTemplates() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Send will send an alert notice by executing the command template
|
// Send will send an alert notice by executing the command template
|
||||||
func (alert Alert) Send(notice AlertNotice) (outputStr string, err error) {
|
func (alert Alert) Send(notice AlertNotice) (output_str string, err error) {
|
||||||
log.Printf("INFO: Sending alert %s for %s", alert.Name, notice.MonitorName)
|
log.Printf("INFO: Sending alert %s for %s", alert.Name, notice.MonitorName)
|
||||||
var cmd *exec.Cmd
|
var cmd *exec.Cmd
|
||||||
if alert.commandTemplate != nil {
|
if alert.commandTemplate != nil {
|
||||||
@@ -109,23 +95,10 @@ func (alert Alert) Send(notice AlertNotice) (outputStr string, err error) {
|
|||||||
|
|
||||||
var output []byte
|
var output []byte
|
||||||
output, err = cmd.CombinedOutput()
|
output, err = cmd.CombinedOutput()
|
||||||
outputStr = string(output)
|
output_str = string(output)
|
||||||
if LogDebug {
|
if LogDebug {
|
||||||
log.Printf("DEBUG: Alert output for: %s\n---\n%s\n---", alert.Name, outputStr)
|
log.Printf("DEBUG: Alert output for: %s\n---\n%s\n---", alert.Name, output_str)
|
||||||
}
|
}
|
||||||
|
|
||||||
return outputStr, err
|
return output_str, err
|
||||||
}
|
|
||||||
|
|
||||||
// NewLogAlert creates an alert that does basic logging using echo
|
|
||||||
func NewLogAlert() *Alert {
|
|
||||||
return &Alert{
|
|
||||||
Name: "log",
|
|
||||||
Command: CommandOrShell{
|
|
||||||
Command: []string{
|
|
||||||
"echo",
|
|
||||||
"{{.MonitorName}} check has failed {{.FailureCount}} times",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
+13
-25
@@ -11,9 +11,14 @@ func TestAlertIsValid(t *testing.T) {
|
|||||||
expected bool
|
expected bool
|
||||||
name string
|
name string
|
||||||
}{
|
}{
|
||||||
{Alert{Command: CommandOrShell{Command: []string{"echo", "test"}}}, true, "Command only"},
|
{Alert{Command: []string{"echo", "test"}}, true, "Command only"},
|
||||||
{Alert{Command: CommandOrShell{ShellCommand: "echo test"}}, true, "CommandShell only"},
|
{Alert{CommandShell: "echo test"}, true, "CommandShell only"},
|
||||||
{Alert{}, false, "No commands"},
|
{Alert{}, false, "No commands"},
|
||||||
|
{
|
||||||
|
Alert{Command: []string{"echo", "test"}, CommandShell: "echo test"},
|
||||||
|
false,
|
||||||
|
"Both commands",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
@@ -34,54 +39,39 @@ func TestAlertSend(t *testing.T) {
|
|||||||
expectedOutput string
|
expectedOutput string
|
||||||
expectErr bool
|
expectErr bool
|
||||||
name string
|
name string
|
||||||
pyCompat bool
|
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
Alert{Command: CommandOrShell{Command: []string{"echo", "{{.MonitorName}}"}}},
|
Alert{Command: []string{"echo", "{{.MonitorName}}"}},
|
||||||
AlertNotice{MonitorName: "test"},
|
AlertNotice{MonitorName: "test"},
|
||||||
"test\n",
|
"test\n",
|
||||||
false,
|
false,
|
||||||
"Command with template",
|
"Command with template",
|
||||||
false,
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
Alert{Command: CommandOrShell{ShellCommand: "echo {{.MonitorName}}"}},
|
Alert{CommandShell: "echo {{.MonitorName}}"},
|
||||||
AlertNotice{MonitorName: "test"},
|
AlertNotice{MonitorName: "test"},
|
||||||
"test\n",
|
"test\n",
|
||||||
false,
|
false,
|
||||||
"Command shell with template",
|
"Command shell with template",
|
||||||
false,
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
Alert{Command: CommandOrShell{Command: []string{"echo", "{{.Bad}}"}}},
|
Alert{Command: []string{"echo", "{{.Bad}}"}},
|
||||||
AlertNotice{MonitorName: "test"},
|
AlertNotice{MonitorName: "test"},
|
||||||
"",
|
"",
|
||||||
true,
|
true,
|
||||||
"Command with bad template",
|
"Command with bad template",
|
||||||
false,
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
Alert{Command: CommandOrShell{ShellCommand: "echo {{.Bad}}"}},
|
Alert{CommandShell: "echo {{.Bad}}"},
|
||||||
AlertNotice{MonitorName: "test"},
|
AlertNotice{MonitorName: "test"},
|
||||||
"",
|
"",
|
||||||
true,
|
true,
|
||||||
"Command shell with bad template",
|
"Command shell with bad template",
|
||||||
false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Alert{Command: CommandOrShell{ShellCommand: "echo {alert_message}"}},
|
|
||||||
AlertNotice{MonitorName: "test", FailureCount: 1},
|
|
||||||
"test check has failed 1 times\n",
|
|
||||||
false,
|
|
||||||
"Command shell with legacy template",
|
|
||||||
true,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
log.Printf("Testing case %s", c.name)
|
log.Printf("Testing case %s", c.name)
|
||||||
// Set PyCompat to value of compat flag
|
|
||||||
PyCompat = c.pyCompat
|
|
||||||
c.alert.BuildTemplates()
|
c.alert.BuildTemplates()
|
||||||
output, err := c.alert.Send(c.notice)
|
output, err := c.alert.Send(c.notice)
|
||||||
hasErr := (err != nil)
|
hasErr := (err != nil)
|
||||||
@@ -93,8 +83,6 @@ func TestAlertSend(t *testing.T) {
|
|||||||
t.Errorf("Send(%v err), expected=%v actual=%v", c.name, "Err", err)
|
t.Errorf("Send(%v err), expected=%v actual=%v", c.name, "Err", err)
|
||||||
log.Printf("Case failed: %s", c.name)
|
log.Printf("Case failed: %s", c.name)
|
||||||
}
|
}
|
||||||
// Set PyCompat back to default value
|
|
||||||
PyCompat = false
|
|
||||||
log.Println("-----")
|
log.Println("-----")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -115,8 +103,8 @@ func TestAlertBuildTemplate(t *testing.T) {
|
|||||||
expectErr bool
|
expectErr bool
|
||||||
name string
|
name string
|
||||||
}{
|
}{
|
||||||
{Alert{Command: CommandOrShell{Command: []string{"echo", "test"}}}, false, "Command only"},
|
{Alert{Command: []string{"echo", "test"}}, false, "Command only"},
|
||||||
{Alert{Command: CommandOrShell{ShellCommand: "echo test"}}, false, "CommandShell only"},
|
{Alert{CommandShell: "echo test"}, false, "CommandShell only"},
|
||||||
{Alert{}, true, "No commands"},
|
{Alert{}, true, "No commands"},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,10 +2,10 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
|
"gopkg.in/yaml.v2"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"log"
|
"log"
|
||||||
|
"os"
|
||||||
"gopkg.in/yaml.v2"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Config type is contains all provided user configuration
|
// Config type is contains all provided user configuration
|
||||||
@@ -15,54 +15,10 @@ type Config struct {
|
|||||||
Alerts map[string]*Alert
|
Alerts map[string]*Alert
|
||||||
}
|
}
|
||||||
|
|
||||||
// CommandOrShell type wraps a string or list of strings
|
|
||||||
// for executing a command directly or in a shell
|
|
||||||
type CommandOrShell struct {
|
|
||||||
ShellCommand string
|
|
||||||
Command []string
|
|
||||||
}
|
|
||||||
|
|
||||||
// Empty checks if the Command has a value
|
|
||||||
func (cos CommandOrShell) Empty() bool {
|
|
||||||
return (cos.ShellCommand == "" && cos.Command == nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
// UnmarshalYAML allows unmarshalling either a string or slice of strings
|
|
||||||
// and parsing them as either a command or a shell command.
|
|
||||||
func (cos *CommandOrShell) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
|
||||||
var cmd []string
|
|
||||||
err := unmarshal(&cmd)
|
|
||||||
// Error indicates this is shell command
|
|
||||||
if err != nil {
|
|
||||||
var shellCmd string
|
|
||||||
err := unmarshal(&shellCmd)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
cos.ShellCommand = shellCmd
|
|
||||||
} else {
|
|
||||||
cos.Command = cmd
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// IsValid checks config validity and returns true if valid
|
// IsValid checks config validity and returns true if valid
|
||||||
func (config Config) IsValid() (isValid bool) {
|
func (config Config) IsValid() (isValid bool) {
|
||||||
isValid = true
|
isValid = true
|
||||||
|
|
||||||
// Validate alerts
|
|
||||||
if config.Alerts == nil || len(config.Alerts) == 0 {
|
|
||||||
// This should never happen because there is a default alert named 'log' for now
|
|
||||||
log.Printf("ERROR: Invalid alert configuration: Must provide at least one alert")
|
|
||||||
isValid = false
|
|
||||||
}
|
|
||||||
for _, alert := range config.Alerts {
|
|
||||||
if !alert.IsValid() {
|
|
||||||
log.Printf("ERROR: Invalid alert configuration: %s", alert.Name)
|
|
||||||
isValid = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate monitors
|
// Validate monitors
|
||||||
if config.Monitors == nil || len(config.Monitors) == 0 {
|
if config.Monitors == nil || len(config.Monitors) == 0 {
|
||||||
log.Printf("ERROR: Invalid monitor configuration: Must provide at least one monitor")
|
log.Printf("ERROR: Invalid monitor configuration: Must provide at least one monitor")
|
||||||
@@ -87,6 +43,18 @@ func (config Config) IsValid() (isValid bool) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Validate alerts
|
||||||
|
if config.Alerts == nil || len(config.Alerts) == 0 {
|
||||||
|
log.Printf("ERROR: Invalid alert configuration: Must provide at least one alert")
|
||||||
|
isValid = false
|
||||||
|
}
|
||||||
|
for _, alert := range config.Alerts {
|
||||||
|
if !alert.IsValid() {
|
||||||
|
log.Printf("ERROR: Invalid alert configuration: %s", alert.Name)
|
||||||
|
isValid = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -109,25 +77,14 @@ func LoadConfig(filePath string) (config Config, err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
err = yaml.Unmarshal(data, &config)
|
// TODO: Decide if this is better expanded here, or only when executing
|
||||||
|
envExpanded := os.ExpandEnv(string(data))
|
||||||
|
err = yaml.Unmarshal([]byte(envExpanded), &config)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if LogDebug {
|
log.Printf("config:\n%v\n", config)
|
||||||
log.Printf("DEBUG: Config values:\n%v\n", config)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add log alert if not present
|
|
||||||
if PyCompat {
|
|
||||||
// Intialize alerts list if not present
|
|
||||||
if config.Alerts == nil {
|
|
||||||
config.Alerts = map[string]*Alert{}
|
|
||||||
}
|
|
||||||
if _, ok := config.Alerts["log"]; !ok {
|
|
||||||
config.Alerts["log"] = NewLogAlert()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !config.IsValid() {
|
if !config.IsValid() {
|
||||||
err = errors.New("Invalid configuration")
|
err = errors.New("Invalid configuration")
|
||||||
|
|||||||
+6
-78
@@ -10,94 +10,22 @@ func TestLoadConfig(t *testing.T) {
|
|||||||
configPath string
|
configPath string
|
||||||
expectErr bool
|
expectErr bool
|
||||||
name string
|
name string
|
||||||
pyCompat bool
|
|
||||||
}{
|
}{
|
||||||
{"./test/valid-config.yml", false, "Valid config file", false},
|
{"./test/valid-config.yml", false, "Valid config file"},
|
||||||
{"./test/valid-default-log-alert.yml", false, "Valid config file with default log alert PyCompat", true},
|
{"./test/does-not-exist", true, "Invalid config path"},
|
||||||
{"./test/valid-default-log-alert.yml", true, "Invalid config file no log alert", false},
|
{"./test/invalid-config-type.yml", true, "Invalid config type for key"},
|
||||||
{"./test/does-not-exist", true, "Invalid config path", false},
|
{"./test/invalid-config-missing-alerts.yml", true, "Invalid config missing alerts"},
|
||||||
{"./test/invalid-config-type.yml", true, "Invalid config type for key", false},
|
{"./test/invalid-config-unknown-alert.yml", true, "Invalid config unknown alert"},
|
||||||
{"./test/invalid-config-missing-alerts.yml", true, "Invalid config missing alerts", false},
|
|
||||||
{"./test/invalid-config-unknown-alert.yml", true, "Invalid config unknown alert", false},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
log.Printf("Testing case %s", c.name)
|
log.Printf("Testing case %s", c.name)
|
||||||
// Set PyCompat based on compatibility mode
|
|
||||||
PyCompat = c.pyCompat
|
|
||||||
_, err := LoadConfig(c.configPath)
|
_, err := LoadConfig(c.configPath)
|
||||||
hasErr := (err != nil)
|
hasErr := (err != nil)
|
||||||
if hasErr != c.expectErr {
|
if hasErr != c.expectErr {
|
||||||
t.Errorf("LoadConfig(%v), expected_error=%v actual=%v", c.name, c.expectErr, err)
|
t.Errorf("LoadConfig(%v), expected=%v actual=%v", c.name, "Err", err)
|
||||||
log.Printf("Case failed: %s", c.name)
|
log.Printf("Case failed: %s", c.name)
|
||||||
}
|
}
|
||||||
// Set PyCompat to default value
|
|
||||||
PyCompat = false
|
|
||||||
log.Println("-----")
|
log.Println("-----")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestMultiLineConfig is a more complicated test stepping through the parsing
|
|
||||||
// and execution of mutli-line strings presented in YAML
|
|
||||||
func TestMultiLineConfig(t *testing.T) {
|
|
||||||
log.Println("Testing multi-line string config")
|
|
||||||
config, err := LoadConfig("./test/valid-verify-multi-line.yml")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("TestMultiLineConfig(load), expected=no_error actual=%v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Println("-----")
|
|
||||||
log.Println("TestMultiLineConfig(parse > string)")
|
|
||||||
expected := "echo 'Some string with stuff'; echo \"<angle brackets>\"; exit 1\n"
|
|
||||||
actual := config.Monitors[0].Command.ShellCommand
|
|
||||||
if expected != actual {
|
|
||||||
t.Errorf("TestMultiLineConfig(>) failed")
|
|
||||||
t.Logf("string expected=`%v`", expected)
|
|
||||||
t.Logf("string actual =`%v`", actual)
|
|
||||||
t.Logf("bytes expected=%v", []byte(expected))
|
|
||||||
t.Logf("bytes actual =%v", []byte(actual))
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Println("-----")
|
|
||||||
log.Println("TestMultiLineConfig(execute > string)")
|
|
||||||
_, notice := config.Monitors[0].Check()
|
|
||||||
if notice == nil {
|
|
||||||
t.Fatalf("Did not receive an alert notice")
|
|
||||||
}
|
|
||||||
expected = "Some string with stuff\n<angle brackets>\n"
|
|
||||||
actual = notice.LastCheckOutput
|
|
||||||
if expected != actual {
|
|
||||||
t.Errorf("TestMultiLineConfig(execute > string) check failed")
|
|
||||||
t.Logf("string expected=`%v`", expected)
|
|
||||||
t.Logf("string actual =`%v`", actual)
|
|
||||||
t.Logf("bytes expected=%v", []byte(expected))
|
|
||||||
t.Logf("bytes actual =%v", []byte(actual))
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Println("-----")
|
|
||||||
log.Println("TestMultiLineConfig(parse | string)")
|
|
||||||
expected = "echo 'Some string with stuff'\necho '<angle brackets>'\n"
|
|
||||||
actual = config.Alerts["log_shell"].Command.ShellCommand
|
|
||||||
if expected != actual {
|
|
||||||
t.Errorf("TestMultiLineConfig(|) failed")
|
|
||||||
t.Logf("string expected=`%v`", expected)
|
|
||||||
t.Logf("string actual =`%v`", actual)
|
|
||||||
t.Logf("bytes expected=%v", []byte(expected))
|
|
||||||
t.Logf("bytes actual =%v", []byte(actual))
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Println("-----")
|
|
||||||
log.Println("TestMultiLineConfig(execute | string)")
|
|
||||||
actual, err = config.Alerts["log_shell"].Send(AlertNotice{})
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("Execution of alert failed")
|
|
||||||
}
|
|
||||||
expected = "Some string with stuff\n<angle brackets>\n"
|
|
||||||
if expected != actual {
|
|
||||||
t.Errorf("TestMultiLineConfig(execute | string) check failed")
|
|
||||||
t.Logf("string expected=`%v`", expected)
|
|
||||||
t.Logf("string actual =`%v`", actual)
|
|
||||||
t.Logf("bytes expected=%v", []byte(expected))
|
|
||||||
t.Logf("bytes actual =%v", []byte(actual))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -3,7 +3,9 @@ package main
|
|||||||
import (
|
import (
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||||
"log"
|
"log"
|
||||||
|
"net/http"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -13,13 +15,6 @@ var (
|
|||||||
|
|
||||||
// ExportMetrics will track whether or not we want to export metrics to prometheus
|
// ExportMetrics will track whether or not we want to export metrics to prometheus
|
||||||
ExportMetrics = false
|
ExportMetrics = false
|
||||||
// MetricsPort is the port to expose metrics on
|
|
||||||
MetricsPort = 8080
|
|
||||||
// Metrics contains all active metrics
|
|
||||||
Metrics = NewMetrics()
|
|
||||||
|
|
||||||
// PyCompat enables support for legacy Python templates
|
|
||||||
PyCompat = false
|
|
||||||
|
|
||||||
// version of minitor being run
|
// version of minitor being run
|
||||||
version = "dev"
|
version = "dev"
|
||||||
@@ -28,13 +23,7 @@ var (
|
|||||||
func checkMonitors(config *Config) error {
|
func checkMonitors(config *Config) error {
|
||||||
for _, monitor := range config.Monitors {
|
for _, monitor := range config.Monitors {
|
||||||
if monitor.ShouldCheck() {
|
if monitor.ShouldCheck() {
|
||||||
success, alertNotice := monitor.Check()
|
_, alertNotice := monitor.Check()
|
||||||
|
|
||||||
hasAlert := alertNotice != nil
|
|
||||||
|
|
||||||
// Track status metrics
|
|
||||||
Metrics.SetMonitorStatus(monitor.Name, success)
|
|
||||||
Metrics.CountCheck(monitor.Name, success, hasAlert)
|
|
||||||
|
|
||||||
// Should probably consider refactoring everything below here
|
// Should probably consider refactoring everything below here
|
||||||
if alertNotice != nil {
|
if alertNotice != nil {
|
||||||
@@ -66,9 +55,6 @@ func checkMonitors(config *Config) error {
|
|||||||
err,
|
err,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Count alert metrics
|
|
||||||
Metrics.CountAlert(monitor.Name, alert.Name)
|
|
||||||
} else {
|
} else {
|
||||||
// This case should never actually happen since we validate against it
|
// This case should never actually happen since we validate against it
|
||||||
log.Printf("ERROR: Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
|
log.Printf("ERROR: Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
|
||||||
@@ -82,14 +68,16 @@ func checkMonitors(config *Config) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func serveMetrics() {
|
||||||
|
http.Handle("/metrics", promhttp.Handler())
|
||||||
|
_ = http.ListenAndServe(":8080", nil)
|
||||||
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
// Get debug flag
|
// Get debug flag
|
||||||
flag.BoolVar(&LogDebug, "debug", false, "Enables debug logs (default: false)")
|
flag.BoolVar(&LogDebug, "debug", false, "Enables debug logs (default: false)")
|
||||||
flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)")
|
flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)")
|
||||||
flag.BoolVar(&PyCompat, "py-compat", false, "Enables support for legacy Python Minitor config. Will eventually be removed. (default: false)")
|
|
||||||
flag.IntVar(&MetricsPort, "metrics-port", 8080, "The port that Prometheus metrics should be exported on, if enabled. (default: 8080)")
|
|
||||||
var showVersion = flag.Bool("version", false, "Display the version of minitor and exit")
|
var showVersion = flag.Bool("version", false, "Display the version of minitor and exit")
|
||||||
var configPath = flag.String("config", "config.yml", "Alternate configuration path (default: config.yml)")
|
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
// Print version if flag is provided
|
// Print version if flag is provided
|
||||||
@@ -99,7 +87,7 @@ func main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Load configuration
|
// Load configuration
|
||||||
config, err := LoadConfig(*configPath)
|
config, err := LoadConfig("config.yml")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("Error loading config: %v", err)
|
log.Fatalf("Error loading config: %v", err)
|
||||||
}
|
}
|
||||||
@@ -107,7 +95,7 @@ func main() {
|
|||||||
// Serve metrics exporter, if specified
|
// Serve metrics exporter, if specified
|
||||||
if ExportMetrics {
|
if ExportMetrics {
|
||||||
log.Println("INFO: Exporting metrics to Prometheus")
|
log.Println("INFO: Exporting metrics to Prometheus")
|
||||||
go ServeMetrics()
|
go serveMetrics()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start main loop
|
// Start main loop
|
||||||
|
|||||||
+9
-9
@@ -18,7 +18,7 @@ func TestCheckMonitors(t *testing.T) {
|
|||||||
Monitors: []*Monitor{
|
Monitors: []*Monitor{
|
||||||
&Monitor{
|
&Monitor{
|
||||||
Name: "Success",
|
Name: "Success",
|
||||||
Command: CommandOrShell{Command: []string{"true"}},
|
Command: []string{"true"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -30,12 +30,12 @@ func TestCheckMonitors(t *testing.T) {
|
|||||||
Monitors: []*Monitor{
|
Monitors: []*Monitor{
|
||||||
&Monitor{
|
&Monitor{
|
||||||
Name: "Failure",
|
Name: "Failure",
|
||||||
Command: CommandOrShell{Command: []string{"false"}},
|
Command: []string{"false"},
|
||||||
AlertAfter: 1,
|
AlertAfter: 1,
|
||||||
},
|
},
|
||||||
&Monitor{
|
&Monitor{
|
||||||
Name: "Failure",
|
Name: "Failure",
|
||||||
Command: CommandOrShell{Command: []string{"false"}},
|
Command: []string{"false"},
|
||||||
AlertDown: []string{"unknown"},
|
AlertDown: []string{"unknown"},
|
||||||
AlertAfter: 1,
|
AlertAfter: 1,
|
||||||
},
|
},
|
||||||
@@ -49,12 +49,12 @@ func TestCheckMonitors(t *testing.T) {
|
|||||||
Monitors: []*Monitor{
|
Monitors: []*Monitor{
|
||||||
&Monitor{
|
&Monitor{
|
||||||
Name: "Success",
|
Name: "Success",
|
||||||
Command: CommandOrShell{Command: []string{"ls"}},
|
Command: []string{"ls"},
|
||||||
alertCount: 1,
|
alertCount: 1,
|
||||||
},
|
},
|
||||||
&Monitor{
|
&Monitor{
|
||||||
Name: "Success",
|
Name: "Success",
|
||||||
Command: CommandOrShell{Command: []string{"true"}},
|
Command: []string{"true"},
|
||||||
AlertUp: []string{"unknown"},
|
AlertUp: []string{"unknown"},
|
||||||
alertCount: 1,
|
alertCount: 1,
|
||||||
},
|
},
|
||||||
@@ -68,14 +68,14 @@ func TestCheckMonitors(t *testing.T) {
|
|||||||
Monitors: []*Monitor{
|
Monitors: []*Monitor{
|
||||||
&Monitor{
|
&Monitor{
|
||||||
Name: "Failure",
|
Name: "Failure",
|
||||||
Command: CommandOrShell{Command: []string{"false"}},
|
Command: []string{"false"},
|
||||||
AlertDown: []string{"good"},
|
AlertDown: []string{"good"},
|
||||||
AlertAfter: 1,
|
AlertAfter: 1,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
Alerts: map[string]*Alert{
|
Alerts: map[string]*Alert{
|
||||||
"good": &Alert{
|
"good": &Alert{
|
||||||
Command: CommandOrShell{Command: []string{"true"}},
|
Command: []string{"true"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -87,7 +87,7 @@ func TestCheckMonitors(t *testing.T) {
|
|||||||
Monitors: []*Monitor{
|
Monitors: []*Monitor{
|
||||||
&Monitor{
|
&Monitor{
|
||||||
Name: "Failure",
|
Name: "Failure",
|
||||||
Command: CommandOrShell{Command: []string{"false"}},
|
Command: []string{"false"},
|
||||||
AlertDown: []string{"bad"},
|
AlertDown: []string{"bad"},
|
||||||
AlertAfter: 1,
|
AlertAfter: 1,
|
||||||
},
|
},
|
||||||
@@ -95,7 +95,7 @@ func TestCheckMonitors(t *testing.T) {
|
|||||||
Alerts: map[string]*Alert{
|
Alerts: map[string]*Alert{
|
||||||
"bad": &Alert{
|
"bad": &Alert{
|
||||||
Name: "bad",
|
Name: "bad",
|
||||||
Command: CommandOrShell{Command: []string{"false"}},
|
Command: []string{"false"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -1,25 +0,0 @@
|
|||||||
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}
|
|
||||||
{{#if build.tags}}
|
|
||||||
tags:
|
|
||||||
{{#each build.tags}}
|
|
||||||
- {{this}}
|
|
||||||
{{/each}}
|
|
||||||
{{/if}}
|
|
||||||
manifests:
|
|
||||||
-
|
|
||||||
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64
|
|
||||||
platform:
|
|
||||||
architecture: amd64
|
|
||||||
os: linux
|
|
||||||
-
|
|
||||||
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64
|
|
||||||
platform:
|
|
||||||
architecture: arm64
|
|
||||||
os: linux
|
|
||||||
variant: v8
|
|
||||||
-
|
|
||||||
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm
|
|
||||||
platform:
|
|
||||||
architecture: arm
|
|
||||||
os: linux
|
|
||||||
variant: v7
|
|
||||||
-101
@@ -1,101 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
|
||||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TODO: Not sure if this is the best way to handle. A global instance for
|
|
||||||
// metrics isn't bad, but it might be nice to curry versions of the metrics
|
|
||||||
// for each monitor. Especially since every monitor has it's own. Perhaps
|
|
||||||
// another new function that essentially curries each metric for a given
|
|
||||||
// monitor name would do. This could be run when validating monitors and
|
|
||||||
// initializing alert templates.
|
|
||||||
|
|
||||||
// MinitorMetrics contains all counters and metrics that Minitor will need to access
|
|
||||||
type MinitorMetrics struct {
|
|
||||||
alertCount *prometheus.CounterVec
|
|
||||||
checkCount *prometheus.CounterVec
|
|
||||||
monitorStatus *prometheus.GaugeVec
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewMetrics creates and initializes all metrics
|
|
||||||
func NewMetrics() *MinitorMetrics {
|
|
||||||
// Initialize all metrics
|
|
||||||
metrics := &MinitorMetrics{
|
|
||||||
alertCount: prometheus.NewCounterVec(
|
|
||||||
prometheus.CounterOpts{
|
|
||||||
Name: "minitor_alert_total",
|
|
||||||
Help: "Number of Minitor alerts",
|
|
||||||
},
|
|
||||||
[]string{"alert", "monitor"},
|
|
||||||
),
|
|
||||||
checkCount: prometheus.NewCounterVec(
|
|
||||||
prometheus.CounterOpts{
|
|
||||||
Name: "minitor_check_total",
|
|
||||||
Help: "Number of Minitor checks",
|
|
||||||
},
|
|
||||||
[]string{"monitor", "status", "is_alert"},
|
|
||||||
),
|
|
||||||
monitorStatus: prometheus.NewGaugeVec(
|
|
||||||
prometheus.GaugeOpts{
|
|
||||||
Name: "minitor_monitor_up_count",
|
|
||||||
Help: "Status of currently responsive monitors",
|
|
||||||
},
|
|
||||||
[]string{"monitor"},
|
|
||||||
),
|
|
||||||
}
|
|
||||||
|
|
||||||
// Register newly created metrics
|
|
||||||
prometheus.MustRegister(metrics.alertCount)
|
|
||||||
prometheus.MustRegister(metrics.checkCount)
|
|
||||||
prometheus.MustRegister(metrics.monitorStatus)
|
|
||||||
|
|
||||||
return metrics
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetMonitorStatus sets the current status of Monitor
|
|
||||||
func (metrics *MinitorMetrics) SetMonitorStatus(monitor string, isUp bool) {
|
|
||||||
val := 0.0
|
|
||||||
if isUp {
|
|
||||||
val = 1.0
|
|
||||||
}
|
|
||||||
metrics.monitorStatus.With(prometheus.Labels{"monitor": monitor}).Set(val)
|
|
||||||
}
|
|
||||||
|
|
||||||
// CountCheck counts the result of a particular Monitor check
|
|
||||||
func (metrics *MinitorMetrics) CountCheck(monitor string, isSuccess bool, isAlert bool) {
|
|
||||||
status := "failure"
|
|
||||||
if isSuccess {
|
|
||||||
status = "success"
|
|
||||||
}
|
|
||||||
|
|
||||||
alertVal := "false"
|
|
||||||
if isAlert {
|
|
||||||
alertVal = "true"
|
|
||||||
}
|
|
||||||
|
|
||||||
metrics.checkCount.With(
|
|
||||||
prometheus.Labels{"monitor": monitor, "status": status, "is_alert": alertVal},
|
|
||||||
).Inc()
|
|
||||||
}
|
|
||||||
|
|
||||||
// CountAlert counts an alert
|
|
||||||
func (metrics *MinitorMetrics) CountAlert(monitor string, alert string) {
|
|
||||||
metrics.alertCount.With(
|
|
||||||
prometheus.Labels{
|
|
||||||
"alert": alert,
|
|
||||||
"monitor": monitor,
|
|
||||||
},
|
|
||||||
).Inc()
|
|
||||||
}
|
|
||||||
|
|
||||||
// ServeMetrics starts an http server with a Prometheus metrics handler
|
|
||||||
func ServeMetrics() {
|
|
||||||
http.Handle("/metrics", promhttp.Handler())
|
|
||||||
host := fmt.Sprintf(":%d", MetricsPort)
|
|
||||||
_ = http.ListenAndServe(host, nil)
|
|
||||||
}
|
|
||||||
+13
-7
@@ -11,7 +11,8 @@ import (
|
|||||||
type Monitor struct {
|
type Monitor struct {
|
||||||
// Config values
|
// Config values
|
||||||
Name string
|
Name string
|
||||||
Command CommandOrShell
|
Command []string
|
||||||
|
CommandShell string `yaml:"command_shell"`
|
||||||
AlertDown []string `yaml:"alert_down"`
|
AlertDown []string `yaml:"alert_down"`
|
||||||
AlertUp []string `yaml:"alert_up"`
|
AlertUp []string `yaml:"alert_up"`
|
||||||
CheckInterval float64 `yaml:"check_interval"`
|
CheckInterval float64 `yaml:"check_interval"`
|
||||||
@@ -28,7 +29,10 @@ type Monitor struct {
|
|||||||
// IsValid returns a boolean indicating if the Monitor has been correctly
|
// IsValid returns a boolean indicating if the Monitor has been correctly
|
||||||
// configured
|
// configured
|
||||||
func (monitor Monitor) IsValid() bool {
|
func (monitor Monitor) IsValid() bool {
|
||||||
return (!monitor.Command.Empty() &&
|
atLeastOneCommand := (monitor.CommandShell != "" || monitor.Command != nil)
|
||||||
|
atMostOneCommand := (monitor.CommandShell == "" || monitor.Command == nil)
|
||||||
|
return (atLeastOneCommand &&
|
||||||
|
atMostOneCommand &&
|
||||||
monitor.getAlertAfter() > 0 &&
|
monitor.getAlertAfter() > 0 &&
|
||||||
monitor.AlertDown != nil)
|
monitor.AlertDown != nil)
|
||||||
}
|
}
|
||||||
@@ -48,10 +52,10 @@ func (monitor Monitor) ShouldCheck() bool {
|
|||||||
// and a possible AlertNotice
|
// and a possible AlertNotice
|
||||||
func (monitor *Monitor) Check() (bool, *AlertNotice) {
|
func (monitor *Monitor) Check() (bool, *AlertNotice) {
|
||||||
var cmd *exec.Cmd
|
var cmd *exec.Cmd
|
||||||
if monitor.Command.Command != nil {
|
if monitor.Command != nil {
|
||||||
cmd = exec.Command(monitor.Command.Command[0], monitor.Command.Command[1:]...)
|
cmd = exec.Command(monitor.Command[0], monitor.Command[1:]...)
|
||||||
} else {
|
} else {
|
||||||
cmd = ShellCommand(monitor.Command.ShellCommand)
|
cmd = ShellCommand(monitor.CommandShell)
|
||||||
}
|
}
|
||||||
|
|
||||||
output, err := cmd.CombinedOutput()
|
output, err := cmd.CombinedOutput()
|
||||||
@@ -151,16 +155,18 @@ func (monitor Monitor) getAlertAfter() int16 {
|
|||||||
// Zero is one!
|
// Zero is one!
|
||||||
if monitor.AlertAfter == 0 {
|
if monitor.AlertAfter == 0 {
|
||||||
return 1
|
return 1
|
||||||
|
} else {
|
||||||
|
return monitor.AlertAfter
|
||||||
}
|
}
|
||||||
return monitor.AlertAfter
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetAlertNames gives a list of alert names for a given monitor status
|
// GetAlertNames gives a list of alert names for a given monitor status
|
||||||
func (monitor Monitor) GetAlertNames(up bool) []string {
|
func (monitor Monitor) GetAlertNames(up bool) []string {
|
||||||
if up {
|
if up {
|
||||||
return monitor.AlertUp
|
return monitor.AlertUp
|
||||||
|
} else {
|
||||||
|
return monitor.AlertDown
|
||||||
}
|
}
|
||||||
return monitor.AlertDown
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (monitor Monitor) createAlertNotice(isUp bool) *AlertNotice {
|
func (monitor Monitor) createAlertNotice(isUp bool) *AlertNotice {
|
||||||
|
|||||||
+13
-8
@@ -13,11 +13,16 @@ func TestMonitorIsValid(t *testing.T) {
|
|||||||
expected bool
|
expected bool
|
||||||
name string
|
name string
|
||||||
}{
|
}{
|
||||||
{Monitor{Command: CommandOrShell{Command: []string{"echo", "test"}}, AlertDown: []string{"log"}}, true, "Command only"},
|
{Monitor{Command: []string{"echo", "test"}, AlertDown: []string{"log"}}, true, "Command only"},
|
||||||
{Monitor{Command: CommandOrShell{ShellCommand: "echo test"}, AlertDown: []string{"log"}}, true, "CommandShell only"},
|
{Monitor{CommandShell: "echo test", AlertDown: []string{"log"}}, true, "CommandShell only"},
|
||||||
{Monitor{Command: CommandOrShell{Command: []string{"echo", "test"}}}, false, "No AlertDown"},
|
{Monitor{Command: []string{"echo", "test"}}, false, "No AlertDown"},
|
||||||
{Monitor{AlertDown: []string{"log"}}, false, "No commands"},
|
{Monitor{AlertDown: []string{"log"}}, false, "No commands"},
|
||||||
{Monitor{Command: CommandOrShell{Command: []string{"echo", "test"}}, AlertDown: []string{"log"}, AlertAfter: -1}, false, "Invalid alert threshold, -1"},
|
{
|
||||||
|
Monitor{Command: []string{"echo", "test"}, CommandShell: "echo test", AlertDown: []string{"log"}},
|
||||||
|
false,
|
||||||
|
"Both commands",
|
||||||
|
},
|
||||||
|
{Monitor{Command: []string{"echo", "test"}, AlertDown: []string{"log"}, AlertAfter: -1}, false, "Invalid alert threshold, -1"},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
@@ -249,22 +254,22 @@ func TestMonitorCheck(t *testing.T) {
|
|||||||
name string
|
name string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
Monitor{Command: CommandOrShell{Command: []string{"echo", "success"}}},
|
Monitor{Command: []string{"echo", "success"}},
|
||||||
expected{isSuccess: true, hasNotice: false, lastOutput: "success\n"},
|
expected{isSuccess: true, hasNotice: false, lastOutput: "success\n"},
|
||||||
"Test successful command",
|
"Test successful command",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
Monitor{Command: CommandOrShell{ShellCommand: "echo success"}},
|
Monitor{CommandShell: "echo success"},
|
||||||
expected{isSuccess: true, hasNotice: false, lastOutput: "success\n"},
|
expected{isSuccess: true, hasNotice: false, lastOutput: "success\n"},
|
||||||
"Test successful command shell",
|
"Test successful command shell",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
Monitor{Command: CommandOrShell{Command: []string{"total", "failure"}}},
|
Monitor{Command: []string{"total", "failure"}},
|
||||||
expected{isSuccess: false, hasNotice: true, lastOutput: ""},
|
expected{isSuccess: false, hasNotice: true, lastOutput: ""},
|
||||||
"Test failed command",
|
"Test failed command",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
Monitor{Command: CommandOrShell{ShellCommand: "false"}},
|
Monitor{CommandShell: "false"},
|
||||||
expected{isSuccess: false, hasNotice: true, lastOutput: ""},
|
expected{isSuccess: false, hasNotice: true, lastOutput: ""},
|
||||||
"Test failed command shell",
|
"Test failed command shell",
|
||||||
},
|
},
|
||||||
|
|||||||
+10
-22
@@ -1,33 +1,21 @@
|
|||||||
---
|
check_interval: 30
|
||||||
check_interval: 5
|
|
||||||
|
|
||||||
monitors:
|
monitors:
|
||||||
- name: Fake Website
|
- name: My Website
|
||||||
command: ['curl', '-s', '-o', '/dev/null', 'https://minitor.mon']
|
command: [ 'curl', '-s', '-o', '/dev/null', 'https://minitor.mon' ]
|
||||||
alert_down: [log_down, mailgun_down, sms_down]
|
alert_down: [ log, mailgun_down, sms_down ]
|
||||||
alert_up: [log_up, email_up]
|
alert_up: [ log, email_up ]
|
||||||
check_interval: 10 # Must be at minimum the global `check_interval`
|
check_interval: 30 # Must be at minimum the global `check_interval`
|
||||||
alert_after: 3
|
alert_after: 3
|
||||||
alert_every: -1 # Defaults to -1 for exponential backoff. 0 to disable repeating
|
alert_every: -1 # Defaults to -1 for exponential backoff. 0 to disable repeating
|
||||||
- name: Real Website
|
|
||||||
command: ['curl', '-s', '-o', '/dev/null', 'https://google.com']
|
|
||||||
alert_down: [log_down, mailgun_down, sms_down]
|
|
||||||
alert_up: [log_up, email_up]
|
|
||||||
check_interval: 5
|
|
||||||
alert_after: 3
|
|
||||||
alert_every: -1
|
|
||||||
|
|
||||||
alerts:
|
alerts:
|
||||||
log_down:
|
|
||||||
command: ["echo", "Minitor failure for {{.MonitorName}}"]
|
|
||||||
log_up:
|
|
||||||
command: ["echo", "Minitor recovery for {{.MonitorName}}"]
|
|
||||||
email_up:
|
email_up:
|
||||||
command: [sendmail, "me@minitor.mon", "Recovered: {monitor_name}", "We're back!"]
|
command: [ sendmail, "me@minitor.mon", "Recovered: {monitor_name}", "We're back!" ]
|
||||||
mailgun_down:
|
mailgun_down:
|
||||||
command: >
|
command: >
|
||||||
curl -s -X POST
|
curl -s -X POST
|
||||||
-F subject="Alert! {{.MonitorName}} failed"
|
-F subject="Alert! {monitor_name} failed"
|
||||||
-F from="Minitor <minitor@minitor.mon>"
|
-F from="Minitor <minitor@minitor.mon>"
|
||||||
-F to=me@minitor.mon
|
-F to=me@minitor.mon
|
||||||
-F text="Our monitor failed"
|
-F text="Our monitor failed"
|
||||||
@@ -35,7 +23,7 @@ alerts:
|
|||||||
-u "api:${MAILGUN_API_KEY}"
|
-u "api:${MAILGUN_API_KEY}"
|
||||||
sms_down:
|
sms_down:
|
||||||
command: >
|
command: >
|
||||||
curl -s -X POST -F "Body=Failure! {{.MonitorName}} has failed"
|
curl -s -X POST -F "Body=Failure! {monitor_name} has failed"
|
||||||
-F "From=${AVAILABLE_NUMBER}" -F "To=${MY_PHONE}"
|
-F "From=${AVAILABLE_NUMBER}" -F "To=${MY_PHONE}"
|
||||||
"https://api.twilio.com/2010-04-01/Accounts/${ACCOUNT_SID}/Messages"
|
"https://api.twilio.com/2010-04-01/Accounts/${ACCOUNT_SID}/Messages"
|
||||||
-u "${ACCOUNT_SID}:${AUTH_TOKEN}"
|
-u "${ACCOUNT_SID}:${AUTH_TOKEN}"
|
||||||
|
|||||||
@@ -1,5 +0,0 @@
|
|||||||
# Minitor Scripts
|
|
||||||
|
|
||||||
A collection of some handy scripts to use with Minitor
|
|
||||||
|
|
||||||
These are not included with the Python package, but they are included in the Docker image in `/app/scripts`.
|
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
#! /bin/bash
|
|
||||||
set -e
|
|
||||||
|
|
||||||
#################
|
|
||||||
# docker_check.sh
|
|
||||||
#
|
|
||||||
# Checks the most recent state exit code of a Docker container
|
|
||||||
#################
|
|
||||||
|
|
||||||
# Docker host will default to a socket
|
|
||||||
# To override, export DOCKER_HOST to a new hostname
|
|
||||||
DOCKER_HOST="${DOCKER_HOST:=socket}"
|
|
||||||
container_name="$1"
|
|
||||||
|
|
||||||
# Curls Docker either using a socket or URL
|
|
||||||
function curl_docker {
|
|
||||||
local path="$1"
|
|
||||||
if [ "$DOCKER_HOST" == "socket" ]; then
|
|
||||||
curl --unix-socket /var/run/docker.sock "http://localhost/$path" 2>/dev/null
|
|
||||||
else
|
|
||||||
curl "http://${DOCKER_HOST}/$path" 2>/dev/null
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# Returns caintainer ID for a given container name
|
|
||||||
function get_container_id {
|
|
||||||
local container_name="$1"
|
|
||||||
curl_docker 'containers/json?all=1' \
|
|
||||||
| jq -r ".[] | {Id, Name: .Names[]} | select(.Name == \"/${container_name}\") | .Id"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Returns container JSON
|
|
||||||
function inspect_container {
|
|
||||||
local container_id=$1
|
|
||||||
curl_docker "containers/$container_id/json"
|
|
||||||
}
|
|
||||||
|
|
||||||
if [ -z "$container_name" ]; then
|
|
||||||
echo "Usage: $0 container_name"
|
|
||||||
echo "Will exit with the last status code of continer with provided name"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
container_id=$(get_container_id $container_name)
|
|
||||||
if [ -z "$container_id" ]; then
|
|
||||||
echo "ERROR: Could not find container with name: $container_name"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
exit_code=$(inspect_container "$container_id" | jq -r .State.ExitCode)
|
|
||||||
|
|
||||||
exit "$exit_code"
|
|
||||||
@@ -1,61 +0,0 @@
|
|||||||
#! /bin/bash
|
|
||||||
set -e
|
|
||||||
|
|
||||||
#################
|
|
||||||
# docker_healthcheck.sh
|
|
||||||
#
|
|
||||||
# Returns the results of a Docker Healthcheck for a container
|
|
||||||
#################
|
|
||||||
|
|
||||||
# Docker host will default to a socket
|
|
||||||
# To override, export DOCKER_HOST to a new hostname
|
|
||||||
DOCKER_HOST="${DOCKER_HOST:=socket}"
|
|
||||||
container_name="$1"
|
|
||||||
|
|
||||||
# Curls Docker either using a socket or URL
|
|
||||||
function curl_docker {
|
|
||||||
local path="$1"
|
|
||||||
if [ "$DOCKER_HOST" == "socket" ]; then
|
|
||||||
curl --unix-socket /var/run/docker.sock "http://localhost/$path" 2>/dev/null
|
|
||||||
else
|
|
||||||
curl "http://${DOCKER_HOST}/$path" 2>/dev/null
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# Returns caintainer ID for a given container name
|
|
||||||
function get_container_id {
|
|
||||||
local container_name="$1"
|
|
||||||
curl_docker 'containers/json?all=1' \
|
|
||||||
| jq -r ".[] | {Id, Name: .Names[]} | select(.Name == \"/${container_name}\") | .Id"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Returns container JSON
|
|
||||||
function inspect_container {
|
|
||||||
local container_id="$1"
|
|
||||||
curl_docker "containers/$container_id/json"
|
|
||||||
}
|
|
||||||
|
|
||||||
if [ -z "$container_name" ]; then
|
|
||||||
echo "Usage: $0 container_name"
|
|
||||||
echo "Will return results of healthcheck for continer with provided name"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
container_id=$(get_container_id "$container_name")
|
|
||||||
if [ -z "$container_id" ]; then
|
|
||||||
echo "ERROR: Could not find container with name: $container_name"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
health=$(inspect_container "$container_id" | jq -r '.State.Health.Status')
|
|
||||||
|
|
||||||
case "$health" in
|
|
||||||
null)
|
|
||||||
echo "No healthcheck results"
|
|
||||||
;;
|
|
||||||
starting|healthy)
|
|
||||||
echo "Status: '$health'"
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
echo "Status: '$health'"
|
|
||||||
exit 1
|
|
||||||
esac
|
|
||||||
@@ -6,3 +6,4 @@ monitors:
|
|||||||
alert_down: [ 'alert_down', 'log_shell', 'log_command' ]
|
alert_down: [ 'alert_down', 'log_shell', 'log_command' ]
|
||||||
# alert_every: -1
|
# alert_every: -1
|
||||||
alert_every: 0
|
alert_every: 0
|
||||||
|
|
||||||
|
|||||||
@@ -1,23 +1,22 @@
|
|||||||
---
|
|
||||||
check_interval: 1
|
check_interval: 1
|
||||||
|
|
||||||
monitors:
|
monitors:
|
||||||
- name: Command
|
- name: Command
|
||||||
command: ['echo', '$PATH']
|
command: ['echo', '$PATH']
|
||||||
alert_down: ['log_command', 'log_shell']
|
alert_down: [ 'log_command', 'log_shell' ]
|
||||||
alert_every: 0
|
alert_every: 0
|
||||||
- name: Shell
|
- name: Shell
|
||||||
command: >
|
command_shell: >
|
||||||
echo 'Some string with stuff';
|
echo 'Some string with stuff';
|
||||||
echo 'another line';
|
echo 'another line';
|
||||||
echo $PATH;
|
echo $PATH;
|
||||||
exit 1
|
exit 1
|
||||||
alert_down: ['log_command', 'log_shell']
|
alert_down: [ 'log_command', 'log_shell' ]
|
||||||
alert_after: 5
|
alert_after: 5
|
||||||
alert_every: 0
|
alert_every: 0
|
||||||
|
|
||||||
alerts:
|
alerts:
|
||||||
log_command:
|
log_command:
|
||||||
command: ['echo', 'regular', '"command!!!"', "{{.MonitorName}}"]
|
command: [ 'echo', 'regular', '"command!!!"', "{{.MonitorName}}" ]
|
||||||
log_shell:
|
log_shell:
|
||||||
command: echo "Failure on {{.MonitorName}} User is $USER"
|
command_shell: echo "Failure on {{.MonitorName}} User is $USER"
|
||||||
|
|||||||
@@ -1,8 +0,0 @@
|
|||||||
---
|
|
||||||
check_interval: 1
|
|
||||||
|
|
||||||
monitors:
|
|
||||||
- name: Command
|
|
||||||
command: ['echo', '$PATH']
|
|
||||||
alert_down: ['log']
|
|
||||||
alert_every: 0
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
---
|
|
||||||
check_interval: 1
|
|
||||||
|
|
||||||
monitors:
|
|
||||||
- name: Shell
|
|
||||||
command: >
|
|
||||||
echo 'Some string with stuff';
|
|
||||||
echo "<angle brackets>";
|
|
||||||
exit 1
|
|
||||||
alert_down: ['log_shell']
|
|
||||||
alert_after: 1
|
|
||||||
alert_every: 0
|
|
||||||
|
|
||||||
alerts:
|
|
||||||
log_shell:
|
|
||||||
command: |
|
|
||||||
echo 'Some string with stuff'
|
|
||||||
echo '<angle brackets>'
|
|
||||||
@@ -5,9 +5,19 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// escapeCommandShell accepts a command to be executed by a shell and escapes it
|
||||||
|
func escapeCommandShell(command string) string {
|
||||||
|
// Remove extra spaces and newlines from ends
|
||||||
|
command = strings.TrimSpace(command)
|
||||||
|
// TODO: Not sure if this part is actually needed. Should verify
|
||||||
|
// Escape double quotes since this will be passed in as an argument
|
||||||
|
command = strings.Replace(command, `"`, `\"`, -1)
|
||||||
|
return command
|
||||||
|
}
|
||||||
|
|
||||||
// ShellCommand takes a string and executes it as a command using `sh`
|
// ShellCommand takes a string and executes it as a command using `sh`
|
||||||
func ShellCommand(command string) *exec.Cmd {
|
func ShellCommand(command string) *exec.Cmd {
|
||||||
shellCommand := []string{"sh", "-c", strings.TrimSpace(command)}
|
shellCommand := []string{"sh", "-c", escapeCommandShell(command)}
|
||||||
//log.Printf("Shell command: %v", shellCommand)
|
//log.Printf("Shell command: %v", shellCommand)
|
||||||
return exec.Command(shellCommand[0], shellCommand[1:]...)
|
return exec.Command(shellCommand[0], shellCommand[1:]...)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user