Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f66276c24f |
+24
-65
@@ -3,15 +3,31 @@ kind: pipeline
|
|||||||
name: test
|
name: test
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
|
|
||||||
- name: test
|
- name: test
|
||||||
image: golang:1.21
|
image: golang:1.12
|
||||||
environment:
|
|
||||||
VERSION: ${DRONE_TAG:-${DRONE_COMMIT}}
|
|
||||||
commands:
|
commands:
|
||||||
|
- make build
|
||||||
- make test
|
- make test
|
||||||
|
|
||||||
- name: check
|
- name: check
|
||||||
image: iamthefij/drone-pre-commit:personal
|
image: python:3
|
||||||
|
commands:
|
||||||
|
- pip install pre-commit==1.20.0
|
||||||
|
- make check
|
||||||
|
|
||||||
|
- name: notify
|
||||||
|
image: drillster/drone-email
|
||||||
|
settings:
|
||||||
|
host:
|
||||||
|
from_secret: SMTP_HOST
|
||||||
|
username:
|
||||||
|
from_secret: SMTP_USER
|
||||||
|
password:
|
||||||
|
from_secret: SMTP_PASS
|
||||||
|
from: drone@iamthefij.com
|
||||||
|
when:
|
||||||
|
status: [changed, failure]
|
||||||
|
|
||||||
---
|
---
|
||||||
kind: pipeline
|
kind: pipeline
|
||||||
@@ -29,72 +45,15 @@ trigger:
|
|||||||
- refs/tags/v*
|
- refs/tags/v*
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: build all binaries
|
|
||||||
image: golang:1.21
|
|
||||||
environment:
|
|
||||||
VERSION: ${DRONE_TAG:-${DRONE_COMMIT}}
|
|
||||||
commands:
|
|
||||||
- make all
|
|
||||||
|
|
||||||
- name: compress binaries for release
|
# Might consider moving this step into the previous pipeline
|
||||||
image: ubuntu
|
- name: push image
|
||||||
commands:
|
image: plugins/docker
|
||||||
- find ./dist -type f -executable -execdir tar -czvf {}.tar.gz {} \;
|
|
||||||
when:
|
|
||||||
event: tag
|
|
||||||
|
|
||||||
- name: upload gitea release
|
|
||||||
image: plugins/gitea-release
|
|
||||||
settings:
|
|
||||||
title: ${DRONE_TAG}
|
|
||||||
files: dist/*.tar.gz
|
|
||||||
checksum:
|
|
||||||
- md5
|
|
||||||
- sha1
|
|
||||||
- sha256
|
|
||||||
- sha512
|
|
||||||
base_url:
|
|
||||||
from_secret: gitea_base_url
|
|
||||||
api_key:
|
|
||||||
from_secret: gitea_token
|
|
||||||
when:
|
|
||||||
event: tag
|
|
||||||
|
|
||||||
- name: Build and publish docker images
|
|
||||||
image: thegeeklab/drone-docker-buildx
|
|
||||||
settings:
|
settings:
|
||||||
repo: iamthefij/minitor-go
|
repo: iamthefij/minitor-go
|
||||||
|
dockerfile: Dockerfile.multi-stage
|
||||||
auto_tag: true
|
auto_tag: true
|
||||||
platforms:
|
|
||||||
- linux/amd64
|
|
||||||
- linux/arm64
|
|
||||||
- linux/arm
|
|
||||||
username:
|
username:
|
||||||
from_secret: docker_username
|
from_secret: docker_username
|
||||||
password:
|
password:
|
||||||
from_secret: docker_password
|
from_secret: docker_password
|
||||||
|
|
||||||
---
|
|
||||||
kind: pipeline
|
|
||||||
name: notify
|
|
||||||
|
|
||||||
depends_on:
|
|
||||||
- test
|
|
||||||
- publish
|
|
||||||
|
|
||||||
trigger:
|
|
||||||
status:
|
|
||||||
- failure
|
|
||||||
|
|
||||||
steps:
|
|
||||||
|
|
||||||
- name: notify
|
|
||||||
image: drillster/drone-email
|
|
||||||
settings:
|
|
||||||
host:
|
|
||||||
from_secret: SMTP_HOST # pragma: whitelist secret
|
|
||||||
username:
|
|
||||||
from_secret: SMTP_USER # pragma: whitelist secret
|
|
||||||
password:
|
|
||||||
from_secret: SMTP_PASS # pragma: whitelist secret
|
|
||||||
from: drone@iamthefij.com
|
|
||||||
|
|||||||
@@ -1,111 +0,0 @@
|
|||||||
name: ci
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
tags:
|
|
||||||
- "v*"
|
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
tests:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v6
|
|
||||||
|
|
||||||
- name: Set up Go
|
|
||||||
uses: actions/setup-go@v6
|
|
||||||
with:
|
|
||||||
go-version-file: go.mod
|
|
||||||
|
|
||||||
- name: Run tests
|
|
||||||
run: make test
|
|
||||||
|
|
||||||
lint:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v6
|
|
||||||
|
|
||||||
- name: Set up Go
|
|
||||||
uses: actions/setup-go@v6
|
|
||||||
with:
|
|
||||||
go-version-file: go.mod
|
|
||||||
|
|
||||||
- name: Set up Python
|
|
||||||
uses: actions/setup-python@v6
|
|
||||||
|
|
||||||
- name: Run pre-commit
|
|
||||||
uses: https://git.iamthefij.com/iamthefij/pre-commit-action@v3.0.2
|
|
||||||
|
|
||||||
release:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: test
|
|
||||||
if: "${{ github.event_name != 'pull_request' }}"
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v6
|
|
||||||
|
|
||||||
- name: Set up Go
|
|
||||||
uses: actions/setup-go@v6
|
|
||||||
with:
|
|
||||||
go-version-file: go.mod
|
|
||||||
|
|
||||||
- name: Build binaries
|
|
||||||
env:
|
|
||||||
VERSION: "${{ github.REF_NAME }}"
|
|
||||||
run: make all
|
|
||||||
|
|
||||||
# Package binaries and create release if this is a tagged build
|
|
||||||
- name: Compress binaries
|
|
||||||
if: "${{ github.ref_type == 'tag' }}"
|
|
||||||
run: find ./dist -type f -executable -execdir tar -czvf {}.tar.gz {} \;
|
|
||||||
|
|
||||||
- name: Upload release
|
|
||||||
uses: https://gitea.com/actions/gitea-release-action@v1
|
|
||||||
if: "${{ github.ref_type == 'tag' }}"
|
|
||||||
with:
|
|
||||||
files: |-
|
|
||||||
dist/*.tar.gz
|
|
||||||
md5sum: true
|
|
||||||
sha256sum: true
|
|
||||||
|
|
||||||
- name: Docker meta
|
|
||||||
id: meta
|
|
||||||
uses: docker/metadata-action@v5
|
|
||||||
with:
|
|
||||||
# list of Docker images to use as base name for tags
|
|
||||||
images: |
|
|
||||||
${{ github.REPOSITORY }}
|
|
||||||
# generate Docker tags based on the following events/attributes
|
|
||||||
tags: |
|
|
||||||
type=ref,event=branch
|
|
||||||
type=ref,event=pr
|
|
||||||
type=semver,pattern={{version}}
|
|
||||||
type=semver,pattern={{major}}.{{minor}}
|
|
||||||
type=semver,pattern={{major}}
|
|
||||||
|
|
||||||
- name: Login to Docker Hub
|
|
||||||
uses: docker/login-action@v3
|
|
||||||
if: "${{ github.event_name != 'pull_request' }}"
|
|
||||||
with:
|
|
||||||
username: ${{ secrets.DOCKER_USERNAME }}
|
|
||||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
|
||||||
|
|
||||||
- name: Set up QEMU
|
|
||||||
uses: docker/setup-qemu-action@v3
|
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
|
||||||
uses: docker/setup-buildx-action@v3
|
|
||||||
|
|
||||||
- name: Build and push
|
|
||||||
uses: docker/build-push-action@v6
|
|
||||||
with:
|
|
||||||
# Use path context so we can access pre-compiled binaries
|
|
||||||
context: .
|
|
||||||
push: ${{ github.event_name != 'pull_request' }}
|
|
||||||
platforms: |
|
|
||||||
linux/amd64
|
|
||||||
linux/arm64
|
|
||||||
tags: ${{ steps.meta.outputs.tags }}
|
|
||||||
labels: ${{ steps.meta.outputs.labels }}
|
|
||||||
Vendored
-3
@@ -14,9 +14,6 @@
|
|||||||
|
|
||||||
# User configuration
|
# User configuration
|
||||||
config.yml
|
config.yml
|
||||||
config.hcl
|
|
||||||
|
|
||||||
# Output binary
|
# Output binary
|
||||||
minitor
|
|
||||||
minitor-go
|
minitor-go
|
||||||
dist/
|
|
||||||
|
|||||||
@@ -1,52 +0,0 @@
|
|||||||
version: "2"
|
|
||||||
linters:
|
|
||||||
enable:
|
|
||||||
- errname
|
|
||||||
- errorlint
|
|
||||||
- exhaustive
|
|
||||||
- goprintffuncname
|
|
||||||
- misspell
|
|
||||||
- mnd
|
|
||||||
- tagliatelle
|
|
||||||
- testpackage
|
|
||||||
- thelper
|
|
||||||
- tparallel
|
|
||||||
- unconvert
|
|
||||||
- wrapcheck
|
|
||||||
- wsl
|
|
||||||
disable:
|
|
||||||
- gochecknoglobals
|
|
||||||
settings:
|
|
||||||
gosec:
|
|
||||||
excludes:
|
|
||||||
- G204
|
|
||||||
tagliatelle:
|
|
||||||
case:
|
|
||||||
rules:
|
|
||||||
json: snake
|
|
||||||
yaml: snake
|
|
||||||
exclusions:
|
|
||||||
generated: lax
|
|
||||||
presets:
|
|
||||||
- comments
|
|
||||||
- common-false-positives
|
|
||||||
- legacy
|
|
||||||
- std-error-handling
|
|
||||||
rules:
|
|
||||||
- linters:
|
|
||||||
- gosec
|
|
||||||
path: _test\.go
|
|
||||||
paths:
|
|
||||||
- third_party$
|
|
||||||
- builtin$
|
|
||||||
- examples$
|
|
||||||
formatters:
|
|
||||||
enable:
|
|
||||||
- gofumpt
|
|
||||||
- goimports
|
|
||||||
exclusions:
|
|
||||||
generated: lax
|
|
||||||
paths:
|
|
||||||
- third_party$
|
|
||||||
- builtin$
|
|
||||||
- examples$
|
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
---
|
---
|
||||||
repos:
|
repos:
|
||||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||||
rev: v6.0.0
|
rev: v2.4.0
|
||||||
hooks:
|
hooks:
|
||||||
- id: check-added-large-files
|
- id: check-added-large-files
|
||||||
- id: check-yaml
|
- id: check-yaml
|
||||||
@@ -10,11 +10,10 @@ repos:
|
|||||||
- id: trailing-whitespace
|
- id: trailing-whitespace
|
||||||
- id: end-of-file-fixer
|
- id: end-of-file-fixer
|
||||||
- id: check-merge-conflict
|
- id: check-merge-conflict
|
||||||
- repo: https://github.com/golangci/golangci-lint
|
- repo: git://github.com/dnephin/pre-commit-golang
|
||||||
rev: v2.7.2
|
rev: v0.3.5
|
||||||
hooks:
|
hooks:
|
||||||
- id: golangci-lint
|
- id: go-fmt
|
||||||
- repo: https://github.com/hadolint/hadolint
|
- id: go-imports
|
||||||
rev: refs/pull/1152/head
|
# - id: gometalinter
|
||||||
hooks:
|
# - id: golangci-lint
|
||||||
- id: hadolint-github
|
|
||||||
|
|||||||
+8
-9
@@ -1,24 +1,23 @@
|
|||||||
FROM alpine:3.23
|
ARG REPO=library
|
||||||
|
FROM ${REPO}/alpine:3.10
|
||||||
RUN mkdir /app
|
RUN mkdir /app
|
||||||
WORKDIR /app/
|
WORKDIR /app/
|
||||||
|
|
||||||
|
# Copy minitor in
|
||||||
|
ARG ARCH=amd64
|
||||||
|
COPY ./minitor-go ./minitor
|
||||||
|
|
||||||
# Add common checking tools
|
# Add common checking tools
|
||||||
# hadolint ignore=DL3018
|
RUN apk --no-cache add bash=~5.0 curl=~7.66 jq=~1.6
|
||||||
RUN apk --no-cache add bash=~5 curl=~8 jq=~1 bind-tools=~9 tzdata
|
|
||||||
|
|
||||||
# Add minitor user for running as non-root
|
# Add minitor user for running as non-root
|
||||||
RUN addgroup -S minitor && adduser -S minitor -G minitor
|
RUN addgroup -S minitor && adduser -S minitor -G minitor
|
||||||
|
|
||||||
# Copy scripts
|
# Copy scripts
|
||||||
COPY ./scripts /app/scripts
|
COPY ./scripts /app/scripts
|
||||||
|
RUN chown -R minitor:minitor /app
|
||||||
RUN chmod -R 755 /app/scripts
|
RUN chmod -R 755 /app/scripts
|
||||||
|
|
||||||
# Copy minitor in
|
|
||||||
ARG TARGETOS
|
|
||||||
ARG TARGETARCH
|
|
||||||
COPY ./dist/minitor-${TARGETOS}-${TARGETARCH} ./minitor
|
|
||||||
|
|
||||||
# Drop to non-root user
|
# Drop to non-root user
|
||||||
USER minitor
|
USER minitor
|
||||||
|
|
||||||
|
|||||||
+10
-7
@@ -1,5 +1,9 @@
|
|||||||
FROM golang:1.25 AS builder
|
ARG REPO=library
|
||||||
|
FROM golang:1.12-alpine AS builder
|
||||||
|
|
||||||
|
RUN apk add --no-cache git=~2
|
||||||
|
|
||||||
|
RUN mkdir /app
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
COPY ./go.mod ./go.sum /app/
|
COPY ./go.mod ./go.sum /app/
|
||||||
@@ -7,13 +11,12 @@ RUN go mod download
|
|||||||
|
|
||||||
COPY ./*.go /app/
|
COPY ./*.go /app/
|
||||||
|
|
||||||
ARG TARGETOS
|
ARG ARCH=amd64
|
||||||
ARG TARGETARCH
|
|
||||||
ARG VERSION=dev
|
ARG VERSION=dev
|
||||||
ENV CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=${TARGETARCH}
|
ENV CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH}
|
||||||
RUN go build -ldflags "-X main.version=${VERSION}" -a -installsuffix nocgo -o minitor .
|
RUN go build -ldflags "-X main.version=${VERSION}" -a -installsuffix nocgo -o minitor .
|
||||||
|
|
||||||
FROM alpine:3.23
|
FROM ${REPO}/alpine:3.10
|
||||||
RUN mkdir /app
|
RUN mkdir /app
|
||||||
WORKDIR /app/
|
WORKDIR /app/
|
||||||
|
|
||||||
@@ -21,14 +24,14 @@ WORKDIR /app/
|
|||||||
COPY --from=builder /app/minitor .
|
COPY --from=builder /app/minitor .
|
||||||
|
|
||||||
# Add common checking tools
|
# Add common checking tools
|
||||||
# hadolint ignore=DL3018
|
RUN apk --no-cache add bash=~5.0 curl=~7.66 jq=~1.6
|
||||||
RUN apk --no-cache add bash=~5 curl=~8 jq=~1 bind-tools=~9 tzdata
|
|
||||||
|
|
||||||
# Add minitor user for running as non-root
|
# Add minitor user for running as non-root
|
||||||
RUN addgroup -S minitor && adduser -S minitor -G minitor
|
RUN addgroup -S minitor && adduser -S minitor -G minitor
|
||||||
|
|
||||||
# Copy scripts
|
# Copy scripts
|
||||||
COPY ./scripts /app/scripts
|
COPY ./scripts /app/scripts
|
||||||
|
RUN chown -R minitor:minitor /app
|
||||||
RUN chmod -R 755 /app/scripts
|
RUN chmod -R 755 /app/scripts
|
||||||
|
|
||||||
# Drop to non-root user
|
# Drop to non-root user
|
||||||
|
|||||||
@@ -1,43 +1,31 @@
|
|||||||
DOCKER_TAG ?= minitor-go-${USER}
|
|
||||||
VERSION ?= $(shell git describe --tags --dirty)
|
|
||||||
GOFILES = *.go go.mod go.sum
|
|
||||||
# Multi-arch targets are generated from this
|
|
||||||
TARGET_ALIAS = minitor-linux-amd64 minitor-linux-arm minitor-linux-arm64 minitor-darwin-amd64
|
|
||||||
TARGETS = $(addprefix dist/,$(TARGET_ALIAS))
|
|
||||||
#
|
|
||||||
# Default make target will run tests
|
|
||||||
.DEFAULT_GOAL = test
|
|
||||||
|
|
||||||
# Build all static Minitor binaries
|
|
||||||
.PHONY: all
|
.PHONY: all
|
||||||
all: $(TARGETS)
|
DOCKER_TAG ?= minitor-go-${USER}
|
||||||
|
|
||||||
# Build all static Linux Minitor binaries. Used in Docker images
|
.PHONY: default
|
||||||
.PHONY: all-linux
|
default: test
|
||||||
all-linux: $(filter dist/minitor-linux-%,$(TARGETS))
|
|
||||||
|
|
||||||
# Build minitor for the current machine
|
|
||||||
minitor: $(GOFILES)
|
|
||||||
@echo Version: $(VERSION)
|
|
||||||
go build -ldflags '-X "main.version=${VERSION}"' -o minitor
|
|
||||||
|
|
||||||
.PHONY: build
|
.PHONY: build
|
||||||
build: minitor
|
build:
|
||||||
|
go build
|
||||||
|
|
||||||
|
minitor-go:
|
||||||
|
go build
|
||||||
|
|
||||||
# Run minitor for the current machine
|
|
||||||
.PHONY: run
|
.PHONY: run
|
||||||
run: minitor
|
run: minitor-go build
|
||||||
./minitor -debug
|
./minitor-go -debug
|
||||||
|
|
||||||
.PHONY: run-metrics
|
.PHONY: run-metrics
|
||||||
run-metrics: minitor
|
run-metrics: minitor-go build
|
||||||
./minitor -debug -metrics
|
./minitor-go -debug -metrics
|
||||||
|
|
||||||
# Run all tests
|
|
||||||
.PHONY: test
|
.PHONY: test
|
||||||
test:
|
test:
|
||||||
go test -coverprofile=coverage.out
|
go test -coverprofile=coverage.out
|
||||||
|
@echo
|
||||||
go tool cover -func=coverage.out
|
go tool cover -func=coverage.out
|
||||||
|
@echo
|
||||||
|
@# Check min coverage percentage
|
||||||
@go tool cover -func=coverage.out | awk -v target=80.0% \
|
@go tool cover -func=coverage.out | awk -v target=80.0% \
|
||||||
'/^total:/ { print "Total coverage: " $$3 " Minimum coverage: " target; if ($$3+0.0 >= target+0.0) print "ok"; else { print "fail"; exit 1; } }'
|
'/^total:/ { print "Total coverage: " $$3 " Minimum coverage: " target; if ($$3+0.0 >= target+0.0) print "ok"; else { print "fail"; exit 1; } }'
|
||||||
|
|
||||||
@@ -46,50 +34,20 @@ test:
|
|||||||
install-hooks:
|
install-hooks:
|
||||||
pre-commit install --install-hooks
|
pre-commit install --install-hooks
|
||||||
|
|
||||||
# Runs pre-commit checks on files
|
# Checks files for encryption
|
||||||
.PHONY: check
|
.PHONY: check
|
||||||
check:
|
check:
|
||||||
pre-commit run --all-files
|
pre-commit run --all-files
|
||||||
|
|
||||||
.PHONY: clean
|
.PHONY: clean
|
||||||
clean:
|
clean:
|
||||||
rm -f ./minitor
|
rm -f ./minitor-go
|
||||||
rm -f ./coverage.out
|
rm -f ./coverage.out
|
||||||
rm -fr ./dist
|
|
||||||
|
|
||||||
.PHONY: docker-build
|
.PHONY: docker-build
|
||||||
docker-build:
|
docker-build:
|
||||||
docker build -f ./Dockerfile.multi-stage -t $(DOCKER_TAG)-linux-amd64 .
|
docker build -f ./Dockerfile.multi-stage -t $(DOCKER_TAG) .
|
||||||
|
|
||||||
.PHONY: docker-run
|
.PHONY: docker-run
|
||||||
docker-run: docker-build
|
docker-run: docker-build
|
||||||
docker run --rm -v $(shell pwd)/sample-config.hcl:/root/config.hcl $(DOCKER_TAG)
|
docker run --rm -v $(shell pwd)/config.yml:/root/config.yml $(DOCKER_TAG)
|
||||||
|
|
||||||
## Multi-arch targets
|
|
||||||
$(TARGETS): $(GOFILES)
|
|
||||||
mkdir -p ./dist
|
|
||||||
GOOS=$(word 2, $(subst -, ,$(@))) GOARCH=$(word 3, $(subst -, ,$(@))) CGO_ENABLED=0 \
|
|
||||||
go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
|
|
||||||
-o $@
|
|
||||||
|
|
||||||
.PHONY: $(TARGET_ALIAS)
|
|
||||||
$(TARGET_ALIAS):
|
|
||||||
$(MAKE) $(addprefix dist/,$@)
|
|
||||||
|
|
||||||
# Arch specific docker build targets
|
|
||||||
.PHONY: docker-build-arm
|
|
||||||
docker-build-arm: dist/minitor-linux-arm
|
|
||||||
docker build --platform linux/arm . -t ${DOCKER_TAG}-linux-arm
|
|
||||||
|
|
||||||
.PHONY: docker-build-arm64
|
|
||||||
docker-build-arm64: dist/minitor-linux-arm64
|
|
||||||
docker build --platform linux/arm64 . -t ${DOCKER_TAG}-linux-arm64
|
|
||||||
|
|
||||||
# Cross run on host architechture
|
|
||||||
.PHONY: docker-run-arm
|
|
||||||
docker-run-arm: docker-build-arm
|
|
||||||
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock --name $(DOCKER_TAG)-run ${DOCKER_TAG}-linux-arm
|
|
||||||
|
|
||||||
.PHONY: docker-run-arm64
|
|
||||||
docker-run-arm64: docker-build-arm64
|
|
||||||
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock --name $(DOCKER_TAG)-run ${DOCKER_TAG}-linux-arm64
|
|
||||||
|
|||||||
@@ -1,220 +1,99 @@
|
|||||||
# [minitor-go](https://git.iamthefij.com/iamthefij/minitor-go)
|
# minitor-go
|
||||||
|
|
||||||
A minimal monitoring system
|
A reimplementation of [Minitor](https://git.iamthefij/iamthefij/minitor) in Go
|
||||||
|
|
||||||
## What does it do?
|
Minitor is already a minimal monitoring tool. Python 3 was a quick way to get something live, but Python itself comes with a large footprint. Thus Go feels like a better fit for the project, longer term.
|
||||||
|
|
||||||
Minitor accepts an HCL configuration file with a set of commands to run and a set of alerts to execute when those commands fail. Minitor has a narow feature set and instead follows a principle to outsource to other command line tools when possible. Thus, it relies on other command line tools to do checks and issue alerts. To make getting started a bit easier, Minitor includes a few scripts to help with common tasks.
|
Initial target is meant to be roughly compatible requiring only minor changes to configuration. Future iterations may diverge to take advantage of Go specific features.
|
||||||
|
|
||||||
## But why?
|
## Differences from Python version
|
||||||
|
|
||||||
I'm running a few small services and found Sensu, Consul, Nagios, etc. to all be far too complicated for my usecase.
|
There are a few key differences between the Python version and the v0.x Go version.
|
||||||
|
|
||||||
## So how do I use it?
|
First, configuration keys cannot have multiple types in Go, so a different key must be used when specifying a Shell command as a string rather than a list of args. Instead of `command`, you must use `command_shell`. Eg:
|
||||||
|
|
||||||
### Running
|
|
||||||
|
|
||||||
Install and execute with:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go install github.com/iamthefij/minitor-go/v2@latest
|
|
||||||
minitor
|
|
||||||
```
|
|
||||||
|
|
||||||
If locally developing you can use:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
make run
|
|
||||||
```
|
|
||||||
|
|
||||||
It will read the contents of `sample-config.hcl` and begin its loop. You could also run it directly and provide a new config file via the `-config` argument.
|
|
||||||
|
|
||||||
|
|
||||||
#### Docker
|
|
||||||
|
|
||||||
You can pull this repository directly from Docker:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker pull iamthefij/minitor-go:latest
|
|
||||||
```
|
|
||||||
|
|
||||||
The Docker image uses a default `config.hcl` copied from `sample-config.hcl`. This won't really do anything for you, so when you run the Docker image, you should supply your own `config.hcl` file:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker run -v $PWD/sample-config.hcl:/app/config.hcl iamthefij/minitor-go:latest
|
|
||||||
```
|
|
||||||
|
|
||||||
Images are provided for `amd64`, `arm`, and `arm64` architechtures.
|
|
||||||
|
|
||||||
You can configure the timezone for the container by passing a `TZ` env variable. Eg. `TZ=America/Los_Angeles`.
|
|
||||||
|
|
||||||
## Configuring
|
|
||||||
|
|
||||||
In this repo, you can explore the `sample-config.hcl` file for an example, but the general structure is as follows. If you are passing environment variables to your commands or alerts, you should be aware that `${VAR}` syntax is reserved for HCL variable interpolation. To avoid issues, you can use `$${VAR}` syntax to escape the `$` character, simply use `$VAR`.
|
|
||||||
|
|
||||||
```hcl
|
|
||||||
|
|
||||||
The global configurations are:
|
|
||||||
|
|
||||||
|key|value|
|
|
||||||
|---|---|
|
|
||||||
|`check_interval`|Maximum frequency to run checks for each monitor as duration, eg. 1m2s.|
|
|
||||||
|`default_alert_after`|A default value used as an `alert_after` value for a monitor if not specified. Defaults 1, which will alert immediately.|
|
|
||||||
|`default_alert_every`|A default value used as an `alert_every` value for a monitor if not specified. Defaults to -1, which will re-alert exponentially.|
|
|
||||||
|`default_alert_down`|Default down alerts to used by a monitor in case none are provided.|
|
|
||||||
|`default_alert_up`|Default up alerts to used by a monitor in case none are provided.|
|
|
||||||
|`monitor`|block listing monitors. Detailed description below|
|
|
||||||
|`alert`|List of all alerts. Detailed description below|
|
|
||||||
|
|
||||||
### Monitors
|
|
||||||
|
|
||||||
Represent your monitors as blocks with a label indicating the name of the monitor.
|
|
||||||
|
|
||||||
```hcl
|
|
||||||
monitor "example" {
|
|
||||||
command = ["echo", "Hello, World!"]
|
|
||||||
alert_down = ["log"]
|
|
||||||
alert_up = ["log"]
|
|
||||||
check_interval = "1m"
|
|
||||||
alert_after = 1
|
|
||||||
alert_every = -1
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Each monitor allows the following configuration:
|
|
||||||
|
|
||||||
|key|value|
|
|
||||||
|---|---|
|
|
||||||
|`name`|Name of the monitor running. This will show up in messages and logs.|
|
|
||||||
|`command`|A list of strings representing a command to be executed. This command's exit value will determine whether the check is successful. This value is mutually exclusive to `shell_command`|
|
|
||||||
|`shell_command`|A single string that represents a shell command to be executed. This command's exit value will determine whether the check is successful. This value is mutually exclusive to `command`|
|
|
||||||
|`alert_down`|A list of Alerts to be triggered when the monitor is in a "down" state|
|
|
||||||
|`alert_up`|A list of Alerts to be triggered when the monitor moves to an "up" state|
|
|
||||||
|`check_interval`|The interval at which this monitor should be checked. This must be greater than the global `check_interval` value|
|
|
||||||
|`alert_after`|Allows specifying the number of failed checks before an alert should be triggered. A value of 1 will start sending alerts after the first failure.|
|
|
||||||
|`alert_every`|Allows specifying how often an alert should be retriggered. There are a few magic numbers here. Defaults to `-1` for an exponential backoff. Setting to `0` disables re-alerting. Positive values will allow retriggering after the specified number of checks|
|
|
||||||
|
|
||||||
### Alerts
|
|
||||||
|
|
||||||
Represent your alerts as blocks with a lable indicating the name of the alert. The name will be used in your monitor setup in `alert_down` and `alert_up`.
|
|
||||||
|
|
||||||
```hcl
|
|
||||||
monitor "example" {
|
|
||||||
command = ["false"]
|
|
||||||
alert_down = ["log"]
|
|
||||||
}
|
|
||||||
|
|
||||||
alert "log" {
|
|
||||||
shell_command = "echo '{{.MonitorName}} is down!'"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Each alert allows the following configuration:
|
|
||||||
|
|
||||||
|key|value|
|
|
||||||
|---|---|
|
|
||||||
|`command`|Specifies the command that should be executed in exec form. This is the command that will be run when the alert is executed. This can be templated with environment variables or the variables shown in the table below. This value is mutually exclusive to `shell_command`|
|
|
||||||
|`shell_command`|Specifies a shell command as a single string. This is the command that will be run when the alert is executed. This can be templated with environment variables or the variables shown in the table below. This value is mutually exclusive to `command`|
|
|
||||||
|
|
||||||
Also, when alerts are executed, they will be passed through Go's format function with arguments for some attributes of the Monitor. The following monitor specific variables can be referenced using Go formatting syntax:
|
|
||||||
|
|
||||||
|token|value|
|
|
||||||
|---|---|
|
|
||||||
|`{{.AlertCount}}`|Number of times this monitor has alerted|
|
|
||||||
|`{{.FailureCount}}`|The total number of sequential failed checks for this monitor|
|
|
||||||
|`{{.LastCheckOutput}}`|The last returned value from the check command to either stderr or stdout|
|
|
||||||
|`{{.LastSuccess}}`|The datetime of the last successful check as a go Time struct|
|
|
||||||
|`{{.MonitorName}}`|The name of the monitor that failed and triggered the alert|
|
|
||||||
|`{{.IsUp}}`|Indicates if the monitor that is alerting is up or not. Can be used in a conditional message template|
|
|
||||||
|
|
||||||
To provide flexible formatting, the following non-standard functions are available in templates:
|
|
||||||
|
|
||||||
|func|description|
|
|
||||||
|---|---|
|
|
||||||
|`ANSIC <Time>`|Formats provided time in ANSIC format|
|
|
||||||
|`UnixDate <Time>`|Formats provided time in UnixDate format|
|
|
||||||
|`RubyDate <Time>`|Formats provided time in RubyDate format|
|
|
||||||
|`RFC822Z <Time>`|Formats provided time in RFC822Z format|
|
|
||||||
|`RFC850 <Time>`|Formats provided time in RFC850 format|
|
|
||||||
|`RFC1123 <Time>`|Formats provided time in RFC1123 format|
|
|
||||||
|`RFC1123Z <Time>`|Formats provided time in RFC1123Z format|
|
|
||||||
|`RFC3339 <Time>`|Formats provided time in RFC3339 format|
|
|
||||||
|`RFC3339Nano <Time>`|Formats provided time in RFC3339Nano format|
|
|
||||||
|`FormatTime <Time> <string template>`|Formats provided time according to provided template|
|
|
||||||
|`InTZ <Time> <string timezone name>`|Converts provided time to parsed timezone from the provided name|
|
|
||||||
|
|
||||||
For more information, check out the [Go documentation for the time module](https://pkg.go.dev/time@go1.20.7#pkg-constants).
|
|
||||||
|
|
||||||
#### Running alerts on startup
|
|
||||||
|
|
||||||
It's not the best feeling to find out your alerts are broken when you're expecting to be alerted about another failure. To avoid this and provide early insight into broken alerts, it is possible to specify a list of alerts to run when Minitor starts up. This can be done using the command line flag `-startup-alerts`. This flag accepts a comma separated list of strings and will run a test of each of those alerts. Minitor will then respond as it typically does for any failed alert. This can be used to allow you time to correct when initially launching, and to allow schedulers to more easily detect a failed deployment of Minitor.
|
|
||||||
|
|
||||||
Eg.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
minitor -startup-alerts=log_down,log_up -config ./config.hcl
|
|
||||||
```
|
|
||||||
|
|
||||||
### Metrics
|
|
||||||
|
|
||||||
Minitor supports exporting metrics for [Prometheus](https://prometheus.io/). Prometheus is an open source tool for reading and querying metrics from different sources. Combined with another tool, [Grafana](https://grafana.com/), it allows building of charts and dashboards. You could also opt to just use Minitor to log check results, and instead do your alerting with Grafana.
|
|
||||||
|
|
||||||
It is also possible to use the metrics endpoint for monitoring Minitor itself! This allows setting up multiple instances of Minitor on different servers and have them monitor each-other so that you can detect a minitor outage.
|
|
||||||
|
|
||||||
To run minitor with metrics, use the `-metrics` flag. The metrics will be served on port `8080` by default, though it can be overriden using `-metrics-port`. They will be accessible on the path `/metrics`. Eg. `localhost:8080/metrics`.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
minitor -metrics
|
|
||||||
# or
|
|
||||||
minitor -metrics -metrics-port 3000
|
|
||||||
```
|
|
||||||
|
|
||||||
## Migrating from v1 to v2
|
|
||||||
|
|
||||||
Minitor v2 introduces some breaking changes from v1. The most notable changes are:
|
|
||||||
- The configuration file is now in HCL format instead of YAML.
|
|
||||||
- The the Python formatting backwards compatability is removed.
|
|
||||||
- The Command and ShellCommand fields are now mutually exclusive.
|
|
||||||
- The check_interval is now strictly a duration string value. Eg. "30s" rather than `30`.
|
|
||||||
- Default alert_every is now -1 (exponential backoff) rather than 0 (no re-alerting).
|
|
||||||
|
|
||||||
For the configuration, a confic that looked like this in v1:
|
|
||||||
|
|
||||||
|
minitor-py:
|
||||||
```yaml
|
```yaml
|
||||||
check_interval: 60
|
|
||||||
|
|
||||||
monitors:
|
monitors:
|
||||||
- name: example
|
- name: Exec command
|
||||||
command: "false"
|
command: ['echo', 'test']
|
||||||
alert_down: ["log"]
|
- name: Shell command
|
||||||
|
command: echo 'test'
|
||||||
|
```
|
||||||
|
|
||||||
|
minitor-go:
|
||||||
|
```yaml
|
||||||
|
monitors:
|
||||||
|
- name: Exec command
|
||||||
|
command: ['echo', 'test']
|
||||||
|
- name: Shell command
|
||||||
|
command_shell: echo 'test'
|
||||||
|
```
|
||||||
|
|
||||||
|
Second, templating for Alert messages has been updated. In the Python version, `str.format(...)` was used with certain keys passed in that could be used to format messages. In the Go version, we use a struct, `AlertNotice` defined in `alert.go` and the built in Go templating format. Eg.
|
||||||
|
|
||||||
|
minitor-py:
|
||||||
|
```yaml
|
||||||
alerts:
|
alerts:
|
||||||
log:
|
log_command:
|
||||||
command: ["echo", "Minitor up={{.IsUp}} for {{.MonitorName}}"]
|
command: ['echo', '{monitor_name}']
|
||||||
|
log_shell:
|
||||||
|
command_shell: 'echo {monitor_name}'
|
||||||
```
|
```
|
||||||
|
|
||||||
Would now look like this in v2:
|
minitor-go:
|
||||||
|
```yaml
|
||||||
```hcl
|
alerts:
|
||||||
check_interval = "1m"
|
log_command:
|
||||||
|
command: ['echo', '{{.MonitorName}}']
|
||||||
monitor "example" {
|
log_shell:
|
||||||
# example showing string to shell command migration
|
command_shell: 'echo {{.MonitorName}}'
|
||||||
shell_command = "false"
|
|
||||||
alert_down = ["log"]
|
|
||||||
check_interval = "1m"
|
|
||||||
}
|
|
||||||
|
|
||||||
alert "log" {
|
|
||||||
# example showing list to exec command migration
|
|
||||||
command = ["echo", "Minitor up={{.IsUp}} for {{.MonitorName}}"]
|
|
||||||
}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Contributing
|
Finally, newlines in a shell command don't terminate a particular command. Semicolons must be used and continuations should not.
|
||||||
|
|
||||||
Whether you're looking to submit a patch or tell me I broke something, you can contribute through the Github mirror and I can merge PRs back to the source repository.
|
minitor-py:
|
||||||
|
```yaml
|
||||||
|
alerts:
|
||||||
|
log_shell:
|
||||||
|
command_shell: >
|
||||||
|
echo "line 1"
|
||||||
|
echo "line 2"
|
||||||
|
echo "continued" \
|
||||||
|
"line"
|
||||||
|
```
|
||||||
|
|
||||||
Primary Repo: https://git.iamthefij.com/iamthefij/minitor.git
|
minitor-go:
|
||||||
|
```yaml
|
||||||
|
alerts:
|
||||||
|
log_shell:
|
||||||
|
command_shell: >
|
||||||
|
echo "line 1";
|
||||||
|
echo "line 2";
|
||||||
|
echo "continued"
|
||||||
|
"line"
|
||||||
|
```
|
||||||
|
|
||||||
Github Mirror: https://github.com/IamTheFij/minitor.git
|
## To do
|
||||||
|
There are two sets of task lists. The first is to get rough parity on key features with the Python version. The second is to make some improvements to the framework.
|
||||||
|
|
||||||
|
Pairity:
|
||||||
|
|
||||||
|
- [x] Run monitor commands
|
||||||
|
- [x] Run monitor commands in a shell
|
||||||
|
- [x] Run alert commands
|
||||||
|
- [x] Run alert commands in a shell
|
||||||
|
- [x] Allow templating of alert commands
|
||||||
|
- [x] Implement Prometheus client to export metrics
|
||||||
|
- [x] Test coverage
|
||||||
|
- [ ] Integration testing (manual or otherwise)
|
||||||
|
|
||||||
|
Improvement (potentially breaking):
|
||||||
|
|
||||||
|
- [ ] Implement leveled logging (maybe glog or logrus)
|
||||||
|
- [ ] Consider switching from YAML to TOML
|
||||||
|
- [ ] Consider value of templating vs injecting values into Env variables
|
||||||
|
- [ ] Consider dropping `alert_up` and `alert_down` in favor of using Go templates that offer more control of messaging
|
||||||
|
- [ ] Async checking
|
||||||
|
- [ ] Use durations rather than seconds checked in event loop
|
||||||
|
- [ ] Revisit metrics and see if they all make sense
|
||||||
|
|||||||
@@ -2,161 +2,89 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"log"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"text/template"
|
"text/template"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.iamthefij.com/iamthefij/slog"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
errNoTemplate = errors.New("no template")
|
|
||||||
|
|
||||||
// ErrAlertFailed indicates that an alert failed to send
|
|
||||||
ErrAlertFailed = errors.New("alert failed")
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Alert is a config driven mechanism for sending a notice
|
// Alert is a config driven mechanism for sending a notice
|
||||||
type Alert struct {
|
type Alert struct {
|
||||||
Name string `hcl:"name,label"`
|
Name string
|
||||||
Command []string `hcl:"command,optional"`
|
Command []string
|
||||||
ShellCommand string `hcl:"shell_command,optional"`
|
CommandShell string `yaml:"command_shell"`
|
||||||
commandTemplate []*template.Template
|
commandTemplate []*template.Template
|
||||||
commandShellTemplate *template.Template
|
commandShellTemplate *template.Template
|
||||||
}
|
}
|
||||||
|
|
||||||
// AlertNotice captures the context for an alert to be sent
|
// AlertNotice captures the context for an alert to be sent
|
||||||
type AlertNotice struct {
|
type AlertNotice struct {
|
||||||
AlertCount int
|
|
||||||
FailureCount int
|
|
||||||
IsUp bool
|
|
||||||
LastSuccess time.Time
|
|
||||||
MonitorName string
|
MonitorName string
|
||||||
|
AlertCount int16
|
||||||
|
FailureCount int16
|
||||||
LastCheckOutput string
|
LastCheckOutput string
|
||||||
|
LastSuccess time.Time
|
||||||
|
IsUp bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate checks that the Alert is properly configured and returns errors if not
|
// IsValid returns a boolean indicating if the Alert has been correctly
|
||||||
func (alert Alert) Validate() error {
|
// configured
|
||||||
hasCommand := len(alert.Command) > 0
|
func (alert Alert) IsValid() bool {
|
||||||
hasShellCommand := alert.ShellCommand != ""
|
atLeastOneCommand := (alert.CommandShell != "" || alert.Command != nil)
|
||||||
|
atMostOneCommand := (alert.CommandShell == "" || alert.Command == nil)
|
||||||
var err error
|
return atLeastOneCommand && atMostOneCommand
|
||||||
|
|
||||||
hasAtLeastOneCommand := hasCommand || hasShellCommand
|
|
||||||
if !hasAtLeastOneCommand {
|
|
||||||
err = errors.Join(err, fmt.Errorf(
|
|
||||||
"%w: alert %s has no command or shell_command configured",
|
|
||||||
ErrInvalidAlert,
|
|
||||||
alert.Name,
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
hasAtMostOneCommand := !(hasCommand && hasShellCommand)
|
|
||||||
if !hasAtMostOneCommand {
|
|
||||||
err = errors.Join(err, fmt.Errorf(
|
|
||||||
"%w: alert %s has both command and shell_command configured",
|
|
||||||
ErrInvalidAlert,
|
|
||||||
alert.Name,
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// BuildTemplates compiles command templates for the Alert
|
// BuildTemplates compiles command templates for the Alert
|
||||||
func (alert *Alert) BuildTemplates() error {
|
func (alert *Alert) BuildTemplates() error {
|
||||||
slog.Debugf("Building template for alert %s", alert.Name)
|
if LogDebug {
|
||||||
|
log.Printf("DEBUG: Building template for alert %s", alert.Name)
|
||||||
// Time format func factory
|
|
||||||
tff := func(formatString string) func(time.Time) string {
|
|
||||||
return func(t time.Time) string {
|
|
||||||
return t.Format(formatString)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
if alert.commandTemplate == nil && alert.Command != nil {
|
||||||
// Create some functions for formatting datetimes in popular formats
|
|
||||||
timeFormatFuncs := template.FuncMap{
|
|
||||||
"ANSIC": tff(time.ANSIC),
|
|
||||||
"UnixDate": tff(time.UnixDate),
|
|
||||||
"RubyDate": tff(time.RubyDate),
|
|
||||||
"RFC822Z": tff(time.RFC822Z),
|
|
||||||
"RFC850": tff(time.RFC850),
|
|
||||||
"RFC1123": tff(time.RFC1123),
|
|
||||||
"RFC1123Z": tff(time.RFC1123Z),
|
|
||||||
"RFC3339": tff(time.RFC3339),
|
|
||||||
"RFC3339Nano": tff(time.RFC3339Nano),
|
|
||||||
"FormatTime": func(t time.Time, timeFormat string) string {
|
|
||||||
return t.Format(timeFormat)
|
|
||||||
},
|
|
||||||
"InTZ": func(t time.Time, tzName string) (time.Time, error) {
|
|
||||||
tz, err := time.LoadLocation(tzName)
|
|
||||||
if err != nil {
|
|
||||||
return t, fmt.Errorf("failed to convert time to specified tz: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return t.In(tz), nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case alert.Command != nil:
|
|
||||||
alert.commandTemplate = []*template.Template{}
|
alert.commandTemplate = []*template.Template{}
|
||||||
for i, cmdPart := range alert.Command {
|
for i, cmdPart := range alert.Command {
|
||||||
alert.commandTemplate = append(alert.commandTemplate, template.Must(
|
alert.commandTemplate = append(alert.commandTemplate, template.Must(
|
||||||
template.New(alert.Name+fmt.Sprint(i)).Funcs(timeFormatFuncs).Parse(cmdPart),
|
template.New(alert.Name+string(i)).Parse(cmdPart),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
case alert.ShellCommand != "":
|
} else if alert.commandShellTemplate == nil && alert.CommandShell != "" {
|
||||||
shellCmd := alert.ShellCommand
|
|
||||||
|
|
||||||
alert.commandShellTemplate = template.Must(
|
alert.commandShellTemplate = template.Must(
|
||||||
template.New(alert.Name).Funcs(timeFormatFuncs).Parse(shellCmd),
|
template.New(alert.Name).Parse(alert.CommandShell),
|
||||||
)
|
)
|
||||||
default:
|
} else {
|
||||||
return fmt.Errorf("No template provided for alert %s: %w", alert.Name, errNoTemplate)
|
return fmt.Errorf("No template provided for alert %s", alert.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Send will send an alert notice by executing the command template
|
// Send will send an alert notice by executing the command template
|
||||||
func (alert Alert) Send(notice AlertNotice) (outputStr string, err error) {
|
func (alert Alert) Send(notice AlertNotice) (output_str string, err error) {
|
||||||
slog.Infof("Sending alert %s for %s", alert.Name, notice.MonitorName)
|
log.Printf("INFO: Sending alert %s for %s", alert.Name, notice.MonitorName)
|
||||||
|
|
||||||
var cmd *exec.Cmd
|
var cmd *exec.Cmd
|
||||||
|
if alert.commandTemplate != nil {
|
||||||
switch {
|
|
||||||
case alert.commandTemplate != nil:
|
|
||||||
command := []string{}
|
command := []string{}
|
||||||
|
|
||||||
for _, cmdTmp := range alert.commandTemplate {
|
for _, cmdTmp := range alert.commandTemplate {
|
||||||
var commandBuffer bytes.Buffer
|
var commandBuffer bytes.Buffer
|
||||||
|
|
||||||
err = cmdTmp.Execute(&commandBuffer, notice)
|
err = cmdTmp.Execute(&commandBuffer, notice)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
command = append(command, commandBuffer.String())
|
command = append(command, commandBuffer.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
cmd = exec.Command(command[0], command[1:]...)
|
cmd = exec.Command(command[0], command[1:]...)
|
||||||
case alert.commandShellTemplate != nil:
|
} else if alert.commandShellTemplate != nil {
|
||||||
var commandBuffer bytes.Buffer
|
var commandBuffer bytes.Buffer
|
||||||
|
|
||||||
err = alert.commandShellTemplate.Execute(&commandBuffer, notice)
|
err = alert.commandShellTemplate.Execute(&commandBuffer, notice)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
shellCommand := commandBuffer.String()
|
shellCommand := commandBuffer.String()
|
||||||
|
|
||||||
cmd = ShellCommand(shellCommand)
|
cmd = ShellCommand(shellCommand)
|
||||||
default:
|
} else {
|
||||||
err = fmt.Errorf("No templates compiled for alert %s: %w", alert.Name, errNoTemplate)
|
err = fmt.Errorf("No templates compiled for alert %v", alert.Name)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -167,17 +95,10 @@ func (alert Alert) Send(notice AlertNotice) (outputStr string, err error) {
|
|||||||
|
|
||||||
var output []byte
|
var output []byte
|
||||||
output, err = cmd.CombinedOutput()
|
output, err = cmd.CombinedOutput()
|
||||||
outputStr = string(output)
|
output_str = string(output)
|
||||||
slog.Debugf("Alert output for: %s\n---\n%s\n---", alert.Name, outputStr)
|
if LogDebug {
|
||||||
|
log.Printf("DEBUG: Alert output for: %s\n---\n%s\n---", alert.Name, output_str)
|
||||||
if err != nil {
|
|
||||||
err = fmt.Errorf(
|
|
||||||
"Alert %s failed to send. Returned %w: %w",
|
|
||||||
alert.Name,
|
|
||||||
err,
|
|
||||||
ErrAlertFailed,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return outputStr, err
|
return output_str, err
|
||||||
}
|
}
|
||||||
|
|||||||
+58
-77
@@ -1,75 +1,69 @@
|
|||||||
package main_test
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"log"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
m "git.iamthefij.com/iamthefij/minitor-go/v2"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestAlertValidate(t *testing.T) {
|
func TestAlertIsValid(t *testing.T) {
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
alert m.Alert
|
alert Alert
|
||||||
expected error
|
expected bool
|
||||||
name string
|
name string
|
||||||
}{
|
}{
|
||||||
{m.Alert{Command: []string{"echo", "test"}}, nil, "Command only"},
|
{Alert{Command: []string{"echo", "test"}}, true, "Command only"},
|
||||||
{m.Alert{ShellCommand: "echo test"}, nil, "CommandShell only"},
|
{Alert{CommandShell: "echo test"}, true, "CommandShell only"},
|
||||||
{m.Alert{Command: []string{"echo", "test"}, ShellCommand: "echo test"}, m.ErrInvalidAlert, "Both commands"},
|
{Alert{}, false, "No commands"},
|
||||||
{m.Alert{}, m.ErrInvalidAlert, "No commands"},
|
{
|
||||||
|
Alert{Command: []string{"echo", "test"}, CommandShell: "echo test"},
|
||||||
|
false,
|
||||||
|
"Both commands",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
c := c
|
log.Printf("Testing case %s", c.name)
|
||||||
|
actual := c.alert.IsValid()
|
||||||
t.Run(c.name, func(t *testing.T) {
|
if actual != c.expected {
|
||||||
t.Parallel()
|
t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expected, actual)
|
||||||
|
log.Printf("Case failed: %s", c.name)
|
||||||
actual := c.alert.Validate()
|
}
|
||||||
hasErr := (actual != nil)
|
log.Println("-----")
|
||||||
expectErr := (c.expected != nil)
|
|
||||||
|
|
||||||
if hasErr != expectErr || !errors.Is(actual, c.expected) {
|
|
||||||
t.Errorf("expected=%t actual=%t", c.expected, actual)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAlertSend(t *testing.T) {
|
func TestAlertSend(t *testing.T) {
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
alert m.Alert
|
alert Alert
|
||||||
notice m.AlertNotice
|
notice AlertNotice
|
||||||
expectedOutput string
|
expectedOutput string
|
||||||
expectErr bool
|
expectErr bool
|
||||||
name string
|
name string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
m.Alert{Command: []string{"echo", "{{.MonitorName}}"}},
|
Alert{Command: []string{"echo", "{{.MonitorName}}"}},
|
||||||
m.AlertNotice{MonitorName: "test"},
|
AlertNotice{MonitorName: "test"},
|
||||||
"test\n",
|
"test\n",
|
||||||
false,
|
false,
|
||||||
"Command with template",
|
"Command with template",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
m.Alert{ShellCommand: "echo {{.MonitorName}}"},
|
Alert{CommandShell: "echo {{.MonitorName}}"},
|
||||||
m.AlertNotice{MonitorName: "test"},
|
AlertNotice{MonitorName: "test"},
|
||||||
"test\n",
|
"test\n",
|
||||||
false,
|
false,
|
||||||
"Command shell with template",
|
"Command shell with template",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
m.Alert{Command: []string{"echo", "{{.Bad}}"}},
|
Alert{Command: []string{"echo", "{{.Bad}}"}},
|
||||||
m.AlertNotice{MonitorName: "test"},
|
AlertNotice{MonitorName: "test"},
|
||||||
"",
|
"",
|
||||||
true,
|
true,
|
||||||
"Command with bad template",
|
"Command with bad template",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
m.Alert{ShellCommand: "echo {{.Bad}}"},
|
Alert{CommandShell: "echo {{.Bad}}"},
|
||||||
m.AlertNotice{MonitorName: "test"},
|
AlertNotice{MonitorName: "test"},
|
||||||
"",
|
"",
|
||||||
true,
|
true,
|
||||||
"Command shell with bad template",
|
"Command shell with bad template",
|
||||||
@@ -77,64 +71,51 @@ func TestAlertSend(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
c := c
|
log.Printf("Testing case %s", c.name)
|
||||||
|
c.alert.BuildTemplates()
|
||||||
t.Run(c.name, func(t *testing.T) {
|
output, err := c.alert.Send(c.notice)
|
||||||
t.Parallel()
|
hasErr := (err != nil)
|
||||||
|
if output != c.expectedOutput {
|
||||||
err := c.alert.BuildTemplates()
|
t.Errorf("Send(%v output), expected=%v actual=%v", c.name, c.expectedOutput, output)
|
||||||
if err != nil {
|
log.Printf("Case failed: %s", c.name)
|
||||||
t.Errorf("Send(%v output), error building templates: %v", c.name, err)
|
}
|
||||||
}
|
if hasErr != c.expectErr {
|
||||||
|
t.Errorf("Send(%v err), expected=%v actual=%v", c.name, "Err", err)
|
||||||
output, err := c.alert.Send(c.notice)
|
log.Printf("Case failed: %s", c.name)
|
||||||
hasErr := (err != nil)
|
}
|
||||||
|
log.Println("-----")
|
||||||
if output != c.expectedOutput {
|
|
||||||
t.Errorf("Send(%v output), expected=%v actual=%v", c.name, c.expectedOutput, output)
|
|
||||||
}
|
|
||||||
|
|
||||||
if hasErr != c.expectErr {
|
|
||||||
t.Errorf("Send(%v err), expected=%v actual=%v", c.name, "Err", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAlertSendNoTemplates(t *testing.T) {
|
func TestAlertSendNoTemplates(t *testing.T) {
|
||||||
alert := m.Alert{}
|
alert := Alert{}
|
||||||
notice := m.AlertNotice{}
|
notice := AlertNotice{}
|
||||||
|
|
||||||
output, err := alert.Send(notice)
|
output, err := alert.Send(notice)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Errorf("Send(no template), expected=%v actual=%v", "Err", output)
|
t.Errorf("Send(no template), expected=%v actual=%v", "Err", output)
|
||||||
}
|
}
|
||||||
|
log.Println("-----")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAlertBuildTemplate(t *testing.T) {
|
func TestAlertBuildTemplate(t *testing.T) {
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
alert m.Alert
|
alert Alert
|
||||||
expectErr bool
|
expectErr bool
|
||||||
name string
|
name string
|
||||||
}{
|
}{
|
||||||
{m.Alert{Command: []string{"echo", "test"}}, false, "Command only"},
|
{Alert{Command: []string{"echo", "test"}}, false, "Command only"},
|
||||||
{m.Alert{ShellCommand: "echo test"}, false, "CommandShell only"},
|
{Alert{CommandShell: "echo test"}, false, "CommandShell only"},
|
||||||
{m.Alert{}, true, "No commands"},
|
{Alert{}, true, "No commands"},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
c := c
|
log.Printf("Testing case %s", c.name)
|
||||||
|
err := c.alert.BuildTemplates()
|
||||||
t.Run(c.name, func(t *testing.T) {
|
hasErr := (err != nil)
|
||||||
t.Parallel()
|
if hasErr != c.expectErr {
|
||||||
|
t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expectErr, err)
|
||||||
err := c.alert.BuildTemplates()
|
log.Printf("Case failed: %s", c.name)
|
||||||
hasErr := (err != nil)
|
}
|
||||||
|
log.Println("-----")
|
||||||
if hasErr != c.expectErr {
|
|
||||||
t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expectErr, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,126 +2,67 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"io/ioutil"
|
||||||
"time"
|
"log"
|
||||||
|
"os"
|
||||||
|
|
||||||
"git.iamthefij.com/iamthefij/slog"
|
"gopkg.in/yaml.v2"
|
||||||
"github.com/hashicorp/hcl/v2/hclsimple"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
ErrLoadingConfig = errors.New("Failed to load or parse configuration")
|
|
||||||
ErrConfigInit = errors.New("Failed to initialize configuration")
|
|
||||||
ErrInvalidConfig = errors.New("Invalid configuration")
|
|
||||||
ErrNoAlerts = errors.New("No alerts provided")
|
|
||||||
ErrInvalidAlert = errors.New("Invalid alert configuration")
|
|
||||||
ErrNoMonitors = errors.New("No monitors provided")
|
|
||||||
ErrInvalidMonitor = errors.New("Invalid monitor configuration")
|
|
||||||
ErrUnknownAlert = errors.New("Unknown alert")
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Config type is contains all provided user configuration
|
// Config type is contains all provided user configuration
|
||||||
type Config struct {
|
type Config struct {
|
||||||
CheckIntervalStr string `hcl:"check_interval"`
|
CheckInterval int64 `yaml:"check_interval"`
|
||||||
CheckInterval time.Duration
|
Monitors []*Monitor
|
||||||
|
Alerts map[string]*Alert
|
||||||
DefaultAlertAfter int `hcl:"default_alert_after,optional"`
|
|
||||||
DefaultAlertEvery *int `hcl:"default_alert_every,optional"`
|
|
||||||
DefaultAlertDown []string `hcl:"default_alert_down,optional"`
|
|
||||||
DefaultAlertUp []string `hcl:"default_alert_up,optional"`
|
|
||||||
Monitors []*Monitor `hcl:"monitor,block"`
|
|
||||||
Alerts []*Alert `hcl:"alert,block"`
|
|
||||||
|
|
||||||
alertLookup map[string]*Alert
|
|
||||||
}
|
|
||||||
|
|
||||||
// Init performs extra initialization on top of loading the config from file
|
|
||||||
func (config *Config) Init() (err error) {
|
|
||||||
config.CheckInterval, err = time.ParseDuration(config.CheckIntervalStr)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to parse top level check_interval duration: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if config.DefaultAlertAfter == 0 {
|
|
||||||
minAlertAfter := 1
|
|
||||||
config.DefaultAlertAfter = minAlertAfter
|
|
||||||
}
|
|
||||||
|
|
||||||
if config.DefaultAlertEvery == nil {
|
|
||||||
defaultDefaultAlertEvery := -1
|
|
||||||
config.DefaultAlertEvery = &defaultDefaultAlertEvery
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, monitor := range config.Monitors {
|
|
||||||
if err = monitor.Init(
|
|
||||||
config.DefaultAlertAfter,
|
|
||||||
config.DefaultAlertEvery,
|
|
||||||
config.DefaultAlertDown,
|
|
||||||
config.DefaultAlertUp,
|
|
||||||
); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
err = config.BuildAllTemplates()
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsValid checks config validity and returns true if valid
|
// IsValid checks config validity and returns true if valid
|
||||||
func (config Config) IsValid() error {
|
func (config Config) IsValid() (isValid bool) {
|
||||||
var err error
|
isValid = true
|
||||||
|
|
||||||
// Validate alerts
|
|
||||||
if len(config.Alerts) == 0 {
|
|
||||||
err = errors.Join(err, ErrNoAlerts)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, alert := range config.Alerts {
|
|
||||||
err = errors.Join(err, alert.Validate())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate monitors
|
// Validate monitors
|
||||||
if len(config.Monitors) == 0 {
|
if config.Monitors == nil || len(config.Monitors) == 0 {
|
||||||
err = errors.Join(err, ErrNoMonitors)
|
log.Printf("ERROR: Invalid monitor configuration: Must provide at least one monitor")
|
||||||
|
isValid = false
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, monitor := range config.Monitors {
|
for _, monitor := range config.Monitors {
|
||||||
err = errors.Join(err, monitor.Validate())
|
if !monitor.IsValid() {
|
||||||
|
log.Printf("ERROR: Invalid monitor configuration: %s", monitor.Name)
|
||||||
|
isValid = false
|
||||||
|
}
|
||||||
// Check that all Monitor alerts actually exist
|
// Check that all Monitor alerts actually exist
|
||||||
for _, isUp := range []bool{true, false} {
|
for _, isUp := range []bool{true, false} {
|
||||||
for _, alertName := range monitor.GetAlertNames(isUp) {
|
for _, alertName := range monitor.GetAlertNames(isUp) {
|
||||||
if _, ok := config.GetAlert(alertName); !ok {
|
if _, ok := config.Alerts[alertName]; !ok {
|
||||||
err = errors.Join(
|
log.Printf(
|
||||||
err,
|
"ERROR: Invalid monitor configuration: %s. Unknown alert %s",
|
||||||
fmt.Errorf("%w: %s. %w: %s", ErrInvalidMonitor, monitor.Name, ErrUnknownAlert, alertName),
|
monitor.Name, alertName,
|
||||||
)
|
)
|
||||||
|
isValid = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return err
|
// Validate alerts
|
||||||
}
|
if config.Alerts == nil || len(config.Alerts) == 0 {
|
||||||
|
log.Printf("ERROR: Invalid alert configuration: Must provide at least one alert")
|
||||||
// GetAlert returns an alert by name
|
isValid = false
|
||||||
func (c Config) GetAlert(name string) (*Alert, bool) {
|
}
|
||||||
if c.alertLookup == nil {
|
for _, alert := range config.Alerts {
|
||||||
c.alertLookup = map[string]*Alert{}
|
if !alert.IsValid() {
|
||||||
for _, alert := range c.Alerts {
|
log.Printf("ERROR: Invalid alert configuration: %s", alert.Name)
|
||||||
c.alertLookup[alert.Name] = alert
|
isValid = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
v, ok := c.alertLookup[name]
|
return
|
||||||
|
|
||||||
return v, ok
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// BuildAllTemplates builds all alert templates
|
// Init performs extra initialization on top of loading the config from file
|
||||||
func (c *Config) BuildAllTemplates() (err error) {
|
func (config *Config) Init() (err error) {
|
||||||
for _, alert := range c.Alerts {
|
for name, alert := range config.Alerts {
|
||||||
|
alert.Name = name
|
||||||
if err = alert.BuildTemplates(); err != nil {
|
if err = alert.BuildTemplates(); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -131,23 +72,30 @@ func (c *Config) BuildAllTemplates() (err error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// LoadConfig will read config from the given path and parse it
|
// LoadConfig will read config from the given path and parse it
|
||||||
func LoadConfig(filePath string) (Config, error) {
|
func LoadConfig(filePath string) (config Config, err error) {
|
||||||
var config Config
|
data, err := ioutil.ReadFile(filePath)
|
||||||
|
if err != nil {
|
||||||
if err := hclsimple.DecodeFile(filePath, nil, &config); err != nil {
|
return
|
||||||
return config, errors.Join(ErrLoadingConfig, err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
slog.Debugf("Config values:\n%v\n", config)
|
// TODO: Decide if this is better expanded here, or only when executing
|
||||||
|
envExpanded := os.ExpandEnv(string(data))
|
||||||
|
err = yaml.Unmarshal([]byte(envExpanded), &config)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if LogDebug {
|
||||||
|
log.Printf("DEBUG: Config values:\n%v\n", config)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !config.IsValid() {
|
||||||
|
err = errors.New("Invalid configuration")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// Finish initializing configuration
|
// Finish initializing configuration
|
||||||
if err := config.Init(); err != nil {
|
err = config.Init()
|
||||||
return config, errors.Join(ErrConfigInit, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := config.IsValid(); err != nil {
|
return
|
||||||
return config, errors.Join(ErrInvalidConfig, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return config, nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
+18
-180
@@ -1,193 +1,31 @@
|
|||||||
package main_test
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"log"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
m "git.iamthefij.com/iamthefij/minitor-go/v2"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestLoadConfig(t *testing.T) {
|
func TestLoadConfig(t *testing.T) {
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
configPath string
|
configPath string
|
||||||
expectedErr error
|
expectErr bool
|
||||||
name string
|
name string
|
||||||
}{
|
}{
|
||||||
{"./test/does-not-exist", m.ErrLoadingConfig, "Invalid config path"},
|
{"./test/valid-config.yml", false, "Valid config file"},
|
||||||
{"./test/invalid-config-wrong-hcl-type.hcl", m.ErrLoadingConfig, "Incorrect HCL type"},
|
{"./test/does-not-exist", true, "Invalid config path"},
|
||||||
{"./test/invalid-config-missing-alerts.hcl", m.ErrNoAlerts, "Invalid config missing alerts"},
|
{"./test/invalid-config-type.yml", true, "Invalid config type for key"},
|
||||||
{"./test/invalid-config-missing-alerts.hcl", m.ErrInvalidConfig, "Invalid config general"},
|
{"./test/invalid-config-missing-alerts.yml", true, "Invalid config missing alerts"},
|
||||||
{"./test/invalid-config-invalid-duration.hcl", m.ErrConfigInit, "Invalid config type for key"},
|
{"./test/invalid-config-unknown-alert.yml", true, "Invalid config unknown alert"},
|
||||||
{"./test/invalid-config-unknown-alert.hcl", m.ErrUnknownAlert, "Invalid config unknown alert"},
|
|
||||||
{"./test/valid-config-default-values.hcl", nil, "Valid config file with default values"},
|
|
||||||
{"./test/valid-config.hcl", nil, "Valid config file"},
|
|
||||||
}
|
|
||||||
for _, c := range cases {
|
|
||||||
c := c
|
|
||||||
|
|
||||||
t.Run(c.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, err := m.LoadConfig(c.configPath)
|
|
||||||
hasErr := (err != nil)
|
|
||||||
expectErr := (c.expectedErr != nil)
|
|
||||||
|
|
||||||
if hasErr != expectErr || !errors.Is(err, c.expectedErr) {
|
|
||||||
t.Errorf("LoadConfig(%v), expected_error=%v actual=%v", c.name, c.expectedErr, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDefaultConfig(t *testing.T) {
|
|
||||||
cases := []struct {
|
|
||||||
configPath string
|
|
||||||
expectedResult m.Config
|
|
||||||
name string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
"./test/valid-config-default-values.hcl",
|
|
||||||
m.Config{
|
|
||||||
CheckInterval: 1 * time.Second,
|
|
||||||
DefaultAlertAfter: 2,
|
|
||||||
DefaultAlertEvery: Ptr(0),
|
|
||||||
DefaultAlertDown: []string{"log_command"},
|
|
||||||
},
|
|
||||||
"override defaults",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"./test/valid-config.hcl",
|
|
||||||
m.Config{
|
|
||||||
CheckInterval: 30 * time.Second,
|
|
||||||
DefaultAlertAfter: 1,
|
|
||||||
DefaultAlertEvery: Ptr(-1),
|
|
||||||
DefaultAlertDown: []string{},
|
|
||||||
},
|
|
||||||
"default defaults",
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
c := c
|
log.Printf("Testing case %s", c.name)
|
||||||
|
_, err := LoadConfig(c.configPath)
|
||||||
t.Run(c.name, func(t *testing.T) {
|
hasErr := (err != nil)
|
||||||
t.Parallel()
|
if hasErr != c.expectErr {
|
||||||
|
t.Errorf("LoadConfig(%v), expected=%v actual=%v", c.name, "Err", err)
|
||||||
config, err := m.LoadConfig(c.configPath)
|
log.Printf("Case failed: %s", c.name)
|
||||||
if err != nil {
|
}
|
||||||
t.Errorf("Got error when loading config file %q: %s", c.configPath, err)
|
log.Println("-----")
|
||||||
}
|
|
||||||
|
|
||||||
// Test Config has default values
|
|
||||||
if config.DefaultAlertAfter != c.expectedResult.DefaultAlertAfter {
|
|
||||||
t.Errorf("Got unexpected DefaultAlertAfter from file %q: expected=%v actual=%v", c.configPath, c.expectedResult.DefaultAlertAfter, config.DefaultAlertAfter)
|
|
||||||
}
|
|
||||||
|
|
||||||
if *config.DefaultAlertEvery != *c.expectedResult.DefaultAlertEvery {
|
|
||||||
t.Errorf("Got unexpected DefaultAlertEvery from file %q: expected=%v actual=%v", c.configPath, *c.expectedResult.DefaultAlertEvery, *config.DefaultAlertEvery)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !m.EqualSliceString(config.DefaultAlertUp, c.expectedResult.DefaultAlertUp) {
|
|
||||||
t.Errorf("Got unexpected DefaultAlertUp from file %q: expected=%v actual=%v", c.configPath, c.expectedResult.DefaultAlertUp, config.DefaultAlertUp)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !m.EqualSliceString(config.DefaultAlertDown, c.expectedResult.DefaultAlertDown) {
|
|
||||||
t.Errorf("Got unexpected DefaultAlertDown from file %q: expected=%v actual=%v", c.configPath, c.expectedResult.DefaultAlertDown, config.DefaultAlertDown)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check that monitor defaults propagate
|
|
||||||
var defaultMonitor *m.Monitor
|
|
||||||
for _, monitor := range config.Monitors {
|
|
||||||
if monitor.Name == "Default" {
|
|
||||||
defaultMonitor = monitor
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if defaultMonitor == nil {
|
|
||||||
t.Errorf("failed to find default monitor in %q", c.configPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
if defaultMonitor.AlertAfter != c.expectedResult.DefaultAlertAfter {
|
|
||||||
t.Errorf("Got unexpected AlertAfter from file %q: expected=%v actual=%v", c.configPath, c.expectedResult.DefaultAlertAfter, defaultMonitor.AlertAfter)
|
|
||||||
}
|
|
||||||
|
|
||||||
if *defaultMonitor.AlertEvery != *c.expectedResult.DefaultAlertEvery {
|
|
||||||
t.Errorf("Got unexpected AlertEvery from file %q: expected=%v actual=%v", c.configPath, *c.expectedResult.DefaultAlertEvery, *defaultMonitor.AlertEvery)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !m.EqualSliceString(defaultMonitor.AlertUp, c.expectedResult.DefaultAlertUp) {
|
|
||||||
t.Errorf("Got unexpected AlertUp from file %q: expected=%v actual=%v", c.configPath, c.expectedResult.DefaultAlertUp, defaultMonitor.AlertUp)
|
|
||||||
}
|
|
||||||
|
|
||||||
// NOTE: Can't compare AlertDown because default is empty and that is invalid
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestMultiLineConfig is a more complicated test stepping through the parsing
|
|
||||||
// and execution of mutli-line strings presented in YAML
|
|
||||||
func TestMultiLineConfig(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
config, err := m.LoadConfig("./test/valid-verify-multi-line.hcl")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("TestMultiLineConfig(load), expected=no_error actual=%v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run("Test Monitor with Indented Multi-Line String", func(t *testing.T) {
|
|
||||||
// Verify indented heredoc is as expected
|
|
||||||
expected := "echo 'Some string with stuff'\necho \"<angle brackets>\"\nexit 1\n"
|
|
||||||
actual := config.Monitors[0].ShellCommand
|
|
||||||
|
|
||||||
if expected != actual {
|
|
||||||
t.Error("Heredoc mismatch")
|
|
||||||
t.Errorf("string expected=`%v`", expected)
|
|
||||||
t.Errorf("string actual =`%v`", actual)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run the monitor and verify the output
|
|
||||||
_, notice := config.Monitors[0].Check()
|
|
||||||
if notice == nil {
|
|
||||||
t.Fatal("Did not receive an alert notice and should have")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify the output of the monitor is as expected
|
|
||||||
expected = "Some string with stuff\n<angle brackets>\n"
|
|
||||||
actual = notice.LastCheckOutput
|
|
||||||
|
|
||||||
if expected != actual {
|
|
||||||
t.Error("Output mismatch")
|
|
||||||
t.Errorf("string expected=`%v`", expected)
|
|
||||||
t.Errorf("string actual =`%v`", actual)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("Test Alert with Multi-Line String", func(t *testing.T) {
|
|
||||||
alert, ok := config.GetAlert("log_shell")
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("Could not find expected alert 'log_shell'")
|
|
||||||
}
|
|
||||||
|
|
||||||
expected := " echo 'Some string with stuff'\n echo '<angle brackets>'\n"
|
|
||||||
actual := alert.ShellCommand
|
|
||||||
|
|
||||||
if expected != actual {
|
|
||||||
t.Error("Heredoc mismatch")
|
|
||||||
t.Errorf("string expected=`%v`", expected)
|
|
||||||
t.Errorf("string actual =`%v`", actual)
|
|
||||||
}
|
|
||||||
|
|
||||||
actual, err = alert.Send(m.AlertNotice{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal("Execution of alert failed")
|
|
||||||
}
|
|
||||||
|
|
||||||
expected = "Some string with stuff\n<angle brackets>\n"
|
|
||||||
if expected != actual {
|
|
||||||
t.Error("Output mismatch")
|
|
||||||
t.Errorf("string expected=`%v`", expected)
|
|
||||||
t.Errorf("string actual =`%v`", actual)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,27 +1,8 @@
|
|||||||
module git.iamthefij.com/iamthefij/minitor-go/v2
|
module git.iamthefij.com/iamthefij/minitor-go
|
||||||
|
|
||||||
go 1.25.0
|
go 1.12
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.iamthefij.com/iamthefij/slog v1.3.0
|
github.com/prometheus/client_golang v1.2.1
|
||||||
github.com/hashicorp/hcl/v2 v2.11.1
|
gopkg.in/yaml.v2 v2.2.4
|
||||||
github.com/prometheus/client_golang v1.23.2
|
|
||||||
)
|
|
||||||
|
|
||||||
require (
|
|
||||||
github.com/agext/levenshtein v1.2.1 // indirect
|
|
||||||
github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
|
|
||||||
github.com/beorn7/perks v1.0.1 // indirect
|
|
||||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
|
||||||
github.com/google/go-cmp v0.7.0 // indirect
|
|
||||||
github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 // indirect
|
|
||||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
|
||||||
github.com/prometheus/client_model v0.6.2 // indirect
|
|
||||||
github.com/prometheus/common v0.66.1 // indirect
|
|
||||||
github.com/prometheus/procfs v0.16.1 // indirect
|
|
||||||
github.com/zclconf/go-cty v1.8.0 // indirect
|
|
||||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
|
||||||
golang.org/x/sys v0.35.0 // indirect
|
|
||||||
golang.org/x/text v0.28.0 // indirect
|
|
||||||
google.golang.org/protobuf v1.36.8 // indirect
|
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,96 +1,78 @@
|
|||||||
git.iamthefij.com/iamthefij/slog v1.3.0 h1:4Hu5PQvDrW5e3FrTS3q2iIXW0iPvhNY/9qJsqDR3K3I=
|
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||||
git.iamthefij.com/iamthefij/slog v1.3.0/go.mod h1:1RUj4hcCompZkAxXCRfUX786tb3cM/Zpkn97dGfUfbg=
|
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||||
github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8=
|
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||||
github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
|
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||||
github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM=
|
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||||
github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk=
|
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||||
github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw=
|
|
||||||
github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
|
|
||||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA=
|
||||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
|
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||||
github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
|
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||||
github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||||
|
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||||
|
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||||
|
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||||
|
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||||
github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
|
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
|
||||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||||
github.com/hashicorp/hcl/v2 v2.11.1 h1:yTyWcXcm9XB0TEkyU/JCRU6rYy4K+mgLtzn2wlrJbcc=
|
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||||
github.com/hashicorp/hcl/v2 v2.11.1/go.mod h1:FwWsfWEjyV/CMj8s/gqAuiviY72rJ1/oayI9WftqcKg=
|
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
|
||||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||||
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
|
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 h1:DpOJ2HYzCv8LZP15IdmG+YdwD2luVPHITV96TkirNBM=
|
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
|
|
||||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
|
||||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||||
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
|
||||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI=
|
||||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U=
|
||||||
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
|
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||||
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
|
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||||
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
|
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
|
||||||
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
|
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
|
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
|
||||||
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
|
github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
|
||||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||||
github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||||
|
github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
|
||||||
|
github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
|
||||||
|
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||||
|
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||||
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
|
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||||
github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
|
|
||||||
github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
|
|
||||||
github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
|
|
||||||
github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8=
|
|
||||||
github.com/zclconf/go-cty v1.8.0 h1:s4AvqaeQzJIu3ndv4gVIhplVD0krU+bgrcLSVUnaWuA=
|
|
||||||
github.com/zclconf/go-cty v1.8.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk=
|
|
||||||
github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8=
|
|
||||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
|
||||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
|
||||||
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
|
|
||||||
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
|
|
||||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
|
|
||||||
golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
|
||||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||||
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
|
||||||
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
|
|
||||||
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
|
|
||||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
|
||||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
|
||||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
|
||||||
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
|
|
||||||
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
|
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
|
||||||
|
|||||||
@@ -1,16 +1,16 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"log"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.iamthefij.com/iamthefij/slog"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
// LogDebug will control whether debug messsages should be logged
|
||||||
|
LogDebug = false
|
||||||
|
|
||||||
// ExportMetrics will track whether or not we want to export metrics to prometheus
|
// ExportMetrics will track whether or not we want to export metrics to prometheus
|
||||||
ExportMetrics = false
|
ExportMetrics = false
|
||||||
// MetricsPort is the port to expose metrics on
|
// MetricsPort is the port to expose metrics on
|
||||||
@@ -20,67 +20,57 @@ var (
|
|||||||
|
|
||||||
// version of minitor being run
|
// version of minitor being run
|
||||||
version = "dev"
|
version = "dev"
|
||||||
|
|
||||||
errUnknownAlert = errors.New("unknown alert")
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func SendAlerts(config *Config, monitor *Monitor, alertNotice *AlertNotice) error {
|
func checkMonitors(config *Config) error {
|
||||||
slog.Debugf("Received an alert notice from %s", alertNotice.MonitorName)
|
|
||||||
alertNames := monitor.GetAlertNames(alertNotice.IsUp)
|
|
||||||
|
|
||||||
if alertNames == nil {
|
|
||||||
// This should only happen for a recovery alert. AlertDown is validated not empty
|
|
||||||
slog.Warningf(
|
|
||||||
"Received alert, but no alert mechanisms exist. MonitorName=%s IsUp=%t",
|
|
||||||
alertNotice.MonitorName, alertNotice.IsUp,
|
|
||||||
)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, alertName := range alertNames {
|
|
||||||
if alert, ok := config.GetAlert(alertName); ok {
|
|
||||||
output, err := alert.Send(*alertNotice)
|
|
||||||
if err != nil {
|
|
||||||
slog.Errorf(
|
|
||||||
"Alert '%s' failed. result=%v: output=%s",
|
|
||||||
alert.Name,
|
|
||||||
err,
|
|
||||||
output,
|
|
||||||
)
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Count alert metrics
|
|
||||||
Metrics.CountAlert(monitor.Name, alert.Name)
|
|
||||||
} else {
|
|
||||||
// This case should never actually happen since we validate against it
|
|
||||||
slog.Errorf("Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
|
|
||||||
|
|
||||||
return fmt.Errorf("unknown alert for monitor %s: %s: %w", alertNotice.MonitorName, alertName, errUnknownAlert)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func CheckMonitors(config *Config) error {
|
|
||||||
// TODO: Run this in goroutines and capture exceptions
|
|
||||||
for _, monitor := range config.Monitors {
|
for _, monitor := range config.Monitors {
|
||||||
if monitor.ShouldCheck() {
|
if monitor.ShouldCheck() {
|
||||||
success, alertNotice := monitor.Check()
|
success, alertNotice := monitor.Check()
|
||||||
|
|
||||||
hasAlert := alertNotice != nil
|
hasAlert := alertNotice != nil
|
||||||
|
|
||||||
// Track status metrics
|
// Track status metrics
|
||||||
Metrics.SetMonitorStatus(monitor.Name, monitor.IsUp())
|
Metrics.SetMonitorStatus(monitor.Name, success)
|
||||||
Metrics.CountCheck(monitor.Name, success, monitor.LastCheckMilliseconds(), hasAlert)
|
Metrics.CountCheck(monitor.Name, success, hasAlert)
|
||||||
|
|
||||||
|
// Should probably consider refactoring everything below here
|
||||||
if alertNotice != nil {
|
if alertNotice != nil {
|
||||||
err := SendAlerts(config, monitor, alertNotice)
|
if LogDebug {
|
||||||
// If there was an error in sending an alert, exit early and bubble it up
|
log.Printf("DEBUG: Recieved an alert notice from %s", alertNotice.MonitorName)
|
||||||
if err != nil {
|
}
|
||||||
return err
|
alertNames := monitor.GetAlertNames(alertNotice.IsUp)
|
||||||
|
if alertNames == nil {
|
||||||
|
// This should only happen for a recovery alert. AlertDown is validated not empty
|
||||||
|
log.Printf(
|
||||||
|
"WARNING: Recieved alert, but no alert mechanisms exist. MonitorName=%s IsUp=%t",
|
||||||
|
alertNotice.MonitorName, alertNotice.IsUp,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
for _, alertName := range alertNames {
|
||||||
|
if alert, ok := config.Alerts[alertName]; ok {
|
||||||
|
output, err := alert.Send(*alertNotice)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf(
|
||||||
|
"ERROR: Alert '%s' failed. result=%v: output=%s",
|
||||||
|
alert.Name,
|
||||||
|
err,
|
||||||
|
output,
|
||||||
|
)
|
||||||
|
return fmt.Errorf(
|
||||||
|
"Unsuccessfully triggered alert '%s'. "+
|
||||||
|
"Crashing to avoid false negatives: %v",
|
||||||
|
alert.Name,
|
||||||
|
err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count alert metrics
|
||||||
|
Metrics.CountAlert(monitor.Name, alert.Name)
|
||||||
|
} else {
|
||||||
|
// This case should never actually happen since we validate against it
|
||||||
|
log.Printf("ERROR: Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
|
||||||
|
return fmt.Errorf("Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -89,75 +79,39 @@ func CheckMonitors(config *Config) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func SendStartupAlerts(config *Config, alertNames []string) error {
|
|
||||||
for _, alertName := range alertNames {
|
|
||||||
var err error
|
|
||||||
|
|
||||||
alert, ok := config.GetAlert(alertName)
|
|
||||||
if !ok {
|
|
||||||
err = fmt.Errorf("unknown alert %s: %w", alertName, errUnknownAlert)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err == nil {
|
|
||||||
_, err = alert.Send(AlertNotice{
|
|
||||||
AlertCount: 0,
|
|
||||||
FailureCount: 0,
|
|
||||||
IsUp: true,
|
|
||||||
LastSuccess: time.Now(),
|
|
||||||
MonitorName: fmt.Sprintf("First Run Alert Test: %s", alert.Name),
|
|
||||||
LastCheckOutput: "",
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
showVersion := flag.Bool("version", false, "Display the version of minitor and exit")
|
// Get debug flag
|
||||||
configPath := flag.String("config", "config.hcl", "Alternate configuration path (default: config.hcl)")
|
flag.BoolVar(&LogDebug, "debug", false, "Enables debug logs (default: false)")
|
||||||
startupAlerts := flag.String("startup-alerts", "", "List of alerts to run on startup. This can help determine unhealthy alerts early on. (default \"\")")
|
|
||||||
|
|
||||||
flag.BoolVar(&slog.DebugLevel, "debug", false, "Enables debug logs (default: false)")
|
|
||||||
flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)")
|
flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)")
|
||||||
flag.IntVar(&MetricsPort, "metrics-port", MetricsPort, "The port that Prometheus metrics should be exported on, if enabled. (default: 8080)")
|
var showVersion = flag.Bool("version", false, "Display the version of minitor and exit")
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
// Print version if flag is provided
|
// Print version if flag is provided
|
||||||
if *showVersion {
|
if *showVersion {
|
||||||
fmt.Println("Minitor version:", version)
|
log.Println("Minitor version:", version)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load configuration
|
// Load configuration
|
||||||
config, err := LoadConfig(*configPath)
|
config, err := LoadConfig("config.yml")
|
||||||
slog.OnErrFatalf(err, "Error loading config")
|
if err != nil {
|
||||||
|
log.Fatalf("Error loading config: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
// Serve metrics exporter, if specified
|
// Serve metrics exporter, if specified
|
||||||
if ExportMetrics {
|
if ExportMetrics {
|
||||||
slog.Infof("Exporting metrics to Prometheus on port %d", MetricsPort)
|
log.Println("INFO: Exporting metrics to Prometheus")
|
||||||
|
|
||||||
go ServeMetrics()
|
go ServeMetrics()
|
||||||
}
|
}
|
||||||
|
|
||||||
if *startupAlerts != "" {
|
|
||||||
alertNames := strings.Split(*startupAlerts, ",")
|
|
||||||
|
|
||||||
err = SendStartupAlerts(&config, alertNames)
|
|
||||||
|
|
||||||
slog.OnErrPanicf(err, "Error running startup alerts")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Start main loop
|
// Start main loop
|
||||||
for {
|
for {
|
||||||
err = CheckMonitors(&config)
|
err = checkMonitors(&config)
|
||||||
slog.OnErrPanicf(err, "Error checking monitors")
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
time.Sleep(config.CheckInterval)
|
sleepTime := time.Duration(config.CheckInterval) * time.Second
|
||||||
|
time.Sleep(sleepTime)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+89
-167
@@ -1,192 +1,114 @@
|
|||||||
package main_test
|
package main
|
||||||
|
|
||||||
import (
|
import "testing"
|
||||||
"testing"
|
|
||||||
|
|
||||||
m "git.iamthefij.com/iamthefij/minitor-go/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
func Ptr[T any](v T) *T {
|
|
||||||
return &v
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestCheckConfig tests the checkConfig function
|
|
||||||
// It also tests results for potentially invalid configuration. For example, no alerts
|
|
||||||
func TestCheckMonitors(t *testing.T) {
|
func TestCheckMonitors(t *testing.T) {
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
config m.Config
|
config Config
|
||||||
expectFailureError bool
|
expectErr bool
|
||||||
expectRecoverError bool
|
name string
|
||||||
name string
|
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
config: m.Config{
|
config: Config{},
|
||||||
CheckIntervalStr: "1s",
|
expectErr: false,
|
||||||
Monitors: []*m.Monitor{
|
name: "Empty",
|
||||||
{
|
|
||||||
Name: "Success",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
expectFailureError: false,
|
|
||||||
expectRecoverError: false,
|
|
||||||
name: "No alerts",
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
config: m.Config{
|
config: Config{
|
||||||
CheckIntervalStr: "1s",
|
Monitors: []*Monitor{
|
||||||
Monitors: []*m.Monitor{
|
&Monitor{
|
||||||
{
|
Name: "Success",
|
||||||
Name: "Failure",
|
|
||||||
AlertDown: []string{"unknown"},
|
|
||||||
AlertUp: []string{"unknown"},
|
|
||||||
AlertAfter: 1,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
expectFailureError: true,
|
|
||||||
expectRecoverError: true,
|
|
||||||
name: "Unknown alerts",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
config: m.Config{
|
|
||||||
CheckIntervalStr: "1s",
|
|
||||||
Monitors: []*m.Monitor{
|
|
||||||
{
|
|
||||||
Name: "Failure",
|
|
||||||
AlertDown: []string{"good"},
|
|
||||||
AlertUp: []string{"good"},
|
|
||||||
AlertAfter: 1,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Alerts: []*m.Alert{{
|
|
||||||
Name: "good",
|
|
||||||
Command: []string{"true"},
|
|
||||||
}},
|
|
||||||
},
|
|
||||||
expectFailureError: false,
|
|
||||||
expectRecoverError: false,
|
|
||||||
name: "Successful alert",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
config: m.Config{
|
|
||||||
CheckIntervalStr: "1s",
|
|
||||||
Monitors: []*m.Monitor{
|
|
||||||
{
|
|
||||||
Name: "Failure",
|
|
||||||
AlertDown: []string{"bad"},
|
|
||||||
AlertUp: []string{"bad"},
|
|
||||||
AlertAfter: 1,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Alerts: []*m.Alert{{
|
|
||||||
Name: "bad",
|
|
||||||
Command: []string{"false"},
|
|
||||||
}},
|
|
||||||
},
|
|
||||||
expectFailureError: true,
|
|
||||||
expectRecoverError: true,
|
|
||||||
name: "Failing alert",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, c := range cases {
|
|
||||||
c := c
|
|
||||||
|
|
||||||
t.Run(c.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
err := c.config.Init()
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("checkMonitors(%s): unexpected error reading config: %v", c.name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, check := range []struct {
|
|
||||||
shellCmd string
|
|
||||||
name string
|
|
||||||
expectErr bool
|
|
||||||
}{
|
|
||||||
{"false", "Failure", c.expectFailureError}, {"true", "Success", c.expectRecoverError},
|
|
||||||
} {
|
|
||||||
// Set the shell command for this check
|
|
||||||
c.config.Monitors[0].ShellCommand = check.shellCmd
|
|
||||||
|
|
||||||
// Run the check
|
|
||||||
err = m.CheckMonitors(&c.config)
|
|
||||||
|
|
||||||
// Check the results
|
|
||||||
if err == nil && check.expectErr {
|
|
||||||
t.Errorf("checkMonitors(%s:%s): Expected error, the code did not error", c.name, check.name)
|
|
||||||
} else if err != nil && !check.expectErr {
|
|
||||||
t.Errorf("checkMonitors(%s:%s): Did not expect an error, but we got one anyway: %v", c.name, check.name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFirstRunAlerts(t *testing.T) {
|
|
||||||
cases := []struct {
|
|
||||||
config m.Config
|
|
||||||
expectErr bool
|
|
||||||
startupAlerts []string
|
|
||||||
name string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
config: m.Config{
|
|
||||||
CheckIntervalStr: "1s",
|
|
||||||
},
|
|
||||||
expectErr: true,
|
|
||||||
startupAlerts: []string{"missing"},
|
|
||||||
name: "Unknown",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
config: m.Config{
|
|
||||||
CheckIntervalStr: "1s",
|
|
||||||
Alerts: []*m.Alert{
|
|
||||||
{
|
|
||||||
Name: "good",
|
|
||||||
Command: []string{"true"},
|
Command: []string{"true"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
expectErr: false,
|
expectErr: false,
|
||||||
startupAlerts: []string{"good"},
|
name: "Monitor success, no alerts",
|
||||||
name: "Successful alert",
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
config: m.Config{
|
config: Config{
|
||||||
CheckIntervalStr: "1s",
|
Monitors: []*Monitor{
|
||||||
Alerts: []*m.Alert{
|
&Monitor{
|
||||||
{
|
Name: "Failure",
|
||||||
|
Command: []string{"false"},
|
||||||
|
AlertAfter: 1,
|
||||||
|
},
|
||||||
|
&Monitor{
|
||||||
|
Name: "Failure",
|
||||||
|
Command: []string{"false"},
|
||||||
|
AlertDown: []string{"unknown"},
|
||||||
|
AlertAfter: 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
expectErr: false,
|
||||||
|
name: "Monitor failure, no and unknown alerts",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
config: Config{
|
||||||
|
Monitors: []*Monitor{
|
||||||
|
&Monitor{
|
||||||
|
Name: "Success",
|
||||||
|
Command: []string{"ls"},
|
||||||
|
alertCount: 1,
|
||||||
|
},
|
||||||
|
&Monitor{
|
||||||
|
Name: "Success",
|
||||||
|
Command: []string{"true"},
|
||||||
|
AlertUp: []string{"unknown"},
|
||||||
|
alertCount: 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
expectErr: false,
|
||||||
|
name: "Monitor recovery, no alerts",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
config: Config{
|
||||||
|
Monitors: []*Monitor{
|
||||||
|
&Monitor{
|
||||||
|
Name: "Failure",
|
||||||
|
Command: []string{"false"},
|
||||||
|
AlertDown: []string{"good"},
|
||||||
|
AlertAfter: 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Alerts: map[string]*Alert{
|
||||||
|
"good": &Alert{
|
||||||
|
Command: []string{"true"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
expectErr: false,
|
||||||
|
name: "Monitor failure, successful alert",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
config: Config{
|
||||||
|
Monitors: []*Monitor{
|
||||||
|
&Monitor{
|
||||||
|
Name: "Failure",
|
||||||
|
Command: []string{"false"},
|
||||||
|
AlertDown: []string{"bad"},
|
||||||
|
AlertAfter: 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Alerts: map[string]*Alert{
|
||||||
|
"bad": &Alert{
|
||||||
Name: "bad",
|
Name: "bad",
|
||||||
Command: []string{"false"},
|
Command: []string{"false"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
expectErr: true,
|
expectErr: true,
|
||||||
startupAlerts: []string{"bad"},
|
name: "Monitor failure, bad alert",
|
||||||
name: "Failed alert",
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
c := c
|
c.config.Init()
|
||||||
|
err := checkMonitors(&c.config)
|
||||||
t.Run(c.name, func(t *testing.T) {
|
if err == nil && c.expectErr {
|
||||||
t.Parallel()
|
t.Errorf("checkMonitors(%s): Expected panic, the code did not panic", c.name)
|
||||||
|
}
|
||||||
err := c.config.Init()
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("sendFirstRunAlerts(%s): unexpected error reading config: %v", c.name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = m.SendStartupAlerts(&c.config, c.startupAlerts)
|
|
||||||
if err == nil && c.expectErr {
|
|
||||||
t.Errorf("sendFirstRunAlerts(%s): Expected error, the code did not error", c.name)
|
|
||||||
} else if err != nil && !c.expectErr {
|
|
||||||
t.Errorf("sendFirstRunAlerts(%s): Did not expect an error, but we got one anyway: %v", c.name, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-17
@@ -19,7 +19,6 @@ import (
|
|||||||
type MinitorMetrics struct {
|
type MinitorMetrics struct {
|
||||||
alertCount *prometheus.CounterVec
|
alertCount *prometheus.CounterVec
|
||||||
checkCount *prometheus.CounterVec
|
checkCount *prometheus.CounterVec
|
||||||
checkTime *prometheus.GaugeVec
|
|
||||||
monitorStatus *prometheus.GaugeVec
|
monitorStatus *prometheus.GaugeVec
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -41,13 +40,6 @@ func NewMetrics() *MinitorMetrics {
|
|||||||
},
|
},
|
||||||
[]string{"monitor", "status", "is_alert"},
|
[]string{"monitor", "status", "is_alert"},
|
||||||
),
|
),
|
||||||
checkTime: prometheus.NewGaugeVec(
|
|
||||||
prometheus.GaugeOpts{
|
|
||||||
Name: "minitor_check_milliseconds",
|
|
||||||
Help: "Time in miliseconds that a check ran for",
|
|
||||||
},
|
|
||||||
[]string{"monitor", "status"},
|
|
||||||
),
|
|
||||||
monitorStatus: prometheus.NewGaugeVec(
|
monitorStatus: prometheus.NewGaugeVec(
|
||||||
prometheus.GaugeOpts{
|
prometheus.GaugeOpts{
|
||||||
Name: "minitor_monitor_up_count",
|
Name: "minitor_monitor_up_count",
|
||||||
@@ -60,7 +52,6 @@ func NewMetrics() *MinitorMetrics {
|
|||||||
// Register newly created metrics
|
// Register newly created metrics
|
||||||
prometheus.MustRegister(metrics.alertCount)
|
prometheus.MustRegister(metrics.alertCount)
|
||||||
prometheus.MustRegister(metrics.checkCount)
|
prometheus.MustRegister(metrics.checkCount)
|
||||||
prometheus.MustRegister(metrics.checkTime)
|
|
||||||
prometheus.MustRegister(metrics.monitorStatus)
|
prometheus.MustRegister(metrics.monitorStatus)
|
||||||
|
|
||||||
return metrics
|
return metrics
|
||||||
@@ -72,12 +63,11 @@ func (metrics *MinitorMetrics) SetMonitorStatus(monitor string, isUp bool) {
|
|||||||
if isUp {
|
if isUp {
|
||||||
val = 1.0
|
val = 1.0
|
||||||
}
|
}
|
||||||
|
|
||||||
metrics.monitorStatus.With(prometheus.Labels{"monitor": monitor}).Set(val)
|
metrics.monitorStatus.With(prometheus.Labels{"monitor": monitor}).Set(val)
|
||||||
}
|
}
|
||||||
|
|
||||||
// CountCheck counts the result of a particular Monitor check
|
// CountCheck counts the result of a particular Monitor check
|
||||||
func (metrics *MinitorMetrics) CountCheck(monitor string, isSuccess bool, ms int64, isAlert bool) {
|
func (metrics *MinitorMetrics) CountCheck(monitor string, isSuccess bool, isAlert bool) {
|
||||||
status := "failure"
|
status := "failure"
|
||||||
if isSuccess {
|
if isSuccess {
|
||||||
status = "success"
|
status = "success"
|
||||||
@@ -91,10 +81,6 @@ func (metrics *MinitorMetrics) CountCheck(monitor string, isSuccess bool, ms int
|
|||||||
metrics.checkCount.With(
|
metrics.checkCount.With(
|
||||||
prometheus.Labels{"monitor": monitor, "status": status, "is_alert": alertVal},
|
prometheus.Labels{"monitor": monitor, "status": status, "is_alert": alertVal},
|
||||||
).Inc()
|
).Inc()
|
||||||
|
|
||||||
metrics.checkTime.With(
|
|
||||||
prometheus.Labels{"monitor": monitor, "status": status},
|
|
||||||
).Set(float64(ms))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// CountAlert counts an alert
|
// CountAlert counts an alert
|
||||||
@@ -110,8 +96,6 @@ func (metrics *MinitorMetrics) CountAlert(monitor string, alert string) {
|
|||||||
// ServeMetrics starts an http server with a Prometheus metrics handler
|
// ServeMetrics starts an http server with a Prometheus metrics handler
|
||||||
func ServeMetrics() {
|
func ServeMetrics() {
|
||||||
http.Handle("/metrics", promhttp.Handler())
|
http.Handle("/metrics", promhttp.Handler())
|
||||||
|
|
||||||
host := fmt.Sprintf(":%d", MetricsPort)
|
host := fmt.Sprintf(":%d", MetricsPort)
|
||||||
|
|
||||||
_ = http.ListenAndServe(host, nil)
|
_ = http.ListenAndServe(host, nil)
|
||||||
}
|
}
|
||||||
|
|||||||
+91
-167
@@ -1,164 +1,86 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"log"
|
||||||
"fmt"
|
|
||||||
"math"
|
"math"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.iamthefij.com/iamthefij/slog"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Monitor represents a particular periodic check of a command
|
// Monitor represents a particular periodic check of a command
|
||||||
type Monitor struct { //nolint:maligned
|
type Monitor struct {
|
||||||
// Config values
|
// Config values
|
||||||
CheckIntervalStr *string `hcl:"check_interval,optional"`
|
Name string
|
||||||
CheckInterval time.Duration
|
Command []string
|
||||||
|
CommandShell string `yaml:"command_shell"`
|
||||||
Name string `hcl:"name,label"`
|
AlertDown []string `yaml:"alert_down"`
|
||||||
AlertCount int
|
AlertUp []string `yaml:"alert_up"`
|
||||||
AlertAfter int `hcl:"alert_after,optional"`
|
CheckInterval float64 `yaml:"check_interval"`
|
||||||
AlertEvery *int `hcl:"alert_every,optional"`
|
AlertAfter int16 `yaml:"alert_after"`
|
||||||
AlertDown []string `hcl:"alert_down,optional"`
|
AlertEvery int16 `yaml:"alert_every"`
|
||||||
AlertUp []string `hcl:"alert_up,optional"`
|
|
||||||
Command []string `hcl:"command,optional"`
|
|
||||||
ShellCommand string `hcl:"shell_command,optional"`
|
|
||||||
|
|
||||||
// Other values
|
// Other values
|
||||||
failureCount int
|
lastCheck time.Time
|
||||||
lastCheck time.Time
|
lastOutput string
|
||||||
lastSuccess time.Time
|
alertCount int16
|
||||||
lastOutput string
|
failureCount int16
|
||||||
lastCheckDuration time.Duration
|
lastSuccess time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// Init initializes the Monitor with default values
|
// IsValid returns a boolean indicating if the Monitor has been correctly
|
||||||
func (monitor *Monitor) Init(defaultAlertAfter int, defaultAlertEvery *int, defaultAlertDown []string, defaultAlertUp []string) error {
|
// configured
|
||||||
// Parse the check_interval string into a time.Duration
|
func (monitor Monitor) IsValid() bool {
|
||||||
if monitor.CheckIntervalStr != nil {
|
atLeastOneCommand := (monitor.CommandShell != "" || monitor.Command != nil)
|
||||||
var err error
|
atMostOneCommand := (monitor.CommandShell == "" || monitor.Command == nil)
|
||||||
|
return (atLeastOneCommand &&
|
||||||
monitor.CheckInterval, err = time.ParseDuration(*monitor.CheckIntervalStr)
|
atMostOneCommand &&
|
||||||
if err != nil {
|
monitor.getAlertAfter() > 0 &&
|
||||||
return fmt.Errorf("failed to parse check_interval duration for monitor %s: %w", monitor.Name, err)
|
monitor.AlertDown != nil)
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set default values for monitor alerts
|
|
||||||
if monitor.AlertAfter == 0 {
|
|
||||||
minAlertAfter := 1
|
|
||||||
monitor.AlertAfter = max(defaultAlertAfter, minAlertAfter)
|
|
||||||
}
|
|
||||||
|
|
||||||
if monitor.AlertEvery == nil {
|
|
||||||
monitor.AlertEvery = defaultAlertEvery
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(monitor.AlertDown) == 0 {
|
|
||||||
monitor.AlertDown = defaultAlertDown
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(monitor.AlertUp) == 0 {
|
|
||||||
monitor.AlertUp = defaultAlertUp
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate checks that the Monitor is properly configured and returns errors if not
|
// ShouldCheck returns a boolean indicating if the Monitor is ready to be
|
||||||
func (monitor Monitor) Validate() error {
|
// be checked again
|
||||||
hasCommand := len(monitor.Command) > 0
|
|
||||||
hasShellCommand := monitor.ShellCommand != ""
|
|
||||||
hasValidAlertAfter := monitor.AlertAfter > 0
|
|
||||||
hasAlertDown := len(monitor.AlertDown) > 0
|
|
||||||
|
|
||||||
var err error
|
|
||||||
|
|
||||||
hasAtLeastOneCommand := hasCommand || hasShellCommand
|
|
||||||
if !hasAtLeastOneCommand {
|
|
||||||
err = errors.Join(err, fmt.Errorf(
|
|
||||||
"%w: monitor %s has no command or shell_command configured",
|
|
||||||
ErrInvalidMonitor,
|
|
||||||
monitor.Name,
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
hasAtMostOneCommand := !(hasCommand && hasShellCommand)
|
|
||||||
if !hasAtMostOneCommand {
|
|
||||||
err = errors.Join(err, fmt.Errorf(
|
|
||||||
"%w: monitor %s has both command and shell_command configured",
|
|
||||||
ErrInvalidMonitor,
|
|
||||||
monitor.Name,
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
if !hasValidAlertAfter {
|
|
||||||
err = errors.Join(err, fmt.Errorf(
|
|
||||||
"%w: monitor %s has invalid alert_after value %d. Must be greater than 0",
|
|
||||||
ErrInvalidMonitor,
|
|
||||||
monitor.Name,
|
|
||||||
monitor.AlertAfter,
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
if !hasAlertDown {
|
|
||||||
err = errors.Join(err, fmt.Errorf(
|
|
||||||
"%w: monitor %s has no alert_down configured. Configure one here or add a default_alert_down",
|
|
||||||
ErrInvalidMonitor,
|
|
||||||
monitor.Name,
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (monitor Monitor) LastOutput() string {
|
|
||||||
return monitor.lastOutput
|
|
||||||
}
|
|
||||||
|
|
||||||
// ShouldCheck returns a boolean indicating if the Monitor is ready to be be checked again
|
|
||||||
func (monitor Monitor) ShouldCheck() bool {
|
func (monitor Monitor) ShouldCheck() bool {
|
||||||
if monitor.lastCheck.IsZero() || monitor.CheckInterval == 0 {
|
if monitor.lastCheck.IsZero() {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
sinceLastCheck := time.Since(monitor.lastCheck)
|
sinceLastCheck := time.Now().Sub(monitor.lastCheck).Seconds()
|
||||||
|
|
||||||
return sinceLastCheck >= monitor.CheckInterval
|
return sinceLastCheck >= monitor.CheckInterval
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check will run the command configured by the Monitor and return a status and a possible AlertNotice
|
// Check will run the command configured by the Monitor and return a status
|
||||||
|
// and a possible AlertNotice
|
||||||
func (monitor *Monitor) Check() (bool, *AlertNotice) {
|
func (monitor *Monitor) Check() (bool, *AlertNotice) {
|
||||||
var cmd *exec.Cmd
|
var cmd *exec.Cmd
|
||||||
if len(monitor.Command) > 0 {
|
if monitor.Command != nil {
|
||||||
cmd = exec.Command(monitor.Command[0], monitor.Command[1:]...)
|
cmd = exec.Command(monitor.Command[0], monitor.Command[1:]...)
|
||||||
} else if monitor.ShellCommand != "" {
|
|
||||||
cmd = ShellCommand(monitor.ShellCommand)
|
|
||||||
} else {
|
} else {
|
||||||
slog.Fatalf("Monitor %s has no command configured", monitor.Name)
|
cmd = ShellCommand(monitor.CommandShell)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkStartTime := time.Now()
|
|
||||||
output, err := cmd.CombinedOutput()
|
output, err := cmd.CombinedOutput()
|
||||||
monitor.lastCheck = time.Now()
|
monitor.lastCheck = time.Now()
|
||||||
monitor.lastOutput = string(output)
|
monitor.lastOutput = string(output)
|
||||||
monitor.lastCheckDuration = monitor.lastCheck.Sub(checkStartTime)
|
|
||||||
|
|
||||||
var alertNotice *AlertNotice
|
var alertNotice *AlertNotice
|
||||||
|
|
||||||
isSuccess := (err == nil)
|
isSuccess := (err == nil)
|
||||||
if isSuccess {
|
if isSuccess {
|
||||||
alertNotice = monitor.Success()
|
alertNotice = monitor.success()
|
||||||
} else {
|
} else {
|
||||||
alertNotice = monitor.Failure()
|
alertNotice = monitor.failure()
|
||||||
}
|
}
|
||||||
|
|
||||||
slog.Debugf("Command output: %s", monitor.lastOutput)
|
if LogDebug {
|
||||||
slog.OnErrWarnf(err, "Command result: %v", err)
|
log.Printf("DEBUG: Command output: %s", monitor.lastOutput)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
if LogDebug {
|
||||||
|
log.Printf("DEBUG: Command result: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
slog.Infof(
|
log.Printf(
|
||||||
"%s success=%t, alert=%t",
|
"INFO: %s success=%t, alert=%t",
|
||||||
monitor.Name,
|
monitor.Name,
|
||||||
isSuccess,
|
isSuccess,
|
||||||
alertNotice != nil,
|
alertNotice != nil,
|
||||||
@@ -167,89 +89,91 @@ func (monitor *Monitor) Check() (bool, *AlertNotice) {
|
|||||||
return isSuccess, alertNotice
|
return isSuccess, alertNotice
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetAlertNames gives a list of alert names for a given monitor status
|
func (monitor Monitor) isUp() bool {
|
||||||
func (monitor Monitor) GetAlertNames(up bool) []string {
|
return monitor.alertCount == 0
|
||||||
if up {
|
|
||||||
return monitor.AlertUp
|
|
||||||
}
|
|
||||||
|
|
||||||
return monitor.AlertDown
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsUp returns the status of the current monitor
|
func (monitor *Monitor) success() (notice *AlertNotice) {
|
||||||
func (monitor Monitor) IsUp() bool {
|
if !monitor.isUp() {
|
||||||
return monitor.AlertCount == 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// LastCheckMilliseconds gives number of miliseconds the last check ran for
|
|
||||||
func (monitor Monitor) LastCheckMilliseconds() int64 {
|
|
||||||
return monitor.lastCheckDuration.Milliseconds()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (monitor *Monitor) Success() (notice *AlertNotice) {
|
|
||||||
if !monitor.IsUp() {
|
|
||||||
// Alert that we have recovered
|
// Alert that we have recovered
|
||||||
notice = monitor.createAlertNotice(true)
|
notice = monitor.createAlertNotice(true)
|
||||||
}
|
}
|
||||||
|
|
||||||
monitor.failureCount = 0
|
monitor.failureCount = 0
|
||||||
monitor.AlertCount = 0
|
monitor.alertCount = 0
|
||||||
monitor.lastSuccess = time.Now()
|
monitor.lastSuccess = time.Now()
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func (monitor *Monitor) Failure() (notice *AlertNotice) {
|
func (monitor *Monitor) failure() (notice *AlertNotice) {
|
||||||
monitor.failureCount++
|
monitor.failureCount++
|
||||||
// If we haven't hit the minimum failures, we can exit
|
// If we haven't hit the minimum failures, we can exit
|
||||||
if monitor.failureCount < monitor.AlertAfter {
|
if monitor.failureCount < monitor.getAlertAfter() {
|
||||||
slog.Debugf(
|
if LogDebug {
|
||||||
"%s failed but did not hit minimum failures. "+
|
log.Printf(
|
||||||
"Count: %v alert after: %v",
|
"DEBUG: %s failed but did not hit minimum failures. "+
|
||||||
monitor.Name,
|
"Count: %v alert after: %v",
|
||||||
monitor.failureCount,
|
monitor.Name,
|
||||||
monitor.AlertAfter,
|
monitor.failureCount,
|
||||||
)
|
monitor.getAlertAfter(),
|
||||||
|
)
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Take number of failures after minimum
|
// Take number of failures after minimum
|
||||||
failureCount := (monitor.failureCount - monitor.AlertAfter)
|
failureCount := (monitor.failureCount - monitor.getAlertAfter())
|
||||||
|
|
||||||
// Use alert cadence to determine if we should alert
|
// Use alert cadence to determine if we should alert
|
||||||
switch {
|
if monitor.AlertEvery > 0 {
|
||||||
case monitor.AlertEvery == nil, *monitor.AlertEvery == 0:
|
// Handle integer number of failures before alerting
|
||||||
|
if failureCount%monitor.AlertEvery == 0 {
|
||||||
|
notice = monitor.createAlertNotice(false)
|
||||||
|
}
|
||||||
|
} else if monitor.AlertEvery == 0 {
|
||||||
// Handle alerting on first failure only
|
// Handle alerting on first failure only
|
||||||
if failureCount == 0 {
|
if failureCount == 0 {
|
||||||
notice = monitor.createAlertNotice(false)
|
notice = monitor.createAlertNotice(false)
|
||||||
}
|
}
|
||||||
case *monitor.AlertEvery > 0:
|
} else {
|
||||||
// Handle integer number of failures before alerting
|
|
||||||
if failureCount%*monitor.AlertEvery == 0 {
|
|
||||||
notice = monitor.createAlertNotice(false)
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
// Handle negative numbers indicating an exponential backoff
|
// Handle negative numbers indicating an exponential backoff
|
||||||
if failureCount >= int(math.Pow(2, float64(monitor.AlertCount))-1) { //nolint:mnd
|
if failureCount >= int16(math.Pow(2, float64(monitor.alertCount))-1) {
|
||||||
notice = monitor.createAlertNotice(false)
|
notice = monitor.createAlertNotice(false)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// If we're going to alert, increment count
|
// If we're going to alert, increment count
|
||||||
if notice != nil {
|
if notice != nil {
|
||||||
monitor.AlertCount++
|
monitor.alertCount++
|
||||||
notice.AlertCount = monitor.AlertCount
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return notice
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (monitor Monitor) getAlertAfter() int16 {
|
||||||
|
// TODO: Come up with a better way than this method
|
||||||
|
// Zero is one!
|
||||||
|
if monitor.AlertAfter == 0 {
|
||||||
|
return 1
|
||||||
|
} else {
|
||||||
|
return monitor.AlertAfter
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetAlertNames gives a list of alert names for a given monitor status
|
||||||
|
func (monitor Monitor) GetAlertNames(up bool) []string {
|
||||||
|
if up {
|
||||||
|
return monitor.AlertUp
|
||||||
|
} else {
|
||||||
|
return monitor.AlertDown
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (monitor Monitor) createAlertNotice(isUp bool) *AlertNotice {
|
func (monitor Monitor) createAlertNotice(isUp bool) *AlertNotice {
|
||||||
// TODO: Maybe add something about recovery status here
|
// TODO: Maybe add something about recovery status here
|
||||||
return &AlertNotice{
|
return &AlertNotice{
|
||||||
MonitorName: monitor.Name,
|
MonitorName: monitor.Name,
|
||||||
AlertCount: monitor.AlertCount,
|
AlertCount: monitor.alertCount,
|
||||||
FailureCount: monitor.failureCount,
|
FailureCount: monitor.failureCount,
|
||||||
LastCheckOutput: monitor.lastOutput,
|
LastCheckOutput: monitor.lastOutput,
|
||||||
LastSuccess: monitor.lastSuccess,
|
LastSuccess: monitor.lastSuccess,
|
||||||
|
|||||||
+161
-186
@@ -1,196 +1,164 @@
|
|||||||
package main_test
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"log"
|
||||||
"reflect"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
m "git.iamthefij.com/iamthefij/minitor-go/v2"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestMonitorValidate(t *testing.T) {
|
// TestMonitorIsValid tests the Monitor.IsValid()
|
||||||
t.Parallel()
|
func TestMonitorIsValid(t *testing.T) {
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
monitor m.Monitor
|
monitor Monitor
|
||||||
expected error
|
expected bool
|
||||||
name string
|
name string
|
||||||
}{
|
}{
|
||||||
{m.Monitor{AlertAfter: 1, Command: []string{"echo", "test"}, AlertDown: []string{"log"}}, nil, "Command only"},
|
{Monitor{Command: []string{"echo", "test"}, AlertDown: []string{"log"}}, true, "Command only"},
|
||||||
{m.Monitor{AlertAfter: 1, ShellCommand: "echo test", AlertDown: []string{"log"}}, nil, "CommandShell only"},
|
{Monitor{CommandShell: "echo test", AlertDown: []string{"log"}}, true, "CommandShell only"},
|
||||||
{m.Monitor{AlertAfter: 1, Command: []string{"echo", "test"}}, m.ErrInvalidMonitor, "No AlertDown"},
|
{Monitor{Command: []string{"echo", "test"}}, false, "No AlertDown"},
|
||||||
{m.Monitor{AlertAfter: 1, AlertDown: []string{"log"}}, m.ErrInvalidMonitor, "No commands"},
|
{Monitor{AlertDown: []string{"log"}}, false, "No commands"},
|
||||||
{m.Monitor{AlertAfter: -1, Command: []string{"echo", "test"}, AlertDown: []string{"log"}}, m.ErrInvalidMonitor, "Invalid alert threshold, -1"},
|
{
|
||||||
|
Monitor{Command: []string{"echo", "test"}, CommandShell: "echo test", AlertDown: []string{"log"}},
|
||||||
|
false,
|
||||||
|
"Both commands",
|
||||||
|
},
|
||||||
|
{Monitor{Command: []string{"echo", "test"}, AlertDown: []string{"log"}, AlertAfter: -1}, false, "Invalid alert threshold, -1"},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
c := c
|
log.Printf("Testing case %s", c.name)
|
||||||
|
actual := c.monitor.IsValid()
|
||||||
t.Run(c.name, func(t *testing.T) {
|
if actual != c.expected {
|
||||||
t.Parallel()
|
t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expected, actual)
|
||||||
|
log.Printf("Case failed: %s", c.name)
|
||||||
actual := c.monitor.Validate()
|
}
|
||||||
hasErr := (actual != nil)
|
log.Println("-----")
|
||||||
expectErr := (c.expected != nil)
|
|
||||||
|
|
||||||
if hasErr != expectErr || !errors.Is(actual, c.expected) {
|
|
||||||
t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expected, actual)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestMonitorShouldCheck tests the Monitor.ShouldCheck()
|
// TestMonitorShouldCheck tests the Monitor.ShouldCheck()
|
||||||
func TestMonitorShouldCheck(t *testing.T) {
|
func TestMonitorShouldCheck(t *testing.T) {
|
||||||
t.Parallel()
|
timeNow := time.Now()
|
||||||
|
timeTenSecAgo := time.Now().Add(time.Second * -10)
|
||||||
|
timeTwentySecAgo := time.Now().Add(time.Second * -20)
|
||||||
|
|
||||||
// Create a monitor that should check every second and then verify it checks with some sleeps
|
cases := []struct {
|
||||||
monitor := m.Monitor{ShellCommand: "true", CheckInterval: time.Second}
|
monitor Monitor
|
||||||
|
expected bool
|
||||||
if !monitor.ShouldCheck() {
|
name string
|
||||||
t.Errorf("New monitor should be ready to check")
|
}{
|
||||||
|
{Monitor{}, true, "Empty"},
|
||||||
|
{Monitor{lastCheck: timeNow, CheckInterval: 15}, false, "Just checked"},
|
||||||
|
{Monitor{lastCheck: timeTenSecAgo, CheckInterval: 15}, false, "-10s"},
|
||||||
|
{Monitor{lastCheck: timeTwentySecAgo, CheckInterval: 15}, true, "-20s"},
|
||||||
}
|
}
|
||||||
|
|
||||||
monitor.Check()
|
for _, c := range cases {
|
||||||
|
actual := c.monitor.ShouldCheck()
|
||||||
if monitor.ShouldCheck() {
|
if actual != c.expected {
|
||||||
t.Errorf("Monitor should not be ready to check after a check")
|
t.Errorf("ShouldCheck(%v), expected=%t actual=%t", c.name, c.expected, actual)
|
||||||
}
|
}
|
||||||
|
|
||||||
time.Sleep(time.Second)
|
|
||||||
|
|
||||||
if !monitor.ShouldCheck() {
|
|
||||||
t.Errorf("Monitor should be ready to check after a second")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestMonitorIsUp tests the Monitor.IsUp()
|
// TestMonitorIsUp tests the Monitor.isUp()
|
||||||
func TestMonitorIsUp(t *testing.T) {
|
func TestMonitorIsUp(t *testing.T) {
|
||||||
t.Parallel()
|
cases := []struct {
|
||||||
|
monitor Monitor
|
||||||
// Creating a monitor that should alert after 2 failures. The monitor should be considered up until we reach two failed checks
|
expected bool
|
||||||
monitor := m.Monitor{ShellCommand: "false", AlertAfter: 2}
|
name string
|
||||||
if !monitor.IsUp() {
|
}{
|
||||||
t.Errorf("New monitor should be considered up")
|
{Monitor{}, true, "Empty"},
|
||||||
|
{Monitor{alertCount: 1}, false, "Has alert"},
|
||||||
|
{Monitor{alertCount: -1}, false, "Negative alerts"},
|
||||||
|
{Monitor{alertCount: 0}, true, "No alerts"},
|
||||||
}
|
}
|
||||||
|
|
||||||
monitor.Check()
|
for _, c := range cases {
|
||||||
|
log.Printf("Testing case %s", c.name)
|
||||||
if !monitor.IsUp() {
|
actual := c.monitor.isUp()
|
||||||
t.Errorf("Monitor should be considered up with one failure and no alerts")
|
if actual != c.expected {
|
||||||
}
|
t.Errorf("isUp(%v), expected=%t actual=%t", c.name, c.expected, actual)
|
||||||
|
log.Printf("Case failed: %s", c.name)
|
||||||
monitor.Check()
|
}
|
||||||
|
log.Println("-----")
|
||||||
if monitor.IsUp() {
|
|
||||||
t.Errorf("Monitor should be considered down with one alert")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestMonitorGetAlertNames tests that proper alert names are returned
|
// TestMonitorGetAlertNames tests that proper alert names are returned
|
||||||
func TestMonitorGetAlertNames(t *testing.T) {
|
func TestMonitorGetAlertNames(t *testing.T) {
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
monitor m.Monitor
|
monitor Monitor
|
||||||
up bool
|
up bool
|
||||||
expected []string
|
expected []string
|
||||||
name string
|
name string
|
||||||
}{
|
}{
|
||||||
{m.Monitor{}, true, nil, "Empty up"},
|
{Monitor{}, true, nil, "Empty up"},
|
||||||
{m.Monitor{}, false, nil, "Empty down"},
|
{Monitor{}, false, nil, "Empty down"},
|
||||||
{m.Monitor{AlertUp: []string{"alert"}}, true, []string{"alert"}, "Return up"},
|
{Monitor{AlertUp: []string{"alert"}}, true, []string{"alert"}, "Return up"},
|
||||||
{m.Monitor{AlertDown: []string{"alert"}}, false, []string{"alert"}, "Return down"},
|
{Monitor{AlertDown: []string{"alert"}}, false, []string{"alert"}, "Return down"},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
c := c
|
log.Printf("Testing case %s", c.name)
|
||||||
|
actual := c.monitor.GetAlertNames(c.up)
|
||||||
t.Run(c.name, func(t *testing.T) {
|
if !EqualSliceString(actual, c.expected) {
|
||||||
t.Parallel()
|
t.Errorf("GetAlertNames(%v), expected=%v actual=%v", c.name, c.expected, actual)
|
||||||
|
log.Printf("Case failed: %s", c.name)
|
||||||
actual := c.monitor.GetAlertNames(c.up)
|
}
|
||||||
if !reflect.DeepEqual(actual, c.expected) {
|
log.Println("-----")
|
||||||
t.Errorf("GetAlertNames(%v), expected=%v actual=%v", c.name, c.expected, actual)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMonitorAlertCount(t *testing.T) {
|
// TestMonitorSuccess tests the Monitor.success()
|
||||||
alertEvery := 1
|
func TestMonitorSuccess(t *testing.T) {
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
checkSuccess bool
|
monitor Monitor
|
||||||
alertCount int
|
expectNotice bool
|
||||||
name string
|
name string
|
||||||
}{
|
}{
|
||||||
{false, 1, "First failure and first alert"},
|
{Monitor{}, false, "Empty"},
|
||||||
{false, 2, "Second failure and first alert"},
|
{Monitor{alertCount: 0}, false, "No alerts"},
|
||||||
{true, 2, "Success should preserve past alert count"},
|
{Monitor{alertCount: 1}, true, "Has alert"},
|
||||||
{false, 1, "First failure and first alert after success"},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Unlike previous tests, this one requires a static Monitor with repeated
|
|
||||||
// calls to the failure method
|
|
||||||
monitor := m.Monitor{AlertAfter: 1, AlertEvery: &alertEvery}
|
|
||||||
|
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
t.Logf("Testing case %s", c.name)
|
log.Printf("Testing case %s", c.name)
|
||||||
|
notice := c.monitor.success()
|
||||||
var notice *m.AlertNotice
|
hasNotice := (notice != nil)
|
||||||
if c.checkSuccess {
|
if hasNotice != c.expectNotice {
|
||||||
notice = monitor.Success()
|
t.Errorf("success(%v), expected=%t actual=%t", c.name, c.expectNotice, hasNotice)
|
||||||
} else {
|
log.Printf("Case failed: %s", c.name)
|
||||||
notice = monitor.Failure()
|
|
||||||
}
|
|
||||||
|
|
||||||
if notice == nil {
|
|
||||||
t.Fatalf("failure(%v) expected notice, got nil", c.name)
|
|
||||||
}
|
|
||||||
|
|
||||||
if notice.AlertCount != c.alertCount {
|
|
||||||
t.Errorf("failure(%v), expected=%v actual=%v", c.name, c.alertCount, notice.AlertCount)
|
|
||||||
t.Logf("Case failed: %s", c.name)
|
|
||||||
}
|
}
|
||||||
|
log.Println("-----")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestMonitorFailureAlertAfter tests that alerts will not trigger until
|
// TestMonitorFailureAlertAfter tests that alerts will not trigger until
|
||||||
// hitting the threshold provided by AlertAfter
|
// hitting the threshold provided by AlertAfter
|
||||||
func TestMonitorFailureAlertAfter(t *testing.T) {
|
func TestMonitorFailureAlertAfter(t *testing.T) {
|
||||||
var alertEveryOne int = 1
|
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
monitor m.Monitor
|
monitor Monitor
|
||||||
numChecks int
|
|
||||||
expectNotice bool
|
expectNotice bool
|
||||||
name string
|
name string
|
||||||
}{
|
}{
|
||||||
{m.Monitor{ShellCommand: "false", AlertAfter: 1}, 1, true, "Empty After 1"}, // Defaults to true because and AlertEvery default to 0
|
{Monitor{AlertAfter: 1}, true, "Empty"}, // Defaults to true because and AlertEvery default to 0
|
||||||
{m.Monitor{ShellCommand: "false", AlertAfter: 1, AlertEvery: &alertEveryOne}, 1, true, "Alert after 1: first failure"},
|
{Monitor{failureCount: 0, AlertAfter: 1, AlertEvery: 1}, true, "Alert after 1: first failure"},
|
||||||
{m.Monitor{ShellCommand: "false", AlertAfter: 1, AlertEvery: &alertEveryOne}, 2, true, "Alert after 1: second failure"},
|
{Monitor{failureCount: 1, AlertAfter: 1, AlertEvery: 1}, true, "Alert after 1: second failure"},
|
||||||
{m.Monitor{ShellCommand: "false", AlertAfter: 20, AlertEvery: &alertEveryOne}, 1, false, "Alert after 20: first failure"},
|
{Monitor{failureCount: 0, AlertAfter: 20, AlertEvery: 1}, false, "Alert after 20: first failure"},
|
||||||
{m.Monitor{ShellCommand: "false", AlertAfter: 20, AlertEvery: &alertEveryOne}, 20, true, "Alert after 20: 20th failure"},
|
{Monitor{failureCount: 19, AlertAfter: 20, AlertEvery: 1}, true, "Alert after 20: 20th failure"},
|
||||||
{m.Monitor{ShellCommand: "false", AlertAfter: 20, AlertEvery: &alertEveryOne}, 21, true, "Alert after 20: 21st failure"},
|
{Monitor{failureCount: 20, AlertAfter: 20, AlertEvery: 1}, true, "Alert after 20: 21st failure"},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
c := c
|
log.Printf("Testing case %s", c.name)
|
||||||
|
notice := c.monitor.failure()
|
||||||
t.Run(c.name, func(t *testing.T) {
|
hasNotice := (notice != nil)
|
||||||
t.Parallel()
|
if hasNotice != c.expectNotice {
|
||||||
|
t.Errorf("failure(%v), expected=%t actual=%t", c.name, c.expectNotice, hasNotice)
|
||||||
hasNotice := false
|
log.Printf("Case failed: %s", c.name)
|
||||||
|
}
|
||||||
for i := 0; i < c.numChecks; i++ {
|
log.Println("-----")
|
||||||
_, notice := c.monitor.Check()
|
|
||||||
hasNotice = (notice != nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
if hasNotice != c.expectNotice {
|
|
||||||
t.Errorf("failure(%v), expected=%t actual=%t", c.name, c.expectNotice, hasNotice)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -198,42 +166,51 @@ func TestMonitorFailureAlertAfter(t *testing.T) {
|
|||||||
// on the expected intervals
|
// on the expected intervals
|
||||||
func TestMonitorFailureAlertEvery(t *testing.T) {
|
func TestMonitorFailureAlertEvery(t *testing.T) {
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
monitor m.Monitor
|
monitor Monitor
|
||||||
expectedNotice []bool
|
expectNotice bool
|
||||||
name string
|
name string
|
||||||
}{
|
}{
|
||||||
{m.Monitor{ShellCommand: "false", AlertAfter: 1}, []bool{true}, "No AlertEvery set"}, // Defaults to true because AlertAfter and AlertEvery default to nil
|
/*
|
||||||
|
TODO: Actually found a bug in original implementation. There is an inconsistency in the way AlertAfter is treated.
|
||||||
|
For "First alert only" (ie. AlertEvery=0), it is the number of failures to ignore before alerting, so AlertAfter=1
|
||||||
|
will ignore the first failure and alert on the second failure
|
||||||
|
For other intervals (ie. AlertEvery=1), it is essentially indexed on one. Essentially making AlertAfter=1 trigger
|
||||||
|
on the first failure.
|
||||||
|
|
||||||
|
For usabilty, this should be consistent. Consistent with what though? minitor-py? Or itself? Dun dun duuuunnnnn!
|
||||||
|
*/
|
||||||
|
{Monitor{AlertAfter: 1}, true, "Empty"}, // Defaults to true because AlertAfter and AlertEvery default to 0
|
||||||
// Alert first time only, after 1
|
// Alert first time only, after 1
|
||||||
{m.Monitor{ShellCommand: "false", AlertAfter: 1, AlertEvery: Ptr(0)}, []bool{true, false, false}, "Alert first time only after 1"},
|
{Monitor{failureCount: 0, AlertAfter: 1, AlertEvery: 0}, true, "Alert first time only after 1: first failure"},
|
||||||
|
{Monitor{failureCount: 1, AlertAfter: 1, AlertEvery: 0}, false, "Alert first time only after 1: second failure"},
|
||||||
|
{Monitor{failureCount: 2, AlertAfter: 1, AlertEvery: 0}, false, "Alert first time only after 1: third failure"},
|
||||||
// Alert every time, after 1
|
// Alert every time, after 1
|
||||||
{m.Monitor{ShellCommand: "false", AlertAfter: 1, AlertEvery: Ptr(1)}, []bool{true, true, true}, "Alert every time after 1"},
|
{Monitor{failureCount: 0, AlertAfter: 1, AlertEvery: 1}, true, "Alert every time after 1: first failure"},
|
||||||
|
{Monitor{failureCount: 1, AlertAfter: 1, AlertEvery: 1}, true, "Alert every time after 1: second failure"},
|
||||||
|
{Monitor{failureCount: 1, AlertAfter: 1, AlertEvery: 1}, true, "Alert every time after 1: third failure"},
|
||||||
// Alert every other time, after 1
|
// Alert every other time, after 1
|
||||||
{m.Monitor{ShellCommand: "false", AlertAfter: 1, AlertEvery: Ptr(2)}, []bool{true, false, true, false}, "Alert every other time after 1"},
|
{Monitor{failureCount: 0, AlertAfter: 1, AlertEvery: 2}, true, "Alert every other time after 1: first failure"},
|
||||||
|
{Monitor{failureCount: 1, AlertAfter: 1, AlertEvery: 2}, false, "Alert every other time after 1: second failure"},
|
||||||
|
{Monitor{failureCount: 2, AlertAfter: 1, AlertEvery: 2}, true, "Alert every other time after 1: third failure"},
|
||||||
|
{Monitor{failureCount: 3, AlertAfter: 1, AlertEvery: 2}, false, "Alert every other time after 1: fourth failure"},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
c := c
|
log.Printf("Testing case %s", c.name)
|
||||||
|
|
||||||
t.Run(c.name, func(t *testing.T) {
|
notice := c.monitor.failure()
|
||||||
t.Parallel()
|
hasNotice := (notice != nil)
|
||||||
|
if hasNotice != c.expectNotice {
|
||||||
for i, expectNotice := range c.expectedNotice {
|
t.Errorf("failure(%v), expected=%t actual=%t", c.name, c.expectNotice, hasNotice)
|
||||||
_, notice := c.monitor.Check()
|
log.Printf("Case failed: %s", c.name)
|
||||||
hasNotice := (notice != nil)
|
}
|
||||||
|
log.Println("-----")
|
||||||
if hasNotice != expectNotice {
|
|
||||||
t.Errorf("failed %s check %d: expected=%t actual=%t", c.name, i, expectNotice, hasNotice)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestMonitorFailureExponential tests that alerts will trigger
|
// TestMonitorFailureExponential tests that alerts will trigger
|
||||||
// with an exponential backoff after repeated failures
|
// with an exponential backoff after repeated failures
|
||||||
func TestMonitorFailureExponential(t *testing.T) {
|
func TestMonitorFailureExponential(t *testing.T) {
|
||||||
var alertEveryExp int = -1
|
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
expectNotice bool
|
expectNotice bool
|
||||||
name string
|
name string
|
||||||
@@ -250,18 +227,17 @@ func TestMonitorFailureExponential(t *testing.T) {
|
|||||||
|
|
||||||
// Unlike previous tests, this one requires a static Monitor with repeated
|
// Unlike previous tests, this one requires a static Monitor with repeated
|
||||||
// calls to the failure method
|
// calls to the failure method
|
||||||
monitor := m.Monitor{ShellCommand: "false", AlertAfter: 1, AlertEvery: &alertEveryExp}
|
monitor := Monitor{failureCount: 0, AlertAfter: 1, AlertEvery: -1}
|
||||||
|
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
t.Run(c.name, func(t *testing.T) {
|
log.Printf("Testing case %s", c.name)
|
||||||
// NOTE: These tests are not parallel because they rely on the state of the Monitor
|
|
||||||
_, notice := monitor.Check()
|
|
||||||
hasNotice := (notice != nil)
|
|
||||||
|
|
||||||
if hasNotice != c.expectNotice {
|
notice := monitor.failure()
|
||||||
t.Errorf("failure(%v), expected=%t actual=%t", c.name, c.expectNotice, hasNotice)
|
hasNotice := (notice != nil)
|
||||||
}
|
if hasNotice != c.expectNotice {
|
||||||
})
|
t.Errorf("failure(%v), expected=%t actual=%t", c.name, c.expectNotice, hasNotice)
|
||||||
|
log.Printf("Case failed: %s", c.name)
|
||||||
|
}
|
||||||
|
log.Println("-----")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -272,54 +248,53 @@ func TestMonitorCheck(t *testing.T) {
|
|||||||
hasNotice bool
|
hasNotice bool
|
||||||
lastOutput string
|
lastOutput string
|
||||||
}
|
}
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
monitor m.Monitor
|
monitor Monitor
|
||||||
expect expected
|
expect expected
|
||||||
name string
|
name string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
m.Monitor{AlertAfter: 1, Command: []string{"echo", "success"}},
|
Monitor{Command: []string{"echo", "success"}},
|
||||||
expected{isSuccess: true, hasNotice: false, lastOutput: "success\n"},
|
expected{isSuccess: true, hasNotice: false, lastOutput: "success\n"},
|
||||||
"Test successful command",
|
"Test successful command",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
m.Monitor{AlertAfter: 1, ShellCommand: "echo success"},
|
Monitor{CommandShell: "echo success"},
|
||||||
expected{isSuccess: true, hasNotice: false, lastOutput: "success\n"},
|
expected{isSuccess: true, hasNotice: false, lastOutput: "success\n"},
|
||||||
"Test successful command shell",
|
"Test successful command shell",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
m.Monitor{AlertAfter: 1, Command: []string{"total", "failure"}},
|
Monitor{Command: []string{"total", "failure"}},
|
||||||
expected{isSuccess: false, hasNotice: true, lastOutput: ""},
|
expected{isSuccess: false, hasNotice: true, lastOutput: ""},
|
||||||
"Test failed command",
|
"Test failed command",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
m.Monitor{AlertAfter: 1, ShellCommand: "false"},
|
Monitor{CommandShell: "false"},
|
||||||
expected{isSuccess: false, hasNotice: true, lastOutput: ""},
|
expected{isSuccess: false, hasNotice: true, lastOutput: ""},
|
||||||
"Test failed command shell",
|
"Test failed command shell",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
c := c
|
log.Printf("Testing case %s", c.name)
|
||||||
|
|
||||||
t.Run(c.name, func(t *testing.T) {
|
isSuccess, notice := c.monitor.Check()
|
||||||
t.Parallel()
|
if isSuccess != c.expect.isSuccess {
|
||||||
|
t.Errorf("Check(%v) (success), expected=%t actual=%t", c.name, c.expect.isSuccess, isSuccess)
|
||||||
|
log.Printf("Case failed: %s", c.name)
|
||||||
|
}
|
||||||
|
|
||||||
isSuccess, notice := c.monitor.Check()
|
hasNotice := (notice != nil)
|
||||||
if isSuccess != c.expect.isSuccess {
|
if hasNotice != c.expect.hasNotice {
|
||||||
t.Errorf("Check(%v) (success), expected=%t actual=%t", c.name, c.expect.isSuccess, isSuccess)
|
t.Errorf("Check(%v) (notice), expected=%t actual=%t", c.name, c.expect.hasNotice, hasNotice)
|
||||||
}
|
log.Printf("Case failed: %s", c.name)
|
||||||
|
}
|
||||||
|
|
||||||
hasNotice := (notice != nil)
|
lastOutput := c.monitor.lastOutput
|
||||||
if hasNotice != c.expect.hasNotice {
|
if lastOutput != c.expect.lastOutput {
|
||||||
t.Errorf("Check(%v) (notice), expected=%t actual=%t", c.name, c.expect.hasNotice, hasNotice)
|
t.Errorf("Check(%v) (output), expected=%v actual=%v", c.name, c.expect.lastOutput, lastOutput)
|
||||||
}
|
log.Printf("Case failed: %s", c.name)
|
||||||
|
}
|
||||||
lastOutput := c.monitor.LastOutput()
|
log.Println("-----")
|
||||||
if lastOutput != c.expect.lastOutput {
|
|
||||||
t.Errorf("Check(%v) (output), expected=%v actual=%v", c.name, c.expect.lastOutput, lastOutput)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,3 +0,0 @@
|
|||||||
{
|
|
||||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json"
|
|
||||||
}
|
|
||||||
@@ -1,52 +0,0 @@
|
|||||||
check_interval = "5s"
|
|
||||||
|
|
||||||
monitor "Fake Website" {
|
|
||||||
command = ["curl", "-s", "-o", "/dev/null", "https://minitor.mon"]
|
|
||||||
alert_down = ["log_down", "mailgun_down", "sms_down"]
|
|
||||||
alert_up = ["log_up", "email_up"]
|
|
||||||
check_interval = "10s" # Must be at minimum the global `check_interval`
|
|
||||||
alert_after = 3
|
|
||||||
alert_every = -1 # Defaults to -1 for exponential backoff. 0 to disable repeating
|
|
||||||
}
|
|
||||||
|
|
||||||
monitor "Real Website" {
|
|
||||||
command = ["curl", "-s", "-o", "/dev/null", "https://google.com"]
|
|
||||||
alert_down = ["log_down", "mailgun_down", "sms_down"]
|
|
||||||
alert_up = ["log_up", "email_up"]
|
|
||||||
check_interval = "5s"
|
|
||||||
alert_after = 3
|
|
||||||
alert_every = -1
|
|
||||||
}
|
|
||||||
|
|
||||||
alert "log_down" {
|
|
||||||
command = ["echo", "Minitor failure for {{.MonitorName}}"]
|
|
||||||
}
|
|
||||||
|
|
||||||
alert "log_up" {
|
|
||||||
command = ["echo", "Minitor recovery for {{.MonitorName}}"]
|
|
||||||
}
|
|
||||||
|
|
||||||
alert "email_up" {
|
|
||||||
command = ["sendmail", "me@minitor.mon", "Recovered: {monitor_name}", "We're back!"]
|
|
||||||
}
|
|
||||||
|
|
||||||
alert "mailgun_down" {
|
|
||||||
shell_command = <<-EOF
|
|
||||||
curl -s -X POST \
|
|
||||||
-F subject="Alert! {{.MonitorName}} failed" \
|
|
||||||
-F from="Minitor <minitor@minitor.mon>" \
|
|
||||||
-F to=me@minitor.mon \
|
|
||||||
-F text="Our monitor failed" \
|
|
||||||
https://api.mailgun.net/v3/minitor.mon/messages \
|
|
||||||
-u "api:$${MAILGUN_API_KEY}"
|
|
||||||
EOF
|
|
||||||
}
|
|
||||||
|
|
||||||
alert "sms_down" {
|
|
||||||
shell_command = <<-EOF
|
|
||||||
curl -s -X POST -F "Body=Failure! {{.MonitorName}} has failed" \
|
|
||||||
-F "From=$${AVAILABLE_NUMBER}" -F "To=$${MY_PHONE}" \
|
|
||||||
"https://api.twilio.com/2010-04-01/Accounts/$${ACCOUNT_SID}/Messages" \
|
|
||||||
-u "$${ACCOUNT_SID}:$${AUTH_TOKEN}"
|
|
||||||
EOF
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
---
|
||||||
|
check_interval: 5
|
||||||
|
|
||||||
|
monitors:
|
||||||
|
- name: Fake Website
|
||||||
|
command: ['curl', '-s', '-o', '/dev/null', 'https://minitor.mon']
|
||||||
|
alert_down: [log_down, mailgun_down, sms_down]
|
||||||
|
alert_up: [log_up, email_up]
|
||||||
|
check_interval: 10 # Must be at minimum the global `check_interval`
|
||||||
|
alert_after: 3
|
||||||
|
alert_every: -1 # Defaults to -1 for exponential backoff. 0 to disable repeating
|
||||||
|
- name: Real Website
|
||||||
|
command: ['curl', '-s', '-o', '/dev/null', 'https://google.com']
|
||||||
|
alert_down: [log_down, mailgun_down, sms_down]
|
||||||
|
alert_up: [log_up, email_up]
|
||||||
|
check_interval: 5
|
||||||
|
alert_after: 3
|
||||||
|
alert_every: -1
|
||||||
|
|
||||||
|
alerts:
|
||||||
|
log_down:
|
||||||
|
command: ["echo", "Minitor failure for {{.MonitorName}}"]
|
||||||
|
log_up:
|
||||||
|
command: ["echo", "Minitor recovery for {{.MonitorName}}"]
|
||||||
|
email_up:
|
||||||
|
command: [sendmail, "me@minitor.mon", "Recovered: {monitor_name}", "We're back!"]
|
||||||
|
mailgun_down:
|
||||||
|
command_shell: >
|
||||||
|
curl -s -X POST
|
||||||
|
-F subject="Alert! {{.MonitorName}} failed"
|
||||||
|
-F from="Minitor <minitor@minitor.mon>"
|
||||||
|
-F to=me@minitor.mon
|
||||||
|
-F text="Our monitor failed"
|
||||||
|
https://api.mailgun.net/v3/minitor.mon/messages
|
||||||
|
-u "api:${MAILGUN_API_KEY}"
|
||||||
|
sms_down:
|
||||||
|
command_shell: >
|
||||||
|
curl -s -X POST -F "Body=Failure! {{.MonitorName}} has failed"
|
||||||
|
-F "From=${AVAILABLE_NUMBER}" -F "To=${MY_PHONE}"
|
||||||
|
"https://api.twilio.com/2010-04-01/Accounts/${ACCOUNT_SID}/Messages"
|
||||||
|
-u "${ACCOUNT_SID}:${AUTH_TOKEN}"
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
#! /bin/sh
|
|
||||||
|
|
||||||
# Used for a basic HTTP health check
|
|
||||||
# Avoids output from non-errors and will fail if the HTTP response is unsuccessful
|
|
||||||
|
|
||||||
curl --silent --show-error --fail -o /dev/null "$@"
|
|
||||||
+3
-15
@@ -11,7 +11,6 @@ set -e
|
|||||||
# To override, export DOCKER_HOST to a new hostname
|
# To override, export DOCKER_HOST to a new hostname
|
||||||
DOCKER_HOST="${DOCKER_HOST:=socket}"
|
DOCKER_HOST="${DOCKER_HOST:=socket}"
|
||||||
container_name="$1"
|
container_name="$1"
|
||||||
num_log_lines="$2"
|
|
||||||
|
|
||||||
# Curls Docker either using a socket or URL
|
# Curls Docker either using a socket or URL
|
||||||
function curl_docker {
|
function curl_docker {
|
||||||
@@ -32,32 +31,21 @@ function get_container_id {
|
|||||||
|
|
||||||
# Returns container JSON
|
# Returns container JSON
|
||||||
function inspect_container {
|
function inspect_container {
|
||||||
local container_id="$1"
|
local container_id=$1
|
||||||
curl_docker "containers/$container_id/json"
|
curl_docker "containers/$container_id/json"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Gets some lines from docker log
|
|
||||||
function get_logs {
|
|
||||||
container_id="$1"
|
|
||||||
num_lines="$2"
|
|
||||||
curl_docker "containers/$container_id/logs?stdout=1&stderr=1" | tail -n "$num_lines"
|
|
||||||
}
|
|
||||||
|
|
||||||
if [ -z "$container_name" ]; then
|
if [ -z "$container_name" ]; then
|
||||||
echo "Usage: $0 container_name [num_log_lines]"
|
echo "Usage: $0 container_name"
|
||||||
echo "Will exit with the last status code of continer with provided name"
|
echo "Will exit with the last status code of continer with provided name"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
container_id=$(get_container_id "$container_name")
|
container_id=$(get_container_id $container_name)
|
||||||
if [ -z "$container_id" ]; then
|
if [ -z "$container_id" ]; then
|
||||||
echo "ERROR: Could not find container with name: $container_name"
|
echo "ERROR: Could not find container with name: $container_name"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
exit_code=$(inspect_container "$container_id" | jq -r .State.ExitCode)
|
exit_code=$(inspect_container "$container_id" | jq -r .State.ExitCode)
|
||||||
|
|
||||||
if [ -n "$num_log_lines" ]; then
|
|
||||||
get_logs "$container_id" "$num_log_lines"
|
|
||||||
fi
|
|
||||||
|
|
||||||
exit "$exit_code"
|
exit "$exit_code"
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ set -e
|
|||||||
# To override, export DOCKER_HOST to a new hostname
|
# To override, export DOCKER_HOST to a new hostname
|
||||||
DOCKER_HOST="${DOCKER_HOST:=socket}"
|
DOCKER_HOST="${DOCKER_HOST:=socket}"
|
||||||
container_name="$1"
|
container_name="$1"
|
||||||
num_log_lines="$2"
|
|
||||||
|
|
||||||
# Curls Docker either using a socket or URL
|
# Curls Docker either using a socket or URL
|
||||||
function curl_docker {
|
function curl_docker {
|
||||||
@@ -36,15 +35,8 @@ function inspect_container {
|
|||||||
curl_docker "containers/$container_id/json"
|
curl_docker "containers/$container_id/json"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Gets some lines from docker log
|
|
||||||
function get_logs {
|
|
||||||
container_id="$1"
|
|
||||||
num_lines="$2"
|
|
||||||
curl_docker "containers/$container_id/logs?stdout=1&stderr=1" | tail -n "$num_lines"
|
|
||||||
}
|
|
||||||
|
|
||||||
if [ -z "$container_name" ]; then
|
if [ -z "$container_name" ]; then
|
||||||
echo "Usage: $0 container_name [num_log_lines]"
|
echo "Usage: $0 container_name"
|
||||||
echo "Will return results of healthcheck for continer with provided name"
|
echo "Will return results of healthcheck for continer with provided name"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
@@ -56,10 +48,6 @@ if [ -z "$container_id" ]; then
|
|||||||
fi
|
fi
|
||||||
health=$(inspect_container "$container_id" | jq -r '.State.Health.Status')
|
health=$(inspect_container "$container_id" | jq -r '.State.Health.Status')
|
||||||
|
|
||||||
if [ -n "$num_log_lines" ]; then
|
|
||||||
get_logs "$container_id" "$num_log_lines"
|
|
||||||
fi
|
|
||||||
|
|
||||||
case "$health" in
|
case "$health" in
|
||||||
null)
|
null)
|
||||||
echo "No healthcheck results"
|
echo "No healthcheck results"
|
||||||
|
|||||||
@@ -1 +0,0 @@
|
|||||||
check_interval = "woops, I'm not an int!"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
check_interval = "1s"
|
|
||||||
|
|
||||||
monitor "Command" {
|
|
||||||
command = ["echo", "$PATH"]
|
|
||||||
alert_down = [ "alert_down", "log_shell", "log_command" ]
|
|
||||||
alert_every = 0
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
check_interval: 1
|
||||||
|
|
||||||
|
monitors:
|
||||||
|
- name: Command
|
||||||
|
command: ['echo', '$PATH']
|
||||||
|
alert_down: [ 'alert_down', 'log_shell', 'log_command' ]
|
||||||
|
# alert_every: -1
|
||||||
|
alert_every: 0
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
check_interval: woops, I'm not an int!
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
check_interval = "1s"
|
|
||||||
|
|
||||||
monitor "Command" {
|
|
||||||
command = ["echo", "$PATH"]
|
|
||||||
alert_down = ["not_log"]
|
|
||||||
alert_every = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
alert "log" {
|
|
||||||
command = ["true"]
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
check_interval: 1
|
||||||
|
|
||||||
|
monitors:
|
||||||
|
- name: Command
|
||||||
|
command: ['echo', '$PATH']
|
||||||
|
alert_down: [ 'not_log']
|
||||||
|
# alert_every: -1
|
||||||
|
alert_every: 0
|
||||||
|
|
||||||
|
|
||||||
|
alerts:
|
||||||
|
log:
|
||||||
|
command: ['true']
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
check_interval = "1s"
|
|
||||||
|
|
||||||
alert "log_command" {
|
|
||||||
command = "should be a list"
|
|
||||||
}
|
|
||||||
|
|
||||||
monitor "Command" {
|
|
||||||
command = ["echo", "$PATH"]
|
|
||||||
alert_down = ["log_command"]
|
|
||||||
alert_every = 2
|
|
||||||
check_interval = "10s"
|
|
||||||
}
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
check_interval = "1s"
|
|
||||||
default_alert_down = ["log_command"]
|
|
||||||
default_alert_every = 0
|
|
||||||
default_alert_after = 2
|
|
||||||
|
|
||||||
monitor "Default" {
|
|
||||||
command = ["echo"]
|
|
||||||
}
|
|
||||||
|
|
||||||
monitor "Command" {
|
|
||||||
command = ["echo", "$PATH"]
|
|
||||||
}
|
|
||||||
|
|
||||||
alert "log_command" {
|
|
||||||
command = ["echo", "default", "'command!!!'", "{{.MonitorName}}"]
|
|
||||||
}
|
|
||||||
@@ -1,34 +0,0 @@
|
|||||||
check_interval = "1s"
|
|
||||||
|
|
||||||
alert "log_command" {
|
|
||||||
command = ["echo", "regular", "'command!!!'", "{{.MonitorName}}"]
|
|
||||||
}
|
|
||||||
|
|
||||||
alert "log_shell" {
|
|
||||||
shell_command = "echo \"Failure on {{.MonitorName}} User is $USER\""
|
|
||||||
}
|
|
||||||
|
|
||||||
monitor "Default" {
|
|
||||||
command = ["echo"]
|
|
||||||
alert_down = ["log_command"]
|
|
||||||
}
|
|
||||||
|
|
||||||
monitor "Command" {
|
|
||||||
command = ["echo", "$PATH"]
|
|
||||||
alert_down = ["log_command", "log_shell"]
|
|
||||||
alert_every = 2
|
|
||||||
check_interval = "10s"
|
|
||||||
}
|
|
||||||
|
|
||||||
monitor "Shell" {
|
|
||||||
shell_command = <<-EOF
|
|
||||||
echo 'Some string with stuff'
|
|
||||||
echo 'another line'
|
|
||||||
echo $PATH
|
|
||||||
exit 1
|
|
||||||
EOF
|
|
||||||
alert_down = ["log_command", "log_shell"]
|
|
||||||
alert_after = 5
|
|
||||||
alert_every = 0
|
|
||||||
check_interval = "1m"
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
check_interval: 1
|
||||||
|
|
||||||
|
monitors:
|
||||||
|
- name: Command
|
||||||
|
command: ['echo', '$PATH']
|
||||||
|
alert_down: [ 'log_command', 'log_shell' ]
|
||||||
|
alert_every: 0
|
||||||
|
- name: Shell
|
||||||
|
command_shell: >
|
||||||
|
echo 'Some string with stuff';
|
||||||
|
echo 'another line';
|
||||||
|
echo $PATH;
|
||||||
|
exit 1
|
||||||
|
alert_down: [ 'log_command', 'log_shell' ]
|
||||||
|
alert_after: 5
|
||||||
|
alert_every: 0
|
||||||
|
|
||||||
|
alerts:
|
||||||
|
log_command:
|
||||||
|
command: [ 'echo', 'regular', '"command!!!"', "{{.MonitorName}}" ]
|
||||||
|
log_shell:
|
||||||
|
command_shell: echo "Failure on {{.MonitorName}} User is $USER"
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
check_interval = "1s"
|
|
||||||
|
|
||||||
monitor "Shell" {
|
|
||||||
shell_command = <<-EOF
|
|
||||||
echo 'Some string with stuff'
|
|
||||||
echo "<angle brackets>"
|
|
||||||
exit 1
|
|
||||||
EOF
|
|
||||||
alert_down = ["log_shell"]
|
|
||||||
alert_after = 1
|
|
||||||
alert_every = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
alert "log_shell" {
|
|
||||||
shell_command = <<EOF
|
|
||||||
echo 'Some string with stuff'
|
|
||||||
echo '<angle brackets>'
|
|
||||||
EOF
|
|
||||||
}
|
|
||||||
@@ -5,10 +5,20 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// escapeCommandShell accepts a command to be executed by a shell and escapes it
|
||||||
|
func escapeCommandShell(command string) string {
|
||||||
|
// Remove extra spaces and newlines from ends
|
||||||
|
command = strings.TrimSpace(command)
|
||||||
|
// TODO: Not sure if this part is actually needed. Should verify
|
||||||
|
// Escape double quotes since this will be passed in as an argument
|
||||||
|
command = strings.Replace(command, `"`, `\"`, -1)
|
||||||
|
return command
|
||||||
|
}
|
||||||
|
|
||||||
// ShellCommand takes a string and executes it as a command using `sh`
|
// ShellCommand takes a string and executes it as a command using `sh`
|
||||||
func ShellCommand(command string) *exec.Cmd {
|
func ShellCommand(command string) *exec.Cmd {
|
||||||
shellCommand := []string{"sh", "-c", strings.TrimSpace(command)}
|
shellCommand := []string{"sh", "-c", escapeCommandShell(command)}
|
||||||
|
//log.Printf("Shell command: %v", shellCommand)
|
||||||
return exec.Command(shellCommand[0], shellCommand[1:]...)
|
return exec.Command(shellCommand[0], shellCommand[1:]...)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -17,12 +27,10 @@ func EqualSliceString(a, b []string) bool {
|
|||||||
if len(a) != len(b) {
|
if len(a) != len(b) {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
for i, val := range a {
|
for i, val := range a {
|
||||||
if val != b[i] {
|
if val != b[i] {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|||||||
+8
-17
@@ -1,9 +1,6 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import "testing"
|
||||||
"fmt"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestUtilEqualSliceString(t *testing.T) {
|
func TestUtilEqualSliceString(t *testing.T) {
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
@@ -24,18 +21,12 @@ func TestUtilEqualSliceString(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
c := c
|
actual := EqualSliceString(c.a, c.b)
|
||||||
|
if actual != c.expected {
|
||||||
t.Run(fmt.Sprintf("%v %v", c.a, c.b), func(t *testing.T) {
|
t.Errorf(
|
||||||
t.Parallel()
|
"EqualSliceString(%v, %v), expected=%v actual=%v",
|
||||||
|
c.a, c.b, c.expected, actual,
|
||||||
actual := EqualSliceString(c.a, c.b)
|
)
|
||||||
if actual != c.expected {
|
}
|
||||||
t.Errorf(
|
|
||||||
"EqualSliceString(%v, %v), expected=%v actual=%v",
|
|
||||||
c.a, c.b, c.expected, actual,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user