Compare commits

..

1 Commits

Author SHA1 Message Date
Ian Fijolek
e4e6618af1 Adds ability to run specified alerts on startup
This is helpful to determine if your alerts are valid before an actual failure
2024-04-03 11:50:49 -07:00
10 changed files with 104 additions and 100 deletions
+42 -6
View File
@@ -60,15 +60,51 @@ steps:
when: when:
event: tag event: tag
- name: Build and publish docker images - name: push image - arm
image: thegeeklab/drone-docker-buildx image: plugins/docker
settings: settings:
repo: iamthefij/minitor-go repo: iamthefij/minitor-go
auto_tag: true auto_tag: true
platforms: auto_tag_suffix: linux-arm
- linux/amd64 username:
- linux/arm64 from_secret: docker_username
- linux/arm password:
from_secret: docker_password
build_args:
- ARCH=arm
- REPO=arm32v7
- name: push image - arm64
image: plugins/docker
settings:
repo: iamthefij/minitor-go
auto_tag: true
auto_tag_suffix: linux-arm64
username:
from_secret: docker_username
password:
from_secret: docker_password
build_args:
- ARCH=arm64
- REPO=arm64v8
- name: push image - amd64
image: plugins/docker
settings:
repo: iamthefij/minitor-go
auto_tag: true
auto_tag_suffix: linux-amd64
username:
from_secret: docker_username
password:
from_secret: docker_password
- name: publish manifest
image: plugins/manifest
settings:
spec: manifest.tmpl
auto_tag: true
ignore_missing: true
username: username:
from_secret: docker_username from_secret: docker_username
password: password:
+5 -5
View File
@@ -1,10 +1,11 @@
FROM alpine:3.18 ARG REPO=library
FROM ${REPO}/alpine:3.18
RUN mkdir /app RUN mkdir /app
WORKDIR /app/ WORKDIR /app/
# Add common checking tools # Add common checking tools
RUN apk --no-cache add bash=~5 curl=~8 jq=~1 bind-tools=~9 tzdata~=2024a RUN apk --no-cache add bash=~5 curl=~8 jq=~1 bind-tools=~9 tzdata~=2023c
# Add minitor user for running as non-root # Add minitor user for running as non-root
RUN addgroup -S minitor && adduser -S minitor -G minitor RUN addgroup -S minitor && adduser -S minitor -G minitor
@@ -14,9 +15,8 @@ COPY ./scripts /app/scripts
RUN chmod -R 755 /app/scripts RUN chmod -R 755 /app/scripts
# Copy minitor in # Copy minitor in
ARG TARGETOS ARG ARCH=amd64
ARG TARGETARCH COPY ./dist/minitor-linux-${ARCH} ./minitor
COPY ./dist/minitor-${TARGETOS}-${TARGETARCH} ./minitor
# Drop to non-root user # Drop to non-root user
USER minitor USER minitor
+5 -5
View File
@@ -1,3 +1,4 @@
ARG REPO=library
FROM golang:1.20 AS builder FROM golang:1.20 AS builder
RUN mkdir /app RUN mkdir /app
@@ -8,13 +9,12 @@ RUN go mod download
COPY ./*.go /app/ COPY ./*.go /app/
ARG TARGETOS ARG ARCH=amd64
ARG TARGETARCH
ARG VERSION=dev ARG VERSION=dev
ENV CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=${TARGETARCH} ENV CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH}
RUN go build -ldflags "-X main.version=${VERSION}" -a -installsuffix nocgo -o minitor . RUN go build -ldflags "-X main.version=${VERSION}" -a -installsuffix nocgo -o minitor .
FROM alpine:3.18 FROM ${REPO}/alpine:3.18
RUN mkdir /app RUN mkdir /app
WORKDIR /app/ WORKDIR /app/
@@ -22,7 +22,7 @@ WORKDIR /app/
COPY --from=builder /app/minitor . COPY --from=builder /app/minitor .
# Add common checking tools # Add common checking tools
RUN apk --no-cache add bash=~5 curl=~8 jq=~1 bind-tools=~9 tzdata~=2024a RUN apk --no-cache add bash=~5 curl=~8 jq=~1 bind-tools=~9 tzdata~=2023c
# Add minitor user for running as non-root # Add minitor user for running as non-root
RUN addgroup -S minitor && adduser -S minitor -G minitor RUN addgroup -S minitor && adduser -S minitor -G minitor
+2 -2
View File
@@ -79,11 +79,11 @@ $(TARGET_ALIAS):
# Arch specific docker build targets # Arch specific docker build targets
.PHONY: docker-build-arm .PHONY: docker-build-arm
docker-build-arm: dist/minitor-linux-arm docker-build-arm: dist/minitor-linux-arm
docker build --platform linux/arm . -t ${DOCKER_TAG}-linux-arm docker build --build-arg REPO=arm32v7 --build-arg ARCH=arm . -t ${DOCKER_TAG}-linux-arm
.PHONY: docker-build-arm64 .PHONY: docker-build-arm64
docker-build-arm64: dist/minitor-linux-arm64 docker-build-arm64: dist/minitor-linux-arm64
docker build --platform linux/arm64 . -t ${DOCKER_TAG}-linux-arm64 docker build --build-arg REPO=arm64v8 --build-arg ARCH=arm64 . -t ${DOCKER_TAG}-linux-arm64
# Cross run on host architechture # Cross run on host architechture
.PHONY: docker-run-arm .PHONY: docker-run-arm
-11
View File
@@ -58,7 +58,6 @@ The global configurations are:
|---|---| |---|---|
|`check_interval`|Maximum frequency to run checks for each monitor as duration, eg. 1m2s.| |`check_interval`|Maximum frequency to run checks for each monitor as duration, eg. 1m2s.|
|`default_alert_after`|A default value used as an `alert_after` value for a monitor if not specified or 0.| |`default_alert_after`|A default value used as an `alert_after` value for a monitor if not specified or 0.|
|`default_alert_every`|A default value used as an `alert_every` value for a monitor if not specified.|
|`default_alert_down`|Default down alerts to used by a monitor in case none are provided.| |`default_alert_down`|Default down alerts to used by a monitor in case none are provided.|
|`default_alert_up`|Default up alerts to used by a monitor in case none are provided.| |`default_alert_up`|Default up alerts to used by a monitor in case none are provided.|
|`monitors`|List of all monitors. Detailed description below| |`monitors`|List of all monitors. Detailed description below|
@@ -119,16 +118,6 @@ To provide flexible formatting, the following non-standard functions are availab
For more information, check out the [Go documentation for the time module](https://pkg.go.dev/time@go1.20.7#pkg-constants). For more information, check out the [Go documentation for the time module](https://pkg.go.dev/time@go1.20.7#pkg-constants).
#### Running alerts on startup
It's not the best feeling to find out your alerts are broken when you're expecting to be alerted about another failure. To avoid this and provide early insight into broken alerts, it is possible to specify a list of alerts to run when Minitor starts up. This can be done using the command line flag `-startup-alerts`. This flag accepts a comma separated list of strings and will run a test of each of those alerts. Minitor will then respond as it typically does for any failed alert. This can be used to allow you time to correct when initially launching, and to allow schedulers to more easily detect a failed deployment of Minitor.
Eg.
```bash
minitor -startup-alerts=log_down,log_up -config ./config.yml
```
### Metrics ### Metrics
Minitor supports exporting metrics for [Prometheus](https://prometheus.io/). Prometheus is an open source tool for reading and querying metrics from different sources. Combined with another tool, [Grafana](https://grafana.com/), it allows building of charts and dashboards. You could also opt to just use Minitor to log check results, and instead do your alerting with Grafana. Minitor supports exporting metrics for [Prometheus](https://prometheus.io/). Prometheus is an open source tool for reading and querying metrics from different sources. Combined with another tool, [Grafana](https://grafana.com/), it allows building of charts and dashboards. You could also opt to just use Minitor to log check results, and instead do your alerting with Grafana.
+6 -10
View File
@@ -92,7 +92,7 @@ func checkMonitors(config *Config) error {
return nil return nil
} }
func sendStartupAlerts(config *Config, alertNames []string) error { func sendFirstRunAlerts(config *Config, alertNames []string) error {
for _, alertName := range alertNames { for _, alertName := range alertNames {
var err error var err error
@@ -123,7 +123,7 @@ func sendStartupAlerts(config *Config, alertNames []string) error {
func main() { func main() {
showVersion := flag.Bool("version", false, "Display the version of minitor and exit") showVersion := flag.Bool("version", false, "Display the version of minitor and exit")
configPath := flag.String("config", "config.yml", "Alternate configuration path (default: config.yml)") configPath := flag.String("config", "config.yml", "Alternate configuration path (default: config.yml)")
startupAlerts := flag.String("startup-alerts", "", "List of alerts to run on startup. This can help determine unhealthy alerts early on. (default \"\")") firstRunAlerts := flag.String("first-run-alerts", "", "List of alerts to run on startup. This can help determine unhealthy alerts early on. (default \"\")")
flag.BoolVar(&slog.DebugLevel, "debug", false, "Enables debug logs (default: false)") flag.BoolVar(&slog.DebugLevel, "debug", false, "Enables debug logs (default: false)")
flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)") flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)")
@@ -138,10 +138,6 @@ func main() {
return return
} }
if PyCompat {
slog.Warningf("Python compatibility mode is enabled. This will be removed in the next major release. Please update your configuration.")
}
// Load configuration // Load configuration
config, err := LoadConfig(*configPath) config, err := LoadConfig(*configPath)
slog.OnErrFatalf(err, "Error loading config: %v", err) slog.OnErrFatalf(err, "Error loading config: %v", err)
@@ -153,12 +149,12 @@ func main() {
go ServeMetrics() go ServeMetrics()
} }
if *startupAlerts != "" { if *firstRunAlerts != "" {
alertNames := strings.Split(*startupAlerts, ",") alertNames := strings.Split(*firstRunAlerts, ",")
err = sendStartupAlerts(&config, alertNames) err = sendFirstRunAlerts(&config, alertNames)
slog.OnErrPanicf(err, "Error running startup alerts") slog.OnErrPanicf(err, "Error running first run alerts")
} }
// Start main loop // Start main loop
+19 -19
View File
@@ -137,22 +137,22 @@ func TestCheckMonitors(t *testing.T) {
func TestFirstRunAlerts(t *testing.T) { func TestFirstRunAlerts(t *testing.T) {
cases := []struct { cases := []struct {
config Config config Config
expectErr bool expectErr bool
startupAlerts []string firstRunAlerts []string
name string name string
}{ }{
{ {
config: Config{}, config: Config{},
expectErr: false, expectErr: false,
startupAlerts: []string{}, firstRunAlerts: []string{},
name: "Empty", name: "Empty",
}, },
{ {
config: Config{}, config: Config{},
expectErr: true, expectErr: true,
startupAlerts: []string{"missing"}, firstRunAlerts: []string{"missing"},
name: "Unknown", name: "Unknown",
}, },
{ {
config: Config{ config: Config{
@@ -162,9 +162,9 @@ func TestFirstRunAlerts(t *testing.T) {
}, },
}, },
}, },
expectErr: false, expectErr: false,
startupAlerts: []string{"good"}, firstRunAlerts: []string{"good"},
name: "Successful alert", name: "Successful alert",
}, },
{ {
config: Config{ config: Config{
@@ -175,9 +175,9 @@ func TestFirstRunAlerts(t *testing.T) {
}, },
}, },
}, },
expectErr: true, expectErr: true,
startupAlerts: []string{"bad"}, firstRunAlerts: []string{"bad"},
name: "Failed alert", name: "Failed alert",
}, },
} }
@@ -187,7 +187,7 @@ func TestFirstRunAlerts(t *testing.T) {
t.Errorf("sendFirstRunAlerts(%s): unexpected error reading config: %v", c.name, err) t.Errorf("sendFirstRunAlerts(%s): unexpected error reading config: %v", c.name, err)
} }
err = sendStartupAlerts(&c.config, c.startupAlerts) err = sendFirstRunAlerts(&c.config, c.firstRunAlerts)
if err == nil && c.expectErr { if err == nil && c.expectErr {
t.Errorf("sendFirstRunAlerts(%s): Expected error, the code did not error", c.name) t.Errorf("sendFirstRunAlerts(%s): Expected error, the code did not error", c.name)
} else if err != nil && !c.expectErr { } else if err != nil && !c.expectErr {
+25
View File
@@ -0,0 +1,25 @@
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}
{{#if build.tags}}
tags:
{{#each build.tags}}
- {{this}}
{{/each}}
{{/if}}
manifests:
-
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64
platform:
architecture: amd64
os: linux
-
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64
platform:
architecture: arm64
os: linux
variant: v8
-
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm
platform:
architecture: arm
os: linux
variant: v7
-1
View File
@@ -149,7 +149,6 @@ func (monitor *Monitor) failure() (notice *AlertNotice) {
// If we're going to alert, increment count // If we're going to alert, increment count
if notice != nil { if notice != nil {
monitor.alertCount++ monitor.alertCount++
notice.AlertCount = monitor.alertCount
} }
return notice return notice
-41
View File
@@ -138,47 +138,6 @@ func TestMonitorSuccess(t *testing.T) {
} }
} }
func TestMonitorAlertCount(t *testing.T) {
var alertEvery int16 = 1
cases := []struct {
checkSuccess bool
alertCount int16
name string
}{
{false, 1, "First failure and first alert"},
{false, 2, "Second failure and first alert"},
{true, 2, "Success should preserve past alert count"},
{false, 1, "First failure and first alert after success"},
}
// Unlike previous tests, this one requires a static Monitor with repeated
// calls to the failure method
monitor := Monitor{failureCount: 0, AlertAfter: 1, AlertEvery: &alertEvery}
for _, c := range cases {
log.Printf("Testing case %s", c.name)
var notice *AlertNotice
if c.checkSuccess {
notice = monitor.success()
} else {
notice = monitor.failure()
}
if notice == nil {
t.Errorf("failure(%v) expected notice, got nil", c.name)
}
if notice.AlertCount != c.alertCount {
t.Errorf("failure(%v), expected=%v actual=%v", c.name, c.alertCount, notice.AlertCount)
log.Printf("Case failed: %s", c.name)
}
log.Println("-----")
}
}
// TestMonitorFailureAlertAfter tests that alerts will not trigger until // TestMonitorFailureAlertAfter tests that alerts will not trigger until
// hitting the threshold provided by AlertAfter // hitting the threshold provided by AlertAfter
func TestMonitorFailureAlertAfter(t *testing.T) { func TestMonitorFailureAlertAfter(t *testing.T) {