Stop getting errors when tzdata updates

This is helpful to determine if your alerts are valid before an actual failure

.drone.yml

@@ -60,51 +60,15 @@ steps:
     when:
       event: tag
 
-  - name: push image - arm
+  - name: Build and publish docker images
-    image: plugins/docker
+    image: thegeeklab/drone-docker-buildx
     settings:
       repo: iamthefij/minitor-go
       auto_tag: true
-      auto_tag_suffix: linux-arm
+      platforms:
-      username:
+        - linux/amd64
-        from_secret: docker_username
+        - linux/arm64
-      password:
+        - linux/arm
-        from_secret: docker_password
-      build_args:
-        - ARCH=arm
-        - REPO=arm32v7
-
-  - name: push image - arm64
-    image: plugins/docker
-    settings:
-      repo: iamthefij/minitor-go
-      auto_tag: true
-      auto_tag_suffix: linux-arm64
-      username:
-        from_secret: docker_username
-      password:
-        from_secret: docker_password
-      build_args:
-        - ARCH=arm64
-        - REPO=arm64v8
-
-  - name: push image - amd64
-    image: plugins/docker
-    settings:
-      repo: iamthefij/minitor-go
-      auto_tag: true
-      auto_tag_suffix: linux-amd64
-      username:
-        from_secret: docker_username
-      password:
-        from_secret: docker_password
-
-  - name: publish manifest
-    image: plugins/manifest
-    settings:
-      spec: manifest.tmpl
-      auto_tag: true
-      ignore_missing: true
       username:
         from_secret: docker_username
       password:

Dockerfile

@@ -1,11 +1,11 @@
-ARG REPO=library
+FROM alpine:3.18
-FROM ${REPO}/alpine:3.18
 
 RUN mkdir /app
 WORKDIR /app/
 
 # Add common checking tools
-RUN apk --no-cache add bash=~5 curl=~8 jq=~1 bind-tools=~9 tzdata~=2023c
+# hadolint ignore=DL3018
+RUN apk --no-cache add bash=~5 curl=~8 jq=~1 bind-tools=~9 tzdata
 
 # Add minitor user for running as non-root
 RUN addgroup -S minitor && adduser -S minitor -G minitor
@@ -15,8 +15,9 @@ COPY ./scripts /app/scripts
 RUN chmod -R 755 /app/scripts
 
 # Copy minitor in
-ARG ARCH=amd64
+ARG TARGETOS
-COPY ./dist/minitor-linux-${ARCH} ./minitor
+ARG TARGETARCH
+COPY ./dist/minitor-${TARGETOS}-${TARGETARCH} ./minitor
 
 # Drop to non-root user
 USER minitor

Dockerfile.multi-stage

@@ -1,4 +1,3 @@
-ARG REPO=library
 FROM golang:1.20 AS builder
 
 RUN mkdir /app
@@ -9,12 +8,13 @@ RUN go mod download
 
 COPY ./*.go /app/
 
-ARG ARCH=amd64
+ARG TARGETOS
+ARG TARGETARCH
 ARG VERSION=dev
-ENV CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH}
+ENV CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=${TARGETARCH}
 RUN go build -ldflags "-X main.version=${VERSION}" -a -installsuffix nocgo -o minitor .
 
-FROM ${REPO}/alpine:3.18
+FROM alpine:3.18
 RUN mkdir /app
 WORKDIR /app/
 
@@ -22,7 +22,8 @@ WORKDIR /app/
 COPY --from=builder /app/minitor .
 
 # Add common checking tools
-RUN apk --no-cache add bash=~5 curl=~8 jq=~1 bind-tools=~9 tzdata~=2023c
+# hadolint ignore=DL3018
+RUN apk --no-cache add bash=~5 curl=~8 jq=~1 bind-tools=~9 tzdata
 
 # Add minitor user for running as non-root
 RUN addgroup -S minitor && adduser -S minitor -G minitor

Makefile

@@ -79,11 +79,11 @@ $(TARGET_ALIAS):
 # Arch specific docker build targets
 .PHONY: docker-build-arm
 docker-build-arm: dist/minitor-linux-arm
-	docker build --build-arg REPO=arm32v7 --build-arg ARCH=arm . -t ${DOCKER_TAG}-linux-arm
+	docker build --platform linux/arm . -t ${DOCKER_TAG}-linux-arm
 
 .PHONY: docker-build-arm64
 docker-build-arm64: dist/minitor-linux-arm64
-	docker build --build-arg REPO=arm64v8 --build-arg ARCH=arm64 . -t ${DOCKER_TAG}-linux-arm64
+	docker build  --platform linux/arm64 . -t ${DOCKER_TAG}-linux-arm64
 
 # Cross run on host architechture
 .PHONY: docker-run-arm

README.md

@@ -58,6 +58,7 @@ The global configurations are:
 |---|---|
 |`check_interval`|Maximum frequency to run checks for each monitor as duration, eg. 1m2s.|
 |`default_alert_after`|A default value used as an `alert_after` value for a monitor if not specified or 0.|
+|`default_alert_every`|A default value used as an `alert_every` value for a monitor if not specified.|
 |`default_alert_down`|Default down alerts to used by a monitor in case none are provided.|
 |`default_alert_up`|Default up alerts to used by a monitor in case none are provided.|
 |`monitors`|List of all monitors. Detailed description below|
@@ -118,6 +119,16 @@ To provide flexible formatting, the following non-standard functions are availab
 
 For more information, check out the [Go documentation for the time module](https://pkg.go.dev/time@go1.20.7#pkg-constants).
 
+#### Running alerts on startup
+
+It's not the best feeling to find out your alerts are broken when you're expecting to be alerted about another failure. To avoid this and provide early insight into broken alerts, it is possible to specify a list of alerts to run when Minitor starts up. This can be done using the command line flag `-startup-alerts`. This flag accepts a comma separated list of strings and will run a test of each of those alerts. Minitor will then respond as it typically does for any failed alert. This can be used to allow you time to correct when initially launching, and to allow schedulers to more easily detect a failed deployment of Minitor.
+
+Eg.
+
+```bash
+minitor -startup-alerts=log_down,log_up -config ./config.yml
+```
+
 ### Metrics
 
 Minitor supports exporting metrics for [Prometheus](https://prometheus.io/). Prometheus is an open source tool for reading and querying metrics from different sources. Combined with another tool, [Grafana](https://grafana.com/), it allows building of charts and dashboards. You could also opt to just use Minitor to log check results, and instead do your alerting with Grafana.

go.mod

@@ -4,7 +4,7 @@ go 1.20
 
 require (
 	git.iamthefij.com/iamthefij/slog v1.3.0
-	github.com/prometheus/client_golang v1.15.0
+	github.com/prometheus/client_golang v1.19.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
@@ -14,10 +14,10 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
-	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/common v0.48.0 // indirect
-	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/rogpeppe/go-internal v1.10.0 // indirect
-	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 )

go.sum

@@ -182,12 +182,16 @@ github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrb
 github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
 github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM=
 github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
+github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
+github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
 github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
+github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
@@ -195,6 +199,8 @@ github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+
 github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
 github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
 github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
+github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
+github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
@@ -203,6 +209,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
 github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
 github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
@@ -365,6 +373,8 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -504,6 +514,10 @@ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
 google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
+google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

health.go

@@ -1,72 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"io"
-	"net/http"
-	"strings"
-)
-
-type HealthCheckHandler struct {
-	isMinitorHealthy bool
-	monitors         []*Monitor
-}
-
-func NewHealthCheckHandler(monitors []*Monitor) *HealthCheckHandler {
-	return &HealthCheckHandler{
-		false,
-		monitors,
-	}
-}
-
-func (hch *HealthCheckHandler) MinitorHealthy(healthy bool) {
-	hch.isMinitorHealthy = healthy
-}
-
-func (hch HealthCheckHandler) MinitorHealthCheck() (bool, string) {
-	if hch.isMinitorHealthy {
-		return true, "OK"
-	} else {
-		return false, "UNHEALTHY"
-	}
-}
-
-func (hch HealthCheckHandler) MonitorsHealthCheck() (bool, string) {
-	downMonitors := []string{}
-
-	for _, monitor := range hch.monitors {
-		if !monitor.IsUp() {
-			downMonitors = append(downMonitors, monitor.Name)
-		}
-	}
-
-	if len(downMonitors) == 0 {
-		return true, "OK"
-	} else {
-		return false, fmt.Sprintf("UNHEALTHY: The following monitors are unhealthy: %s", strings.Join(downMonitors, ", "))
-	}
-}
-
-func (hch HealthCheckHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	var healthy bool
-
-	var body string
-
-	if monitors := r.URL.Query().Get("monitors"); monitors != "" {
-		healthy, body = hch.MonitorsHealthCheck()
-	} else {
-		healthy, body = hch.MinitorHealthCheck()
-	}
-
-	if healthy {
-		w.WriteHeader(http.StatusOK)
-	} else {
-		w.WriteHeader(http.StatusServiceUnavailable)
-	}
-
-	_, _ = io.WriteString(w, body)
-}
-
-func HandleHealthCheck() {
-	http.Handle("/metrics", HealthChecks)
-}

health_test.go

@@ -1,79 +0,0 @@
-package main
-
-import (
-	"testing"
-)
-
-func TestNewHealthCheck(t *testing.T) {
-	monitors := []*Monitor{
-		{Name: "Test Monitor"},
-	}
-	hc := NewHealthCheckHandler(monitors)
-
-	monitors[0].alertCount++
-
-	if healthy, _ := hc.MinitorHealthCheck(); healthy {
-		t.Errorf("Initial hc state should be unhealthy until some successful alert is sent")
-	}
-
-	if healthy, _ := hc.MonitorsHealthCheck(); healthy {
-		t.Errorf("Faking an alert on the monitor pointer should make this unhealthy")
-	}
-}
-
-func TestMinitorHealthCheck(t *testing.T) {
-	monitors := []*Monitor{
-		{Name: "Test Monitor"},
-	}
-	hc := NewHealthCheckHandler(monitors)
-
-	t.Run("MinitorHealthCheck(healthy)", func(t *testing.T) {
-		hc.MinitorHealthy(true)
-		healthy, body := hc.MinitorHealthCheck()
-		if !healthy {
-			t.Errorf("Expected healthy check")
-		}
-		if body != "OK" {
-			t.Errorf("Expected OK response")
-		}
-	})
-
-	t.Run("MinitorHealthCheck(unhealthy)", func(t *testing.T) {
-		hc.MinitorHealthy(false)
-		healthy, body := hc.MinitorHealthCheck()
-		if healthy {
-			t.Errorf("Expected healthy check")
-		}
-		if body != "UNHEALTHY" {
-			t.Errorf("Expected UNHEALTHY response")
-		}
-	})
-}
-
-func TestMonitorsHealthCheck(t *testing.T) {
-	monitors := []*Monitor{
-		{Name: "Test Monitor"},
-	}
-	hc := NewHealthCheckHandler(monitors)
-
-	t.Run("MonitorsHealthCheck(healthy)", func(t *testing.T) {
-		healthy, body := hc.MonitorsHealthCheck()
-		if !healthy {
-			t.Errorf("Expected healthy check")
-		}
-		if body != "OK" {
-			t.Errorf("Expected OK response")
-		}
-	})
-
-	t.Run("MonitorsHealthCheck(unhealthy)", func(t *testing.T) {
-		monitors[0].alertCount++
-		healthy, body := hc.MonitorsHealthCheck()
-		if healthy {
-			t.Errorf("Expected healthy check")
-		}
-		if body != "UNHEALTHY: The following monitors are unhealthy: Test Monitor" {
-			t.Errorf("Expected UNHEALTHY response")
-		}
-	})
-}

main.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"net/http"
+	"strings"
 	"time"
 
 	"git.iamthefij.com/iamthefij/slog"
@@ -17,10 +17,6 @@ var (
 	MetricsPort = 8080
 	// Metrics contains all active metrics
 	Metrics = NewMetrics()
-	// Self monitor rather than panicing
-	SelfMonitor = false
-	// HealthChecks contains health check values
-	HealthChecks *HealthCheckHandler = nil
 
 	// PyCompat enables support for legacy Python templates
 	PyCompat = false
@@ -56,13 +52,7 @@ func sendAlerts(config *Config, monitor *Monitor, alertNotice *AlertNotice) erro
 					output,
 				)
 
-				if SelfMonitor {
-					Metrics.SetMonitorStatus(fmt.Sprintf("Alert %s", alertName), false)
-				}
-
 				return err
-			} else {
-				Metrics.SetMonitorStatus(fmt.Sprintf("Alert %s", alertName), true)
 			}
 
 			// Count alert metrics
@@ -80,8 +70,6 @@ func sendAlerts(config *Config, monitor *Monitor, alertNotice *AlertNotice) erro
 
 func checkMonitors(config *Config) error {
 	// TODO: Run this in goroutines and capture exceptions
-	healthy := true
-
 	for _, monitor := range config.Monitors {
 		if monitor.ShouldCheck() {
 			success, alertNotice := monitor.Check()
@@ -93,42 +81,54 @@ func checkMonitors(config *Config) error {
 
 			if alertNotice != nil {
 				err := sendAlerts(config, monitor, alertNotice)
-				// If there was an error in sending an alert, mark as unhealthy or bubble up
+				// If there was an error in sending an alert, exit early and bubble it up
 				if err != nil {
-					if SelfMonitor {
+					return err
-						healthy = false
-					} else {
-						return err
-					}
 				}
 			}
 		}
 	}
 
-	if HealthChecks != nil {
+	return nil
-		HealthChecks.MinitorHealthy(healthy)
+}
+
+func sendStartupAlerts(config *Config, alertNames []string) error {
+	for _, alertName := range alertNames {
+		var err error
+
+		alert, ok := config.Alerts[alertName]
+		if !ok {
+			err = fmt.Errorf("unknown alert %s: %w", alertName, errUnknownAlert)
+		}
+
+		if err == nil {
+			_, err = alert.Send(AlertNotice{
+				AlertCount:      0,
+				FailureCount:    0,
+				IsUp:            true,
+				LastSuccess:     time.Now(),
+				MonitorName:     fmt.Sprintf("First Run Alert Test: %s", alert.Name),
+				LastCheckOutput: "",
+			})
+		}
+
+		if err != nil {
+			return err
+		}
 	}
 
 	return nil
 }
 
-// ServeMetricsAndHealth starts the default http server
-func ServeMetricsAndHealth() {
-	host := fmt.Sprintf(":%d", MetricsPort)
-
-	_ = http.ListenAndServe(host, nil)
-}
-
 func main() {
 	showVersion := flag.Bool("version", false, "Display the version of minitor and exit")
 	configPath := flag.String("config", "config.yml", "Alternate configuration path (default: config.yml)")
+	startupAlerts := flag.String("startup-alerts", "", "List of alerts to run on startup. This can help determine unhealthy alerts early on. (default \"\")")
 
 	flag.BoolVar(&slog.DebugLevel, "debug", false, "Enables debug logs (default: false)")
 	flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)")
 	flag.BoolVar(&PyCompat, "py-compat", false, "Enables support for legacy Python Minitor config. Will eventually be removed. (default: false)")
-	flag.IntVar(&MetricsPort, "metrics-port", MetricsPort, "The port that Prometheus metrics and healthchecks should be exported on, if enabled. (default: 8080)")
+	flag.IntVar(&MetricsPort, "metrics-port", MetricsPort, "The port that Prometheus metrics should be exported on, if enabled. (default: 8080)")
-	flag.BoolVar(&SelfMonitor, "self-monitor", false, "Enables self-monitoring. Export metrics rather than panic when alerts fail. (default: false)")
-
 	flag.Parse()
 
 	// Print version if flag is provided
@@ -138,6 +138,10 @@ func main() {
 		return
 	}
 
+	if PyCompat {
+		slog.Warningf("Python compatibility mode is enabled. This will be removed in the next major release. Please update your configuration.")
+	}
+
 	// Load configuration
 	config, err := LoadConfig(*configPath)
 	slog.OnErrFatalf(err, "Error loading config: %v", err)
@@ -145,19 +149,16 @@ func main() {
 	// Serve metrics exporter, if specified
 	if ExportMetrics {
 		slog.Infof("Exporting metrics to Prometheus on port %d", MetricsPort)
-		HandleMetrics()
+
+		go ServeMetrics()
 	}
 
-	if SelfMonitor {
+	if *startupAlerts != "" {
-		slog.Infof("Starting healthcheck endpoint on port %d", MetricsPort)
+		alertNames := strings.Split(*startupAlerts, ",")
 
-		HealthChecks = NewHealthCheckHandler(config.Monitors)
+		err = sendStartupAlerts(&config, alertNames)
 
-		HandleHealthCheck()
+		slog.OnErrPanicf(err, "Error running startup alerts")
-	}
-
-	if ExportMetrics || SelfMonitor {
-		go ServeMetricsAndHealth()
 	}
 
 	// Start main loop

main_test.go

@@ -4,10 +4,9 @@ import "testing"
 
 func TestCheckMonitors(t *testing.T) {
 	cases := []struct {
-		config      Config
+		config    Config
-		expectErr   bool
+		expectErr bool
-		name        string
+		name      string
-		selfMonitor bool
 	}{
 		{
 			config:    Config{},
@@ -23,9 +22,8 @@ func TestCheckMonitors(t *testing.T) {
 					},
 				},
 			},
-			expectErr:   false,
+			expectErr: false,
-			name:        "Monitor success, no alerts",
+			name:      "Monitor success, no alerts",
-			selfMonitor: false,
 		},
 		{
 			config: Config{
@@ -37,9 +35,8 @@ func TestCheckMonitors(t *testing.T) {
 					},
 				},
 			},
-			expectErr:   false,
+			expectErr: false,
-			name:        "Monitor failure, no alerts",
+			name:      "Monitor failure, no alerts",
-			selfMonitor: false,
 		},
 		{
 			config: Config{
@@ -51,9 +48,8 @@ func TestCheckMonitors(t *testing.T) {
 					},
 				},
 			},
-			expectErr:   false,
+			expectErr: false,
-			name:        "Monitor recovery, no alerts",
+			name:      "Monitor recovery, no alerts",
-			selfMonitor: false,
 		},
 		{
 			config: Config{
@@ -66,9 +62,8 @@ func TestCheckMonitors(t *testing.T) {
 					},
 				},
 			},
-			expectErr:   true,
+			expectErr: true,
-			name:        "Monitor failure, unknown alerts",
+			name:      "Monitor failure, unknown alerts",
-			selfMonitor: false,
 		},
 		{
 			config: Config{
@@ -81,24 +76,8 @@ func TestCheckMonitors(t *testing.T) {
 					},
 				},
 			},
-			expectErr:   true,
+			expectErr: true,
-			name:        "Monitor recovery, unknown alerts",
+			name:      "Monitor recovery, unknown alerts",
-			selfMonitor: false,
-		},
-		{
-			config: Config{
-				Monitors: []*Monitor{
-					{
-						Name:       "Success",
-						Command:    CommandOrShell{Command: []string{"true"}},
-						AlertUp:    []string{"unknown"},
-						alertCount: 1,
-					},
-				},
-			},
-			expectErr:   false,
-			name:        "Monitor recovery, unknown alerts, with Health Check",
-			selfMonitor: true,
 		},
 		{
 			config: Config{
@@ -116,9 +95,8 @@ func TestCheckMonitors(t *testing.T) {
 					},
 				},
 			},
-			expectErr:   false,
+			expectErr: false,
-			name:        "Monitor failure, successful alert",
+			name:      "Monitor failure, successful alert",
-			selfMonitor: false,
 		},
 		{
 			config: Config{
@@ -137,36 +115,12 @@ func TestCheckMonitors(t *testing.T) {
 					},
 				},
 			},
-			expectErr:   true,
+			expectErr: true,
-			name:        "Monitor failure, bad alert",
+			name:      "Monitor failure, bad alert",
-			selfMonitor: false,
-		},
-		{
-			config: Config{
-				Monitors: []*Monitor{
-					{
-						Name:       "Failure",
-						Command:    CommandOrShell{Command: []string{"false"}},
-						AlertDown:  []string{"bad"},
-						AlertAfter: 1,
-					},
-				},
-				Alerts: map[string]*Alert{
-					"bad": {
-						Name:    "bad",
-						Command: CommandOrShell{Command: []string{"false"}},
-					},
-				},
-			},
-			expectErr:   false,
-			name:        "Monitor failure, bad alert, with Health Check",
-			selfMonitor: true,
 		},
 	}
 
 	for _, c := range cases {
-		SelfMonitor = c.selfMonitor
-
 		err := c.config.Init()
 		if err != nil {
 			t.Errorf("checkMonitors(%s): unexpected error reading config: %v", c.name, err)
@@ -180,3 +134,64 @@ func TestCheckMonitors(t *testing.T) {
 		}
 	}
 }
+
+func TestFirstRunAlerts(t *testing.T) {
+	cases := []struct {
+		config        Config
+		expectErr     bool
+		startupAlerts []string
+		name          string
+	}{
+		{
+			config:        Config{},
+			expectErr:     false,
+			startupAlerts: []string{},
+			name:          "Empty",
+		},
+		{
+			config:        Config{},
+			expectErr:     true,
+			startupAlerts: []string{"missing"},
+			name:          "Unknown",
+		},
+		{
+			config: Config{
+				Alerts: map[string]*Alert{
+					"good": {
+						Command: CommandOrShell{Command: []string{"true"}},
+					},
+				},
+			},
+			expectErr:     false,
+			startupAlerts: []string{"good"},
+			name:          "Successful alert",
+		},
+		{
+			config: Config{
+				Alerts: map[string]*Alert{
+					"bad": {
+						Name:    "bad",
+						Command: CommandOrShell{Command: []string{"false"}},
+					},
+				},
+			},
+			expectErr:     true,
+			startupAlerts: []string{"bad"},
+			name:          "Failed alert",
+		},
+	}
+
+	for _, c := range cases {
+		err := c.config.Init()
+		if err != nil {
+			t.Errorf("sendFirstRunAlerts(%s): unexpected error reading config: %v", c.name, err)
+		}
+
+		err = sendStartupAlerts(&c.config, c.startupAlerts)
+		if err == nil && c.expectErr {
+			t.Errorf("sendFirstRunAlerts(%s): Expected error, the code did not error", c.name)
+		} else if err != nil && !c.expectErr {
+			t.Errorf("sendFirstRunAlerts(%s): Did not expect an error, but we got one anyway: %v", c.name, err)
+		}
+	}
+}

manifest.tmpl

@@ -1,25 +0,0 @@
-image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}
-{{#if build.tags}}
-tags:
-{{#each build.tags}}
-  - {{this}}
-{{/each}}
-{{/if}}
-manifests:
-  -
-    image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64
-    platform:
-      architecture: amd64
-      os: linux
-  -
-    image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64
-    platform:
-      architecture: arm64
-      os: linux
-      variant: v8
-  -
-    image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm
-    platform:
-      architecture: arm
-      os: linux
-      variant: v7

metrics.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"net/http"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -106,7 +107,11 @@ func (metrics *MinitorMetrics) CountAlert(monitor string, alert string) {
 	).Inc()
 }
 
-// HandleMetrics add Prometheus metrics handler to default http server
+// ServeMetrics starts an http server with a Prometheus metrics handler
-func HandleMetrics() {
+func ServeMetrics() {
 	http.Handle("/metrics", promhttp.Handler())
+
+	host := fmt.Sprintf(":%d", MetricsPort)
+
+	_ = http.ListenAndServe(host, nil)
 }

monitor.go

@@ -149,6 +149,7 @@ func (monitor *Monitor) failure() (notice *AlertNotice) {
 	// If we're going to alert, increment count
 	if notice != nil {
 		monitor.alertCount++
+		notice.AlertCount = monitor.alertCount
 	}
 
 	return notice

monitor_test.go

@@ -138,6 +138,47 @@ func TestMonitorSuccess(t *testing.T) {
 	}
 }
 
+func TestMonitorAlertCount(t *testing.T) {
+	var alertEvery int16 = 1
+
+	cases := []struct {
+		checkSuccess bool
+		alertCount   int16
+		name         string
+	}{
+		{false, 1, "First failure and first alert"},
+		{false, 2, "Second failure and first alert"},
+		{true, 2, "Success should preserve past alert count"},
+		{false, 1, "First failure and first alert after success"},
+	}
+
+	// Unlike previous tests, this one requires a static Monitor with repeated
+	// calls to the failure method
+	monitor := Monitor{failureCount: 0, AlertAfter: 1, AlertEvery: &alertEvery}
+
+	for _, c := range cases {
+		log.Printf("Testing case %s", c.name)
+
+		var notice *AlertNotice
+		if c.checkSuccess {
+			notice = monitor.success()
+		} else {
+			notice = monitor.failure()
+		}
+
+		if notice == nil {
+			t.Errorf("failure(%v) expected notice, got nil", c.name)
+		}
+
+		if notice.AlertCount != c.alertCount {
+			t.Errorf("failure(%v), expected=%v actual=%v", c.name, c.alertCount, notice.AlertCount)
+			log.Printf("Case failed: %s", c.name)
+		}
+
+		log.Println("-----")
+	}
+}
+
 // TestMonitorFailureAlertAfter tests that alerts will not trigger until
 // hitting the threshold provided by AlertAfter
 func TestMonitorFailureAlertAfter(t *testing.T) {