Adds ability to run specified alerts on startup

This is helpful to determine if your alerts are valid before an actual failure

.drone.yml

@@ -4,7 +4,7 @@ name: test
 
 steps:
   - name: test
-    image: golang:1.17
+    image: golang:1.20
     environment:
       VERSION: ${DRONE_TAG:-${DRONE_COMMIT}}
     commands:
@@ -30,7 +30,7 @@ trigger:
 
 steps:
   - name: build all binaries
-    image: golang:1.17
+    image: golang:1.20
     environment:
       VERSION: ${DRONE_TAG:-${DRONE_COMMIT}}
     commands:

Dockerfile

@@ -1,11 +1,11 @@
 ARG REPO=library
-FROM ${REPO}/alpine:3.12
+FROM ${REPO}/alpine:3.18
 
 RUN mkdir /app
 WORKDIR /app/
 
 # Add common checking tools
-RUN apk --no-cache add bash=~5.0 curl=~7.79 jq=~1.6 bind-tools=~9.16
+RUN apk --no-cache add bash=~5 curl=~8 jq=~1 bind-tools=~9 tzdata~=2023c
 
 # Add minitor user for running as non-root
 RUN addgroup -S minitor && adduser -S minitor -G minitor

Dockerfile.multi-stage

@@ -1,5 +1,5 @@
 ARG REPO=library
-FROM golang:1.17 AS builder
+FROM golang:1.20 AS builder
 
 RUN mkdir /app
 WORKDIR /app
@@ -14,7 +14,7 @@ ARG VERSION=dev
 ENV CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH}
 RUN go build -ldflags "-X main.version=${VERSION}" -a -installsuffix nocgo -o minitor .
 
-FROM ${REPO}/alpine:3.12
+FROM ${REPO}/alpine:3.18
 RUN mkdir /app
 WORKDIR /app/
 
@@ -22,7 +22,7 @@ WORKDIR /app/
 COPY --from=builder /app/minitor .
 
 # Add common checking tools
-RUN apk --no-cache add bash=~5.0 curl=~7.79 jq=~1.6 bind-tools=~9.16
+RUN apk --no-cache add bash=~5 curl=~8 jq=~1 bind-tools=~9 tzdata~=2023c
 
 # Add minitor user for running as non-root
 RUN addgroup -S minitor && adduser -S minitor -G minitor

README.md

@@ -46,6 +46,8 @@ docker run -v $PWD/config.yml:/app/config.yml iamthefij/minitor-go:latest
 
 Images are provided for `amd64`, `arm`, and `arm64` architechtures.
 
+Timezone configuration for the container is set by passing the `TZ` env variable. Eg. `TZ=America/Los_Angeles`.
+
 ## Configuring
 
 In this repo, you can explore the `sample-config.yml` file for an example, but the general structure is as follows. It should be noted that environment variable interpolation happens on load of the YAML file.
@@ -94,10 +96,28 @@ Also, when alerts are executed, they will be passed through Go's format function
 |`{{.AlertCount}}`|Number of times this monitor has alerted|
 |`{{.FailureCount}}`|The total number of sequential failed checks for this monitor|
 |`{{.LastCheckOutput}}`|The last returned value from the check command to either stderr or stdout|
-|`{{.LastSuccess}}`|The ISO datetime of the last successful check|
+|`{{.LastSuccess}}`|The datetime of the last successful check as a go Time struct|
 |`{{.MonitorName}}`|The name of the monitor that failed and triggered the alert|
 |`{{.IsUp}}`|Indicates if the monitor that is alerting is up or not. Can be used in a conditional message template|
 
+To provide flexible formatting, the following non-standard functions are available in templates:
+
+|func|description|
+|---|---|
+|`ANSIC <Time>`|Formats provided time in ANSIC format|
+|`UnixDate <Time>`|Formats provided time in UnixDate format|
+|`RubyDate <Time>`|Formats provided time in RubyDate format|
+|`RFC822Z <Time>`|Formats provided time in RFC822Z format|
+|`RFC850 <Time>`|Formats provided time in RFC850 format|
+|`RFC1123 <Time>`|Formats provided time in RFC1123 format|
+|`RFC1123Z <Time>`|Formats provided time in RFC1123Z format|
+|`RFC3339 <Time>`|Formats provided time in RFC3339 format|
+|`RFC3339Nano <Time>`|Formats provided time in RFC3339Nano format|
+|`FormatTime <Time> <string template>`|Formats provided time according to provided template|
+|`InTZ <Time> <string timezone name>`|Converts provided time to parsed timezone from the provided name|
+
+For more information, check out the [Go documentation for the time module](https://pkg.go.dev/time@go1.20.7#pkg-constants).
+
 ### Metrics
 
 Minitor supports exporting metrics for [Prometheus](https://prometheus.io/). Prometheus is an open source tool for reading and querying metrics from different sources. Combined with another tool, [Grafana](https://grafana.com/), it allows building of charts and dashboards. You could also opt to just use Minitor to log check results, and instead do your alerting with Grafana.

alert.go

@@ -57,6 +57,37 @@ func (alert *Alert) BuildTemplates() error {
 
 	slog.Debugf("Building template for alert %s", alert.Name)
 
+	// Time format func factory
+	tff := func(formatString string) func(time.Time) string {
+		return func(t time.Time) string {
+			return t.Format(formatString)
+		}
+	}
+
+	// Create some functions for formatting datetimes in popular formats
+	timeFormatFuncs := template.FuncMap{
+		"ANSIC":       tff(time.ANSIC),
+		"UnixDate":    tff(time.UnixDate),
+		"RubyDate":    tff(time.RubyDate),
+		"RFC822Z":     tff(time.RFC822Z),
+		"RFC850":      tff(time.RFC850),
+		"RFC1123":     tff(time.RFC1123),
+		"RFC1123Z":    tff(time.RFC1123Z),
+		"RFC3339":     tff(time.RFC3339),
+		"RFC3339Nano": tff(time.RFC3339Nano),
+		"FormatTime": func(t time.Time, timeFormat string) string {
+			return t.Format(timeFormat)
+		},
+		"InTZ": func(t time.Time, tzName string) (time.Time, error) {
+			tz, err := time.LoadLocation(tzName)
+			if err != nil {
+				return t, fmt.Errorf("failed to convert time to specified tz: %w", err)
+			}
+
+			return t.In(tz), nil
+		},
+	}
+
 	switch {
 	case alert.commandTemplate == nil && alert.Command.Command != nil:
 		alert.commandTemplate = []*template.Template{}
@@ -66,7 +97,7 @@ func (alert *Alert) BuildTemplates() error {
 			}
 
 			alert.commandTemplate = append(alert.commandTemplate, template.Must(
-				template.New(alert.Name+fmt.Sprint(i)).Parse(cmdPart),
+				template.New(alert.Name+fmt.Sprint(i)).Funcs(timeFormatFuncs).Parse(cmdPart),
 			))
 		}
 	case alert.commandShellTemplate == nil && alert.Command.ShellCommand != "":
@@ -77,7 +108,7 @@ func (alert *Alert) BuildTemplates() error {
 		}
 
 		alert.commandShellTemplate = template.Must(
-			template.New(alert.Name).Parse(shellCmd),
+			template.New(alert.Name).Funcs(timeFormatFuncs).Parse(shellCmd),
 		)
 	default:
 		return fmt.Errorf("No template provided for alert %s: %w", alert.Name, errNoTemplate)
@@ -137,7 +168,7 @@ func (alert Alert) Send(notice AlertNotice) (outputStr string, err error) {
 
 	if err != nil {
 		err = fmt.Errorf(
-			"Alert '%s' failed to send. Returned %v: %w",
+			"Alert %s failed to send. Returned %w: %w",
 			alert.Name,
 			err,
 			ErrAlertFailed,

go.mod

@@ -1,10 +1,10 @@
 module git.iamthefij.com/iamthefij/minitor-go
 
-go 1.17
+go 1.20
 
 require (
 	git.iamthefij.com/iamthefij/slog v1.3.0
-	github.com/prometheus/client_golang v1.15.0
+	github.com/prometheus/client_golang v1.19.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
@@ -14,10 +14,10 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
-	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/common v0.48.0 // indirect
-	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/rogpeppe/go-internal v1.10.0 // indirect
-	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 )

go.sum

@@ -182,12 +182,16 @@ github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrb
 github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
 github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM=
 github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
+github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
+github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
 github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
+github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
@@ -195,6 +199,8 @@ github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+
 github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
 github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
 github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
+github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
+github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
@@ -203,6 +209,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
 github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
 github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
@@ -365,6 +373,8 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -504,6 +514,10 @@ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
 google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
+google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

main.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"strings"
 	"time"
 
 	"git.iamthefij.com/iamthefij/slog"
@@ -91,9 +92,38 @@ func checkMonitors(config *Config) error {
 	return nil
 }
 
+func sendFirstRunAlerts(config *Config, alertNames []string) error {
+	for _, alertName := range alertNames {
+		var err error
+
+		alert, ok := config.Alerts[alertName]
+		if !ok {
+			err = fmt.Errorf("unknown alert %s: %w", alertName, errUnknownAlert)
+		}
+
+		if err == nil {
+			_, err = alert.Send(AlertNotice{
+				AlertCount:      0,
+				FailureCount:    0,
+				IsUp:            true,
+				LastSuccess:     time.Now(),
+				MonitorName:     fmt.Sprintf("First Run Alert Test: %s", alert.Name),
+				LastCheckOutput: "",
+			})
+		}
+
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func main() {
 	showVersion := flag.Bool("version", false, "Display the version of minitor and exit")
 	configPath := flag.String("config", "config.yml", "Alternate configuration path (default: config.yml)")
+	firstRunAlerts := flag.String("first-run-alerts", "", "List of alerts to run on startup. This can help determine unhealthy alerts early on. (default \"\")")
 
 	flag.BoolVar(&slog.DebugLevel, "debug", false, "Enables debug logs (default: false)")
 	flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)")
@@ -119,6 +149,14 @@ func main() {
 		go ServeMetrics()
 	}
 
+	if *firstRunAlerts != "" {
+		alertNames := strings.Split(*firstRunAlerts, ",")
+
+		err = sendFirstRunAlerts(&config, alertNames)
+
+		slog.OnErrPanicf(err, "Error running first run alerts")
+	}
+
 	// Start main loop
 	for {
 		err = checkMonitors(&config)

main_test.go

@@ -134,3 +134,64 @@ func TestCheckMonitors(t *testing.T) {
 		}
 	}
 }
+
+func TestFirstRunAlerts(t *testing.T) {
+	cases := []struct {
+		config         Config
+		expectErr      bool
+		firstRunAlerts []string
+		name           string
+	}{
+		{
+			config:         Config{},
+			expectErr:      false,
+			firstRunAlerts: []string{},
+			name:           "Empty",
+		},
+		{
+			config:         Config{},
+			expectErr:      true,
+			firstRunAlerts: []string{"missing"},
+			name:           "Unknown",
+		},
+		{
+			config: Config{
+				Alerts: map[string]*Alert{
+					"good": {
+						Command: CommandOrShell{Command: []string{"true"}},
+					},
+				},
+			},
+			expectErr:      false,
+			firstRunAlerts: []string{"good"},
+			name:           "Successful alert",
+		},
+		{
+			config: Config{
+				Alerts: map[string]*Alert{
+					"bad": {
+						Name:    "bad",
+						Command: CommandOrShell{Command: []string{"false"}},
+					},
+				},
+			},
+			expectErr:      true,
+			firstRunAlerts: []string{"bad"},
+			name:           "Failed alert",
+		},
+	}
+
+	for _, c := range cases {
+		err := c.config.Init()
+		if err != nil {
+			t.Errorf("sendFirstRunAlerts(%s): unexpected error reading config: %v", c.name, err)
+		}
+
+		err = sendFirstRunAlerts(&c.config, c.firstRunAlerts)
+		if err == nil && c.expectErr {
+			t.Errorf("sendFirstRunAlerts(%s): Expected error, the code did not error", c.name)
+		} else if err != nil && !c.expectErr {
+			t.Errorf("sendFirstRunAlerts(%s): Did not expect an error, but we got one anyway: %v", c.name, err)
+		}
+	}
+}