Set alert_every to 0 for mdstat_raid monitor to prevent repeated alerts

.drone.yml

@@ -1,100 +1,53 @@
----
 kind: pipeline
-name: test
+type: kubernetes
+name: default
 
-steps:
-  - name: test
-    image: golang:1.21
-    environment:
-      VERSION: ${DRONE_TAG:-${DRONE_COMMIT}}
-    commands:
-      - make test
-
-  - name: check
-    image: iamthefij/drone-pre-commit:personal
-
----
-kind: pipeline
-name: publish
-
-depends_on:
-  - test
+node_selector:
+  zone: dev
 
 trigger:
   event:
     - push
     - tag
-  refs:
-    - refs/heads/master
-    - refs/tags/v*
+
+workspace:
+  path: /drone/src
 
 steps:
-  - name: build all binaries
-    image: golang:1.21
-    environment:
-      VERSION: ${DRONE_TAG:-${DRONE_COMMIT}}
-    commands:
-      - make all
-
-  - name: compress binaries for release
-    image: ubuntu
-    commands:
-      - find ./dist -type f -executable -execdir tar -czvf {}.tar.gz {} \;
-    when:
-      event: tag
-
-  - name: upload gitea release
-    image: plugins/gitea-release
+  - name: pull image to dockerhub
+    image: docker.io/owncloudci/drone-docker-buildx:4
+    privileged: true
     settings:
-      title: ${DRONE_TAG}
-      files: dist/*.tar.gz
-      checksum:
-        - md5
-        - sha1
-        - sha256
-        - sha512
-      base_url:
-        from_secret: gitea_base_url
-      api_key:
-        from_secret: gitea_token
-    when:
-      event: tag
-
-  - name: Build and publish docker images
-    image: thegeeklab/drone-docker-buildx
-    settings:
-      repo: iamthefij/minitor-go
-      auto_tag: true
+      cache-from: [ "safebox/minitor" ]
+      repo: safebox/minitor
+      tags: latest
+      username:
+        from_secret: dockerhub-username
+      password: 
+        from_secret: dockerhub-password
       platforms:
         - linux/amd64
         - linux/arm64
-        - linux/arm
-      username:
-        from_secret: docker_username
-      password:
-        from_secret: docker_password
+    when:  
+      event:
+        - tag
 
----
-kind: pipeline
-name: notify
-
-depends_on:
-  - test
-  - publish
-
-trigger:
-  status:
-    - failure
-
-steps:
-
-  - name: notify
-    image: drillster/drone-email
+  - name: build multiarch from dev
+    image: docker.io/owncloudci/drone-docker-buildx:4
+    privileged: true
     settings:
-      host:
-        from_secret: SMTP_HOST  # pragma: whitelist secret
+      cache-from: [ "registry.dev.format.hu/minitor" ]
+      registry: registry.dev.format.hu
+      repo: registry.dev.format.hu/minitor
+      tags: latest
+      dockerfile: Dockerfile
       username:
-        from_secret: SMTP_USER  # pragma: whitelist secret
+        from_secret: dev-hu-registry-username
       password: 
-        from_secret: SMTP_PASS  # pragma: whitelist secret
-      from: drone@iamthefij.com
+        from_secret: dev-hu-registry-password
+      platforms:
+        - linux/amd64
+        - linux/arm64
+    when:  
+      event:
+        - push

.gitea/workflows/build.yml

@@ -1,113 +0,0 @@
-name: ci
-on:
-  push:
-    branches:
-      - main
-      - master
-    tags:
-      - "v*"
-  pull_request:
-    branches:
-      - main
-      - master
-
-jobs:
-  tests:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Go
-        uses: actions/setup-go@v6
-        with:
-          go-version-file: go.mod
-
-      - name: Run tests
-        run: make test
-
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Go
-        uses: actions/setup-go@v6
-        with:
-          go-version-file: go.mod
-
-      - name: Set up Python
-        uses: actions/setup-python@v6
-
-      - name: Run pre-commit
-        uses: https://git.iamthefij.com/iamthefij/pre-commit-action@v3.0.2
-
-  release:
-    runs-on: ubuntu-latest
-    needs: test
-    if: "${{ github.event_name != 'pull_request' }}"
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Go
-        uses: actions/setup-go@v6
-        with:
-          go-version-file: go.mod
-
-      - name: Build binaries
-        env:
-          VERSION: "${{ vars.REF_NAME }}"
-        run: make all
-
-      # Package binaries and create release if this is a tagged build
-      - name: Compress binaries
-        if: "${{ github.event_name == 'tag' }}"
-        run: find ./dist -type f -executable -execdir tar -czvf {}.tar.gz {} \;
-
-      - name: Upload release
-        uses: https://gitea.com/actions/gitea-release-action@v1
-        if: "${{ github.event_name == 'tag' }}"
-        with:
-          files: |-
-            dist/*.tar.gz
-          md5sum: true
-          sha256sum: true
-
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          # list of Docker images to use as base name for tags
-          images: |
-            ${{ github.REPOSITORY }}
-          # generate Docker tags based on the following events/attributes
-          tags: |
-            type=ref,event=branch
-            type=ref,event=pr
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build and push
-        uses: docker/build-push-action@v6
-        with:
-          # Use path context so we can access pre-compiled binaries
-          context: .
-          push: ${{ github.event_name != 'pull_request' }}
-          platforms: |
-            linux/amd64
-            linux/arm64
-            linux/arm
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}

.gitignore

@@ -14,6 +14,7 @@
 
 # User configuration
 config.yml
+config.hcl
 
 # Output binary
 minitor

Dockerfile

@@ -1,8 +1,117 @@
-FROM alpine:3.23
+FROM golang:1.25 AS gomail-builder
+ARG TARGETARCH=amd64
+ARG TARGETOS=linux
+WORKDIR /gomail
+RUN { \
+    echo 'package main'; \
+    echo ''; \
+    echo 'import ('; \
+    echo '    "fmt"'; \
+    echo '    "io"'; \
+    echo '    "net/mail"'; \
+    echo '    "net/smtp"'; \
+    echo '    "os"'; \
+    echo '    "strings"'; \
+    echo ')'; \
+    echo ''; \
+    echo 'func main() {'; \
+    echo '    readHeaders := strings.Contains(strings.Join(os.Args[1:], " "), "-t")'; \
+    echo '    recipients := []string{}'; \
+    echo '    for _, arg := range os.Args[1:] {'; \
+    echo '        if !strings.HasPrefix(arg, "-") {'; \
+    echo '            recipients = append(recipients, arg)'; \
+    echo '        }'; \
+    echo '    }'; \
+    echo ''; \
+    echo '    body, err := io.ReadAll(os.Stdin)'; \
+    echo '    if err != nil {'; \
+    echo '        fmt.Fprintln(os.Stderr, err)'; \
+    echo '        os.Exit(1)'; \
+    echo '    }'; \
+    echo ''; \
+    echo '    if readHeaders {'; \
+    echo '        msg, parseErr := mail.ReadMessage(strings.NewReader(string(body)))'; \
+    echo '        if parseErr != nil {'; \
+    echo '            fmt.Fprintln(os.Stderr, parseErr)'; \
+    echo '            os.Exit(1)'; \
+    echo '        }'; \
+    echo '        for _, hdr := range []string{"To", "Cc", "Bcc"} {'; \
+    echo '            if val := msg.Header.Get(hdr); val != "" {'; \
+    echo '                addrs, addrErr := mail.ParseAddressList(val)'; \
+    echo '                if addrErr != nil {'; \
+    echo '                    fmt.Fprintln(os.Stderr, addrErr)'; \
+    echo '                    os.Exit(1)'; \
+    echo '                }'; \
+    echo '                for _, addr := range addrs {'; \
+    echo '                    recipients = append(recipients, addr.Address)'; \
+    echo '                }'; \
+    echo '            }'; \
+    echo '        }'; \
+    echo '    }'; \
+    echo ''; \
+    echo '    if len(recipients) == 0 {'; \
+    echo '        fmt.Fprintln(os.Stderr, "usage: sendmail [-t] recipient...")'; \
+    echo '        os.Exit(1)'; \
+    echo '    }'; \
+    echo ''; \
+    echo '    relay := os.Getenv("SMTP_RELAY")'; \
+    echo '    if relay == "" {'; \
+    echo '        relay = "172.17.0.2"'; \
+    echo '    }'; \
+    echo ''; \
+    echo '    port := os.Getenv("SMTP_PORT")'; \
+    echo '    if port == "" {'; \
+    echo '        port = "25"'; \
+    echo '    }'; \
+    echo ''; \
+    echo '    sender := os.Getenv("EMAIL_FROM")'; \
+    echo '    if sender == "" {'; \
+    echo '        fmt.Fprintln(os.Stderr, "[sendmail] EMAIL_FROM is not set, skipping")'; \
+    echo '        os.Exit(0)'; \
+    echo '    }'; \
+    echo ''; \
+    echo '    debug := os.Getenv("DEBUG") != ""'; \
+    echo '    if debug {'; \
+    echo '        fmt.Fprintln(os.Stderr, fmt.Sprintf("[sendmail] relay=%s port=%s sender=%s recipients=%v", relay, port, sender, recipients))'; \
+    echo '        fmt.Fprintln(os.Stderr, "[sendmail] body:")'; \
+    echo '        fmt.Fprintln(os.Stderr, string(body))'; \
+    echo '    }'; \
+    echo ''; \
+    echo '    if err = smtp.SendMail(relay+":"+port, nil, sender, recipients, body); err != nil {'; \
+    echo '        fmt.Fprintln(os.Stderr, err)'; \
+    echo '        os.Exit(1)'; \
+    echo '    }'; \
+    echo '    if debug {'; \
+    echo '        fmt.Fprintln(os.Stderr, "[sendmail] sent successfully")'; \
+    echo '    }'; \
+    echo '}'; \
+    } > main.go
+RUN go mod init gomail && \
+    CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o /usr/local/bin/sendmail .
 
+
+FROM golang:1.25 AS builder
+WORKDIR /app
+COPY ./go.mod ./go.sum /app/
+RUN go mod download
+COPY ./*.go /app/
+RUN rm -f /app/gomail.go
+
+ARG TARGETARCH=amd64
+ARG TARGETOS=linux
+ARG VERSION=dev
+RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags "-X main.version=${VERSION}" -a -installsuffix nocgo -o minitor .
+
+FROM alpine:3.23
 RUN mkdir /app
 WORKDIR /app/
 
+# Copy minitor in
+COPY --from=builder /app/minitor .
+
+# Copy sendmail (gomail) in
+COPY --from=gomail-builder /usr/local/bin/sendmail /usr/local/bin/sendmail
+
 # Add common checking tools
 # hadolint ignore=DL3018
 RUN apk --no-cache add bash=~5 curl=~8 jq=~1 bind-tools=~9 tzdata
@@ -12,13 +121,9 @@ RUN addgroup -S minitor && adduser -S minitor -G minitor
 
 # Copy scripts
 COPY ./scripts /app/scripts
+COPY default_config.hcl /app/config.hcl
 RUN chmod -R 755 /app/scripts
 
-# Copy minitor in
-ARG TARGETOS
-ARG TARGETARCH
-COPY ./dist/minitor-${TARGETOS}-${TARGETARCH} ./minitor
-
 # Drop to non-root user
 USER minitor
 

Dockerfile.multi-stage

@@ -1,39 +0,0 @@
-FROM golang:1.25 AS builder
-
-WORKDIR /app
-
-COPY ./go.mod ./go.sum /app/
-RUN go mod download
-
-COPY ./*.go /app/
-
-ARG TARGETOS
-ARG TARGETARCH
-ARG VERSION=dev
-ENV CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=${TARGETARCH}
-RUN go build -ldflags "-X main.version=${VERSION}" -a -installsuffix nocgo -o minitor .
-
-FROM alpine:3.23
-RUN mkdir /app
-WORKDIR /app/
-
-# Copy minitor in
-COPY --from=builder /app/minitor .
-
-# Add common checking tools
-# hadolint ignore=DL3018
-RUN apk --no-cache add bash=~5 curl=~8 jq=~1 bind-tools=~9 tzdata
-
-# Add minitor user for running as non-root
-RUN addgroup -S minitor && adduser -S minitor -G minitor
-
-# Copy scripts
-COPY ./scripts /app/scripts
-RUN chmod -R 755 /app/scripts
-
-# Drop to non-root user
-USER minitor
-
-ENTRYPOINT [ "./minitor" ]
-
-# vim: set filetype=dockerfile:

README.md

@@ -17,7 +17,7 @@ I'm running a few small services and found Sensu, Consul, Nagios, etc. to all be
 Install and execute with:
 
 ```bash
-go install github.com/iamthefij/minitor-go@latest
+go install github.com/iamthefij/minitor-go/v2@latest
 minitor
 ```
 
@@ -50,15 +50,17 @@ You can configure the timezone for the container by passing a `TZ` env variable.
 
 ## Configuring
 
-In this repo, you can explore the `sample-config.hcl` file for an example, but the general structure is as follows. It should be noted that environment variable interpolation happens on load of the HCL file.
+In this repo, you can explore the `sample-config.hcl` file for an example, but the general structure is as follows. If you are passing environment variables to your commands or alerts, you should be aware that `${VAR}` syntax is reserved for HCL variable interpolation. To avoid issues, you can use `$${VAR}` syntax to escape the `$` character, simply use `$VAR`.
+
+```hcl
 
 The global configurations are:
 
 |key|value|
 |---|---|
 |`check_interval`|Maximum frequency to run checks for each monitor as duration, eg. 1m2s.|
-|`default_alert_after`|A default value used as an `alert_after` value for a monitor if not specified or 0.|
-|`default_alert_every`|A default value used as an `alert_every` value for a monitor if not specified.|
+|`default_alert_after`|A default value used as an `alert_after` value for a monitor if not specified. Defaults 1, which will alert immediately.|
+|`default_alert_every`|A default value used as an `alert_every` value for a monitor if not specified. Defaults to -1, which will re-alert exponentially.|
 |`default_alert_down`|Default down alerts to used by a monitor in case none are provided.|
 |`default_alert_up`|Default up alerts to used by a monitor in case none are provided.|
 |`monitor`|block listing monitors. Detailed description below|
@@ -167,6 +169,48 @@ minitor -metrics
 minitor -metrics -metrics-port 3000
 ```
 
+## Migrating from v1 to v2
+
+Minitor v2 introduces some breaking changes from v1. The most notable changes are:
+  - The configuration file is now in HCL format instead of YAML.
+  - The the Python formatting backwards compatability is removed.
+  - The Command and ShellCommand fields are now mutually exclusive.
+  - The check_interval is now strictly a duration string value. Eg. "30s" rather than `30`.
+  - Default alert_every is now -1 (exponential backoff) rather than 0 (no re-alerting).
+
+For the configuration, a confic that looked like this in v1:
+
+```yaml
+check_interval: 60
+
+monitors:
+  - name: example
+    command: "false"
+    alert_down: ["log"]
+
+alerts:
+  log:
+    command: ["echo", "Minitor up={{.IsUp}} for {{.MonitorName}}"]
+```
+
+Would now look like this in v2:
+
+```hcl
+check_interval = "1m"
+
+monitor "example" {
+  # example showing string to shell command migration
+  shell_command = "false"
+  alert_down = ["log"]
+  check_interval = "1m"
+}
+
+alert "log" {
+  # example showing list to exec command migration
+  command = ["echo", "Minitor up={{.IsUp}} for {{.MonitorName}}"]
+}
+```
+
 ## Contributing
 
 Whether you're looking to submit a patch or tell me I broke something, you can contribute through the Github mirror and I can merge PRs back to the source repository.
@@ -174,11 +218,3 @@ Whether you're looking to submit a patch or tell me I broke something, you can c
 Primary Repo: https://git.iamthefij.com/iamthefij/minitor.git
 
 Github Mirror: https://github.com/IamTheFij/minitor.git
-
-## Original Minitor
-
-This is a reimplementation of [Minitor](https://git.iamthefij.com/iamthefij/minitor) in Go
-
-Minitor is already a minimal monitoring tool. Python 3 was a quick way to get something live, but Python itself comes with a large footprint. Thus Go feels like a better fit for the project, longer term.
-
-Initial target is meant to be roughly compatible requiring only minor changes to configuration. Future iterations may diverge to take advantage of Go specific features.

alert_test.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 	"testing"
 
-	m "git.iamthefij.com/iamthefij/minitor-go"
+	m "git.iamthefij.com/iamthefij/minitor-go/v2"
 )
 
 func TestAlertValidate(t *testing.T) {

config_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 	"time"
 
-	m "git.iamthefij.com/iamthefij/minitor-go"
+	m "git.iamthefij.com/iamthefij/minitor-go/v2"
 )
 
 func TestLoadConfig(t *testing.T) {

default_config.hcl

@@ -0,0 +1,29 @@
+check_interval = "1s"
+
+monitor "mdstat_raid" {
+  command = [
+    "sh",
+    "-c",
+    "grep -q '\\[U_\\|_U\\]' /host_proc/mdstat && exit 1 || exit 0"
+  ]
+  check_interval = "30s"
+  alert_after = 1
+  alert_every = 0
+
+  alert_down = ["email_alert"]
+  alert_up   = ["email_recovery"]
+}
+alert "email_alert" {
+  command = [
+      "sh",
+          "-c",
+            "EMAIL=$EMAIL_RECIPIENT; printf 'Subject: RAID ALERT\nTo: %s\n\nRAID degraded\n' \"$EMAIL\" | /usr/local/bin/sendmail -t || true"
+                ]
+}
+alert "email_recovery" {
+  command = [
+      "sh",
+          "-c",
+              "EMAIL=$EMAIL_RECIPIENT; printf 'Subject: RAID ALERT\nTo: %s\n\nRAID clean\n' \"$EMAIL\" | /usr/local/bin/sendmail -t || true"
+                ]
+}

go.mod

@@ -1,6 +1,6 @@
-module git.iamthefij.com/iamthefij/minitor-go
+module git.iamthefij.com/iamthefij/minitor-go/v2
 
-go 1.23.0
+go 1.25.0
 
 require (
 	git.iamthefij.com/iamthefij/slog v1.3.0

main.go

@@ -119,7 +119,7 @@ func SendStartupAlerts(config *Config, alertNames []string) error {
 
 func main() {
 	showVersion := flag.Bool("version", false, "Display the version of minitor and exit")
-	configPath := flag.String("config", "config.yml", "Alternate configuration path (default: config.yml)")
+	configPath := flag.String("config", "config.hcl", "Alternate configuration path (default: config.hcl)")
 	startupAlerts := flag.String("startup-alerts", "", "List of alerts to run on startup. This can help determine unhealthy alerts early on. (default \"\")")
 
 	flag.BoolVar(&slog.DebugLevel, "debug", false, "Enables debug logs (default: false)")

main_test.go

@@ -3,7 +3,7 @@ package main_test
 import (
 	"testing"
 
-	m "git.iamthefij.com/iamthefij/minitor-go"
+	m "git.iamthefij.com/iamthefij/minitor-go/v2"
 )
 
 func Ptr[T any](v T) *T {

monitor_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	m "git.iamthefij.com/iamthefij/minitor-go"
+	m "git.iamthefij.com/iamthefij/minitor-go/v2"
 )
 
 func TestMonitorValidate(t *testing.T) {

sample-config.hcl

@@ -38,15 +38,15 @@ alert "mailgun_down" {
     -F to=me@minitor.mon \
     -F text="Our monitor failed" \
     https://api.mailgun.net/v3/minitor.mon/messages \
-    -u "api:${MAILGUN_API_KEY}"
+    -u "api:$${MAILGUN_API_KEY}"
   EOF
 }
 
 alert "sms_down" {
   shell_command = <<-EOF
   curl -s -X POST -F "Body=Failure! {{.MonitorName}} has failed" \
-    -F "From=${AVAILABLE_NUMBER}" -F "To=${MY_PHONE}" \
-    "https://api.twilio.com/2010-04-01/Accounts/${ACCOUNT_SID}/Messages" \
-    -u "${ACCOUNT_SID}:${AUTH_TOKEN}"
+    -F "From=$${AVAILABLE_NUMBER}" -F "To=$${MY_PHONE}" \
+    "https://api.twilio.com/2010-04-01/Accounts/$${ACCOUNT_SID}/Messages" \
+    -u "$${ACCOUNT_SID}:$${AUTH_TOKEN}"
   EOF
 }