uncomment test stuff

.drone.yml

@@ -4,7 +4,7 @@ name: test
 
 steps:
   - name: test
-    image: golang:1.15
+    image: golang:1.12
     environment:
       VERSION: ${DRONE_TAG:-${DRONE_COMMIT}}
     commands:
@@ -30,7 +30,7 @@ trigger:
 
 steps:
   - name: build all binaries
-    image: golang:1.15
+    image: golang:1.12
     environment:
       VERSION: ${DRONE_TAG:-${DRONE_COMMIT}}
     commands:

.gitignore

@@ -17,5 +17,4 @@ config.yml
 
 # Output binary
 minitor
-minitor-go
 dist/

.golangci.yml

@@ -1,48 +0,0 @@
----
-linters:
-  enable:
-    - asciicheck
-    - bodyclose
-    - dogsled
-    - dupl
-    - exhaustive
-    - gochecknoinits
-    - gocognit
-    - gocritic
-    - gocyclo
-    - goerr113
-    - gofumpt
-    - goimports
-    - gomnd
-    - goprintffuncname
-    # - gosec
-    # - ifshort
-    - interfacer
-    - maligned
-    - misspell
-    - nakedret
-    - nestif
-    - nlreturn
-    - noctx
-    - unparam
-    - wsl
-    # - errorlint
-  disable:
-    - gochecknoglobals
-
-linters-settings:
-  gosec:
-    excludes:
-      - G204
-#   gomnd:
-#     settings:
-#       mnd:
-#         ignored-functions: math.*
-
-issues:
-  exclude-rules:
-    - path: _test\.go
-      linters:
-        - errcheck
-        - gosec
-        - maligned

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.4.0
+    rev: v2.4.0
     hooks:
       - id: check-added-large-files
       - id: check-yaml
@@ -11,12 +11,14 @@ repos:
       - id: end-of-file-fixer
       - id: check-merge-conflict
   - repo: git://github.com/dnephin/pre-commit-golang
-    rev: v0.4.0
+    rev: v0.3.5
     hooks:
       - id: go-fmt
       - id: go-imports
-      - id: golangci-lint
+      # - id: gometalinter
-  - repo: https://github.com/hadolint/hadolint
+      # - id: golangci-lint
-    rev: v2.4.0
+  # - repo: https://github.com/IamTheFij/docker-pre-commit
-    hooks:
+  #   rev: v2.0.0
-      - id: hadolint
+  #   hooks:
+  #     - id: docker-compose-check
+  #     - id: hadolint

Dockerfile

@@ -1,11 +1,15 @@
 ARG REPO=library
-FROM ${REPO}/alpine:3.12
+FROM multiarch/qemu-user-static:4.2.0-2 as qemu-user-static
+FROM ${REPO}/alpine:3.10
+
+# Copying all qemu files because amd64 doesn't exist and cannot condional copy
+COPY --from=qemu-user-static /usr/bin/qemu-* /usr/bin/
 
 RUN mkdir /app
 WORKDIR /app/
 
 # Add common checking tools
-RUN apk --no-cache add bash=~5.0 curl=~7.76 jq=~1.6
+RUN apk --no-cache add bash=~5.0 curl=~7.66 jq=~1.6
 
 # Add minitor user for running as non-root
 RUN addgroup -S minitor && adduser -S minitor -G minitor

README.md

@@ -54,7 +54,7 @@ The global configurations are:
 
 |key|value|
 |---|---|
-|`check_interval`|Maximum frequency to run checks for each monitor as duration, eg. 1m2s.|
+|`check_interval`|Maximum frequency to run checks for each monitor|
 |`monitors`|List of all monitors. Detailed description below|
 |`alerts`|List of all alerts. Detailed description below|
 
@@ -111,7 +111,7 @@ minitor -metrics -metrics-port 3000
 
 ## Contributing
 
-Whether you're looking to submit a patch or tell me I broke something, you can contribute through the Github mirror and I can merge PRs back to the source repository.
+Whether you're looking to submit a patch or just tell me I broke something, you can contribute through the Github mirror and I can merge PRs back to the source repository.
 
 Primary Repo: https://git.iamthefij.com/iamthefij/minitor.git
 
@@ -143,25 +143,15 @@ alerts:
     command: 'echo {{.MonitorName}}'
 ```
 
-Interval durations have changed from being an integer number of seconds to a duration string supported by Go, for example:
-
-minitor-py:
-```yaml
-check_interval: 90
-```
-
-minitor-go:
-```yaml
-check_interval: 1m30s
-```
-
 For the time being, legacy configs for the Python version of Minitor should be compatible if you apply the `-py-compat` flag when running Minitor. Eventually, this flag will go away when later breaking changes are introduced.
 
 ## Future
 
 Future, potentially breaking changes
 
+  - [ ] Implement leveled logging (maybe glog or logrus)
   - [ ] Consider value of templating vs injecting values into Env variables
   - [ ] Async checking
   - [ ] Revisit metrics and see if they all make sense
   - [ ] Consider dropping `alert_up` and `alert_down` in favor of using Go templates that offer more control of messaging (Breaking)
+  - [ ] Use durations rather than seconds checked in event loop (Potentially breaking)

alert.go

@@ -2,21 +2,12 @@ package main
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
+	"log"
 	"os/exec"
 	"strings"
 	"text/template"
 	"time"
-
-	"git.iamthefij.com/iamthefij/slog"
-)
-
-var (
-	errNoTemplate = errors.New("no template")
-
-	// ErrAlertFailed indicates that an alert failed to send
-	ErrAlertFailed = errors.New("alert failed")
 )
 
 // Alert is a config driven mechanism for sending a notice
@@ -29,12 +20,12 @@ type Alert struct {
 
 // AlertNotice captures the context for an alert to be sent
 type AlertNotice struct {
+	MonitorName     string
 	AlertCount      int16
 	FailureCount    int16
-	IsUp            bool
-	LastSuccess     time.Time
-	MonitorName     string
 	LastCheckOutput string
+	LastSuccess     time.Time
+	IsUp            bool
 }
 
 // IsValid returns a boolean indicating if the Alert has been correctly
@@ -54,33 +45,29 @@ func (alert *Alert) BuildTemplates() error {
 		"{last_success}", "{{.LastSuccess}}",
 		"{monitor_name}", "{{.MonitorName}}",
 	)
-
+	if LogDebug {
-	slog.Debugf("Building template for alert %s", alert.Name)
+		log.Printf("DEBUG: Building template for alert %s", alert.Name)
-
+	}
-	switch {
+	if alert.commandTemplate == nil && alert.Command.Command != nil {
-	case alert.commandTemplate == nil && alert.Command.Command != nil:
 		alert.commandTemplate = []*template.Template{}
 		for i, cmdPart := range alert.Command.Command {
 			if PyCompat {
 				cmdPart = legacy.Replace(cmdPart)
 			}
-
 			alert.commandTemplate = append(alert.commandTemplate, template.Must(
 				template.New(alert.Name+fmt.Sprint(i)).Parse(cmdPart),
 			))
 		}
-	case alert.commandShellTemplate == nil && alert.Command.ShellCommand != "":
+	} else if alert.commandShellTemplate == nil && alert.Command.ShellCommand != "" {
 		shellCmd := alert.Command.ShellCommand
-
 		if PyCompat {
 			shellCmd = legacy.Replace(shellCmd)
 		}
-
 		alert.commandShellTemplate = template.Must(
 			template.New(alert.Name).Parse(shellCmd),
 		)
-	default:
+	} else {
-		return fmt.Errorf("No template provided for alert %s: %w", alert.Name, errNoTemplate)
+		return fmt.Errorf("No template provided for alert %s", alert.Name)
 	}
 
 	return nil
@@ -88,40 +75,30 @@ func (alert *Alert) BuildTemplates() error {
 
 // Send will send an alert notice by executing the command template
 func (alert Alert) Send(notice AlertNotice) (outputStr string, err error) {
-	slog.Infof("Sending alert %s for %s", alert.Name, notice.MonitorName)
+	log.Printf("INFO: Sending alert %s for %s", alert.Name, notice.MonitorName)
-
 	var cmd *exec.Cmd
-
+	if alert.commandTemplate != nil {
-	switch {
-	case alert.commandTemplate != nil:
 		command := []string{}
-
 		for _, cmdTmp := range alert.commandTemplate {
 			var commandBuffer bytes.Buffer
-
 			err = cmdTmp.Execute(&commandBuffer, notice)
 			if err != nil {
 				return
 			}
-
 			command = append(command, commandBuffer.String())
 		}
-
 		cmd = exec.Command(command[0], command[1:]...)
-	case alert.commandShellTemplate != nil:
+	} else if alert.commandShellTemplate != nil {
 		var commandBuffer bytes.Buffer
-
 		err = alert.commandShellTemplate.Execute(&commandBuffer, notice)
 		if err != nil {
 			return
 		}
-
 		shellCommand := commandBuffer.String()
 
 		cmd = ShellCommand(shellCommand)
-	default:
+	} else {
-		err = fmt.Errorf("No templates compiled for alert %s: %w", alert.Name, errNoTemplate)
+		err = fmt.Errorf("No templates compiled for alert %v", alert.Name)
-
 		return
 	}
 
@@ -133,15 +110,8 @@ func (alert Alert) Send(notice AlertNotice) (outputStr string, err error) {
 	var output []byte
 	output, err = cmd.CombinedOutput()
 	outputStr = string(output)
-	slog.Debugf("Alert output for: %s\n---\n%s\n---", alert.Name, outputStr)
+	if LogDebug {
-
+		log.Printf("DEBUG: Alert output for: %s\n---\n%s\n---", alert.Name, outputStr)
-	if err != nil {
-		err = fmt.Errorf(
-			"Alert '%s' failed to send. Returned %v: %w",
-			alert.Name,
-			err,
-			ErrAlertFailed,
-		)
 	}
 
 	return outputStr, err

alert_test.go

@@ -18,13 +18,11 @@ func TestAlertIsValid(t *testing.T) {
 
 	for _, c := range cases {
 		log.Printf("Testing case %s", c.name)
-
 		actual := c.alert.IsValid()
 		if actual != c.expected {
 			t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expected, actual)
 			log.Printf("Case failed: %s", c.name)
 		}
-
 		log.Println("-----")
 	}
 }
@@ -102,28 +100,19 @@ func TestAlertSend(t *testing.T) {
 		log.Printf("Testing case %s", c.name)
 		// Set PyCompat to value of compat flag
 		PyCompat = c.pyCompat
-
+		c.alert.BuildTemplates()
-		err := c.alert.BuildTemplates()
-		if err != nil {
-			t.Errorf("Send(%v output), error building templates: %v", c.name, err)
-		}
-
 		output, err := c.alert.Send(c.notice)
 		hasErr := (err != nil)
-
 		if output != c.expectedOutput {
 			t.Errorf("Send(%v output), expected=%v actual=%v", c.name, c.expectedOutput, output)
 			log.Printf("Case failed: %s", c.name)
 		}
-
 		if hasErr != c.expectErr {
 			t.Errorf("Send(%v err), expected=%v actual=%v", c.name, "Err", err)
 			log.Printf("Case failed: %s", c.name)
 		}
-
 		// Set PyCompat back to default value
 		PyCompat = false
-
 		log.Println("-----")
 	}
 }
@@ -131,12 +120,10 @@ func TestAlertSend(t *testing.T) {
 func TestAlertSendNoTemplates(t *testing.T) {
 	alert := Alert{}
 	notice := AlertNotice{}
-
 	output, err := alert.Send(notice)
 	if err == nil {
 		t.Errorf("Send(no template), expected=%v actual=%v", "Err", output)
 	}
-
 	log.Println("-----")
 }
 
@@ -155,12 +142,10 @@ func TestAlertBuildTemplate(t *testing.T) {
 		log.Printf("Testing case %s", c.name)
 		err := c.alert.BuildTemplates()
 		hasErr := (err != nil)
-
 		if hasErr != c.expectErr {
 			t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expectErr, err)
 			log.Printf("Case failed: %s", c.name)
 		}
-
 		log.Println("-----")
 	}
 }

config.go

@@ -3,17 +3,14 @@ package main
 import (
 	"errors"
 	"io/ioutil"
-	"time"
+	"log"
 
-	"git.iamthefij.com/iamthefij/slog"
 	"gopkg.in/yaml.v2"
 )
 
-var errInvalidConfig = errors.New("Invalid configuration")
-
 // Config type is contains all provided user configuration
 type Config struct {
-	CheckInterval SecondsOrDuration `yaml:"check_interval"`
+	CheckInterval int64 `yaml:"check_interval"`
 	Monitors      []*Monitor
 	Alerts        map[string]*Alert
 }
@@ -38,48 +35,17 @@ func (cos *CommandOrShell) UnmarshalYAML(unmarshal func(interface{}) error) erro
 	// Error indicates this is shell command
 	if err != nil {
 		var shellCmd string
-
 		err := unmarshal(&shellCmd)
 		if err != nil {
 			return err
 		}
-
 		cos.ShellCommand = shellCmd
 	} else {
 		cos.Command = cmd
 	}
-
 	return nil
 }
 
-// SecondsOrDuration wraps a duration value for parsing a duration or seconds from YAML
-// NOTE: This should be removed in favor of only parsing durations once compatibility is broken
-type SecondsOrDuration struct {
-	value time.Duration
-}
-
-// Value returns a duration value
-func (sod SecondsOrDuration) Value() time.Duration {
-	return sod.value
-}
-
-// UnmarshalYAML allows unmarshalling a duration value or seconds if an int was provided
-func (sod *SecondsOrDuration) UnmarshalYAML(unmarshal func(interface{}) error) error {
-	var seconds int64
-	err := unmarshal(&seconds)
-
-	if err == nil {
-		sod.value = time.Second * time.Duration(seconds)
-
-		return nil
-	}
-
-	// Error indicates that we don't have an int
-	err = unmarshal(&sod.value)
-
-	return err
-}
-
 // IsValid checks config validity and returns true if valid
 func (config Config) IsValid() (isValid bool) {
 	isValid = true
@@ -87,50 +53,41 @@ func (config Config) IsValid() (isValid bool) {
 	// Validate alerts
 	if config.Alerts == nil || len(config.Alerts) == 0 {
 		// This should never happen because there is a default alert named 'log' for now
-		slog.Errorf("Invalid alert configuration: Must provide at least one alert")
+		log.Printf("ERROR: Invalid alert configuration: Must provide at least one alert")
-
 		isValid = false
 	}
-
 	for _, alert := range config.Alerts {
 		if !alert.IsValid() {
-			slog.Errorf("Invalid alert configuration: %+v", alert.Name)
+			log.Printf("ERROR: Invalid alert configuration: %s", alert.Name)
-
 			isValid = false
-		} else {
-			slog.Debugf("Loaded alert %s", alert.Name)
 		}
 	}
 
 	// Validate monitors
 	if config.Monitors == nil || len(config.Monitors) == 0 {
-		slog.Errorf("Invalid monitor configuration: Must provide at least one monitor")
+		log.Printf("ERROR: Invalid monitor configuration: Must provide at least one monitor")
-
 		isValid = false
 	}
-
 	for _, monitor := range config.Monitors {
 		if !monitor.IsValid() {
-			slog.Errorf("Invalid monitor configuration: %s", monitor.Name)
+			log.Printf("ERROR: Invalid monitor configuration: %s", monitor.Name)
-
 			isValid = false
 		}
 		// Check that all Monitor alerts actually exist
 		for _, isUp := range []bool{true, false} {
 			for _, alertName := range monitor.GetAlertNames(isUp) {
 				if _, ok := config.Alerts[alertName]; !ok {
-					slog.Errorf(
+					log.Printf(
-						"Invalid monitor configuration: %s. Unknown alert %s",
+						"ERROR: Invalid monitor configuration: %s. Unknown alert %s",
 						monitor.Name, alertName,
 					)
-
 					isValid = false
 				}
 			}
 		}
 	}
 
-	return isValid
+	return
 }
 
 // Init performs extra initialization on top of loading the config from file
@@ -157,30 +114,28 @@ func LoadConfig(filePath string) (config Config, err error) {
 		return
 	}
 
-	slog.Debugf("Config values:\n%v\n", config)
+	if LogDebug {
+		log.Printf("DEBUG: Config values:\n%v\n", config)
+	}
 
 	// Add log alert if not present
 	if PyCompat {
-		// Initialize alerts list if not present
+		// Intialize alerts list if not present
 		if config.Alerts == nil {
 			config.Alerts = map[string]*Alert{}
 		}
-
 		if _, ok := config.Alerts["log"]; !ok {
 			config.Alerts["log"] = NewLogAlert()
 		}
 	}
 
-	// Finish initializing configuration
-	if err = config.Init(); err != nil {
-		return
-	}
-
 	if !config.IsValid() {
-		err = errInvalidConfig
+		err = errors.New("Invalid configuration")
-
 		return
 	}
 
-	return config, err
+	// Finish initializing configuration
+	err = config.Init()
+
+	return
 }

config_test.go

@@ -3,7 +3,6 @@ package main
 import (
 	"log"
 	"testing"
-	"time"
 )
 
 func TestLoadConfig(t *testing.T) {
@@ -28,50 +27,20 @@ func TestLoadConfig(t *testing.T) {
 		PyCompat = c.pyCompat
 		_, err := LoadConfig(c.configPath)
 		hasErr := (err != nil)
-
 		if hasErr != c.expectErr {
 			t.Errorf("LoadConfig(%v), expected_error=%v actual=%v", c.name, c.expectErr, err)
 			log.Printf("Case failed: %s", c.name)
 		}
-
 		// Set PyCompat to default value
 		PyCompat = false
+		log.Println("-----")
 	}
 }
 
-func TestIntervalParsing(t *testing.T) {
-	log.Printf("Testing case TestIntervalParsing")
-
-	config, err := LoadConfig("./test/valid-config.yml")
-	if err != nil {
-		t.Errorf("Failed loading config: %v", err)
-	}
-
-	oneSecond := time.Second
-	tenSeconds := 10 * time.Second
-	oneMinute := time.Minute
-
-	// validate top level interval seconds represented as an int
-	if config.CheckInterval.Value() != oneSecond {
-		t.Errorf("Incorrectly parsed int seconds. expected=%v actual=%v", oneSecond, config.CheckInterval)
-	}
-
-	if config.Monitors[0].CheckInterval.Value() != tenSeconds {
-		t.Errorf("Incorrectly parsed seconds duration. expected=%v actual=%v", oneSecond, config.CheckInterval)
-	}
-
-	if config.Monitors[1].CheckInterval.Value() != oneMinute {
-		t.Errorf("Incorrectly parsed seconds duration. expected=%v actual=%v", oneSecond, config.CheckInterval)
-	}
-
-	log.Println("-----")
-}
-
 // TestMultiLineConfig is a more complicated test stepping through the parsing
 // and execution of mutli-line strings presented in YAML
 func TestMultiLineConfig(t *testing.T) {
 	log.Println("Testing multi-line string config")
-
 	config, err := LoadConfig("./test/valid-verify-multi-line.yml")
 	if err != nil {
 		t.Fatalf("TestMultiLineConfig(load), expected=no_error actual=%v", err)
@@ -79,10 +48,8 @@ func TestMultiLineConfig(t *testing.T) {
 
 	log.Println("-----")
 	log.Println("TestMultiLineConfig(parse > string)")
-
 	expected := "echo 'Some string with stuff'; echo \"<angle brackets>\"; exit 1\n"
 	actual := config.Monitors[0].Command.ShellCommand
-
 	if expected != actual {
 		t.Errorf("TestMultiLineConfig(>) failed")
 		t.Logf("string expected=`%v`", expected)
@@ -93,15 +60,12 @@ func TestMultiLineConfig(t *testing.T) {
 
 	log.Println("-----")
 	log.Println("TestMultiLineConfig(execute > string)")
-
 	_, notice := config.Monitors[0].Check()
 	if notice == nil {
 		t.Fatalf("Did not receive an alert notice")
 	}
-
 	expected = "Some string with stuff\n<angle brackets>\n"
 	actual = notice.LastCheckOutput
-
 	if expected != actual {
 		t.Errorf("TestMultiLineConfig(execute > string) check failed")
 		t.Logf("string expected=`%v`", expected)
@@ -112,10 +76,8 @@ func TestMultiLineConfig(t *testing.T) {
 
 	log.Println("-----")
 	log.Println("TestMultiLineConfig(parse | string)")
-
 	expected = "echo 'Some string with stuff'\necho '<angle brackets>'\n"
 	actual = config.Alerts["log_shell"].Command.ShellCommand
-
 	if expected != actual {
 		t.Errorf("TestMultiLineConfig(|) failed")
 		t.Logf("string expected=`%v`", expected)
@@ -126,12 +88,10 @@ func TestMultiLineConfig(t *testing.T) {
 
 	log.Println("-----")
 	log.Println("TestMultiLineConfig(execute | string)")
-
 	actual, err = config.Alerts["log_shell"].Send(AlertNotice{})
 	if err != nil {
 		t.Errorf("Execution of alert failed")
 	}
-
 	expected = "Some string with stuff\n<angle brackets>\n"
 	if expected != actual {
 		t.Errorf("TestMultiLineConfig(execute | string) check failed")

go.mod

@@ -1,9 +1,8 @@
 module git.iamthefij.com/iamthefij/minitor-go
 
-go 1.15
+go 1.12
 
 require (
-	git.iamthefij.com/iamthefij/slog v1.3.0
 	github.com/prometheus/client_golang v1.2.1
 	gopkg.in/yaml.v2 v2.2.4
 )

go.sum

@@ -1,5 +1,3 @@
-git.iamthefij.com/iamthefij/slog v1.3.0 h1:4Hu5PQvDrW5e3FrTS3q2iIXW0iPvhNY/9qJsqDR3K3I=
-git.iamthefij.com/iamthefij/slog v1.3.0/go.mod h1:1RUj4hcCompZkAxXCRfUX786tb3cM/Zpkn97dGfUfbg=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=

main.go

@@ -1,15 +1,16 @@
 package main
 
 import (
-	"errors"
 	"flag"
 	"fmt"
+	"log"
 	"time"
-
-	"git.iamthefij.com/iamthefij/slog"
 )
 
 var (
+	// LogDebug will control whether debug messsages should be logged
+	LogDebug = false
+
 	// ExportMetrics will track whether or not we want to export metrics to prometheus
 	ExportMetrics = false
 	// MetricsPort is the port to expose metrics on
@@ -22,49 +23,8 @@ var (
 
 	// version of minitor being run
 	version = "dev"
-
-	errUnknownAlert = errors.New("unknown alert")
 )
 
-func sendAlerts(config *Config, monitor *Monitor, alertNotice *AlertNotice) error {
-	slog.Debugf("Received an alert notice from %s", alertNotice.MonitorName)
-	alertNames := monitor.GetAlertNames(alertNotice.IsUp)
-
-	if alertNames == nil {
-		// This should only happen for a recovery alert. AlertDown is validated not empty
-		slog.Warningf(
-			"Received alert, but no alert mechanisms exist. MonitorName=%s IsUp=%t",
-			alertNotice.MonitorName, alertNotice.IsUp,
-		)
-	}
-
-	for _, alertName := range alertNames {
-		if alert, ok := config.Alerts[alertName]; ok {
-			output, err := alert.Send(*alertNotice)
-			if err != nil {
-				slog.Errorf(
-					"Alert '%s' failed. result=%v: output=%s",
-					alert.Name,
-					err,
-					output,
-				)
-
-				return err
-			}
-
-			// Count alert metrics
-			Metrics.CountAlert(monitor.Name, alert.Name)
-		} else {
-			// This case should never actually happen since we validate against it
-			slog.Errorf("Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
-
-			return fmt.Errorf("unknown alert for monitor %s: %s: %w", alertNotice.MonitorName, alertName, errUnknownAlert)
-		}
-	}
-
-	return nil
-}
-
 func checkMonitors(config *Config) error {
 	for _, monitor := range config.Monitors {
 		if monitor.ShouldCheck() {
@@ -74,10 +34,47 @@ func checkMonitors(config *Config) error {
 
 			// Track status metrics
 			Metrics.SetMonitorStatus(monitor.Name, monitor.IsUp())
-			Metrics.CountCheck(monitor.Name, success, monitor.LastCheckMilliseconds(), hasAlert)
+			Metrics.CountCheck(monitor.Name, success, hasAlert)
 
+			// Should probably consider refactoring everything below here
 			if alertNotice != nil {
-				return sendAlerts(config, monitor, alertNotice)
+				if LogDebug {
+					log.Printf("DEBUG: Recieved an alert notice from %s", alertNotice.MonitorName)
+				}
+				alertNames := monitor.GetAlertNames(alertNotice.IsUp)
+				if alertNames == nil {
+					// This should only happen for a recovery alert. AlertDown is validated not empty
+					log.Printf(
+						"WARNING: Recieved alert, but no alert mechanisms exist. MonitorName=%s IsUp=%t",
+						alertNotice.MonitorName, alertNotice.IsUp,
+					)
+				}
+				for _, alertName := range alertNames {
+					if alert, ok := config.Alerts[alertName]; ok {
+						output, err := alert.Send(*alertNotice)
+						if err != nil {
+							log.Printf(
+								"ERROR: Alert '%s' failed. result=%v: output=%s",
+								alert.Name,
+								err,
+								output,
+							)
+							return fmt.Errorf(
+								"Unsuccessfully triggered alert '%s'. "+
+									"Crashing to avoid false negatives: %v",
+								alert.Name,
+								err,
+							)
+						}
+
+						// Count alert metrics
+						Metrics.CountAlert(monitor.Name, alert.Name)
+					} else {
+						// This case should never actually happen since we validate against it
+						log.Printf("ERROR: Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
+						return fmt.Errorf("Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
+					}
+				}
 			}
 		}
 	}
@@ -86,30 +83,30 @@ func checkMonitors(config *Config) error {
 }
 
 func main() {
-	showVersion := flag.Bool("version", false, "Display the version of minitor and exit")
+	// Get debug flag
-	configPath := flag.String("config", "config.yml", "Alternate configuration path (default: config.yml)")
+	flag.BoolVar(&LogDebug, "debug", false, "Enables debug logs (default: false)")
-
-	flag.BoolVar(&slog.DebugLevel, "debug", false, "Enables debug logs (default: false)")
 	flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)")
 	flag.BoolVar(&PyCompat, "py-compat", false, "Enables support for legacy Python Minitor config. Will eventually be removed. (default: false)")
 	flag.IntVar(&MetricsPort, "metrics-port", 8080, "The port that Prometheus metrics should be exported on, if enabled. (default: 8080)")
+	var showVersion = flag.Bool("version", false, "Display the version of minitor and exit")
+	var configPath = flag.String("config", "config.yml", "Alternate configuration path (default: config.yml)")
 	flag.Parse()
 
 	// Print version if flag is provided
 	if *showVersion {
-		fmt.Println("Minitor version:", version)
+		log.Println("Minitor version:", version)
-
 		return
 	}
 
 	// Load configuration
 	config, err := LoadConfig(*configPath)
-	slog.OnErrFatalf(err, "Error loading config: %v", err)
+	if err != nil {
+		log.Fatalf("Error loading config: %v", err)
+	}
 
 	// Serve metrics exporter, if specified
 	if ExportMetrics {
-		slog.Infof("Exporting metrics to Prometheus on port %d", MetricsPort)
+		log.Println("INFO: Exporting metrics to Prometheus")
-
 		go ServeMetrics()
 	}
 
@@ -120,6 +117,7 @@ func main() {
 			panic(err)
 		}
 
-		time.Sleep(config.CheckInterval.Value())
+		sleepTime := time.Duration(config.CheckInterval) * time.Second
+		time.Sleep(sleepTime)
 	}
 }

main_test.go

@@ -33,10 +33,16 @@ func TestCheckMonitors(t *testing.T) {
 						Command:    CommandOrShell{Command: []string{"false"}},
 						AlertAfter: 1,
 					},
+					&Monitor{
+						Name:       "Failure",
+						Command:    CommandOrShell{Command: []string{"false"}},
+						AlertDown:  []string{"unknown"},
+						AlertAfter: 1,
+					},
 				},
 			},
 			expectErr: false,
-			name:      "Monitor failure, no alerts",
+			name:      "Monitor failure, no and unknown alerts",
 		},
 		{
 			config: Config{
@@ -46,28 +52,6 @@ func TestCheckMonitors(t *testing.T) {
 						Command:    CommandOrShell{Command: []string{"ls"}},
 						alertCount: 1,
 					},
-				},
-			},
-			expectErr: false,
-			name:      "Monitor recovery, no alerts",
-		},
-		{
-			config: Config{
-				Monitors: []*Monitor{
-					&Monitor{
-						Name:       "Failure",
-						Command:    CommandOrShell{Command: []string{"false"}},
-						AlertDown:  []string{"unknown"},
-						AlertAfter: 1,
-					},
-				},
-			},
-			expectErr: true,
-			name:      "Monitor failure, unknown alerts",
-		},
-		{
-			config: Config{
-				Monitors: []*Monitor{
 					&Monitor{
 						Name:       "Success",
 						Command:    CommandOrShell{Command: []string{"true"}},
@@ -76,8 +60,8 @@ func TestCheckMonitors(t *testing.T) {
 					},
 				},
 			},
-			expectErr: true,
+			expectErr: false,
-			name:      "Monitor recovery, unknown alerts",
+			name:      "Monitor recovery, no alerts",
 		},
 		{
 			config: Config{
@@ -121,16 +105,10 @@ func TestCheckMonitors(t *testing.T) {
 	}
 
 	for _, c := range cases {
-		err := c.config.Init()
+		c.config.Init()
-		if err != nil {
+		err := checkMonitors(&c.config)
-			t.Errorf("checkMonitors(%s): unexpected error reading config: %v", c.name, err)
-		}
-
-		err = checkMonitors(&c.config)
 		if err == nil && c.expectErr {
 			t.Errorf("checkMonitors(%s): Expected panic, the code did not panic", c.name)
-		} else if err != nil && !c.expectErr {
-			t.Errorf("checkMonitors(%s): Did not expect an error, but we got one anyway: %v", c.name, err)
 		}
 	}
 }

metrics.go

@@ -19,7 +19,6 @@ import (
 type MinitorMetrics struct {
 	alertCount    *prometheus.CounterVec
 	checkCount    *prometheus.CounterVec
-	checkTime     *prometheus.GaugeVec
 	monitorStatus *prometheus.GaugeVec
 }
 
@@ -41,13 +40,6 @@ func NewMetrics() *MinitorMetrics {
 			},
 			[]string{"monitor", "status", "is_alert"},
 		),
-		checkTime: prometheus.NewGaugeVec(
-			prometheus.GaugeOpts{
-				Name: "minitor_check_milliseconds",
-				Help: "Time in miliseconds that a check ran for",
-			},
-			[]string{"monitor", "status"},
-		),
 		monitorStatus: prometheus.NewGaugeVec(
 			prometheus.GaugeOpts{
 				Name: "minitor_monitor_up_count",
@@ -60,7 +52,6 @@ func NewMetrics() *MinitorMetrics {
 	// Register newly created metrics
 	prometheus.MustRegister(metrics.alertCount)
 	prometheus.MustRegister(metrics.checkCount)
-	prometheus.MustRegister(metrics.checkTime)
 	prometheus.MustRegister(metrics.monitorStatus)
 
 	return metrics
@@ -72,12 +63,11 @@ func (metrics *MinitorMetrics) SetMonitorStatus(monitor string, isUp bool) {
 	if isUp {
 		val = 1.0
 	}
-
 	metrics.monitorStatus.With(prometheus.Labels{"monitor": monitor}).Set(val)
 }
 
 // CountCheck counts the result of a particular Monitor check
-func (metrics *MinitorMetrics) CountCheck(monitor string, isSuccess bool, ms int64, isAlert bool) {
+func (metrics *MinitorMetrics) CountCheck(monitor string, isSuccess bool, isAlert bool) {
 	status := "failure"
 	if isSuccess {
 		status = "success"
@@ -91,10 +81,6 @@ func (metrics *MinitorMetrics) CountCheck(monitor string, isSuccess bool, ms int
 	metrics.checkCount.With(
 		prometheus.Labels{"monitor": monitor, "status": status, "is_alert": alertVal},
 	).Inc()
-
-	metrics.checkTime.With(
-		prometheus.Labels{"monitor": monitor, "status": status},
-	).Set(float64(ms))
 }
 
 // CountAlert counts an alert
@@ -110,8 +96,6 @@ func (metrics *MinitorMetrics) CountAlert(monitor string, alert string) {
 // ServeMetrics starts an http server with a Prometheus metrics handler
 func ServeMetrics() {
 	http.Handle("/metrics", promhttp.Handler())
-
 	host := fmt.Sprintf(":%d", MetricsPort)
-
 	_ = http.ListenAndServe(host, nil)
 }

monitor.go

@@ -1,31 +1,28 @@
 package main
 
 import (
+	"log"
 	"math"
 	"os/exec"
 	"time"
-
-	"git.iamthefij.com/iamthefij/slog"
 )
 
 // Monitor represents a particular periodic check of a command
-type Monitor struct { //nolint:maligned
+type Monitor struct {
 	// Config values
-	AlertAfter    int16             `yaml:"alert_after"`
-	AlertEvery    int16             `yaml:"alert_every"`
-	CheckInterval SecondsOrDuration `yaml:"check_interval"`
 	Name          string
+	Command       CommandOrShell
 	AlertDown     []string `yaml:"alert_down"`
 	AlertUp       []string `yaml:"alert_up"`
-	Command       CommandOrShell
+	CheckInterval float64  `yaml:"check_interval"`
-
+	AlertAfter    int16    `yaml:"alert_after"`
+	AlertEvery    int16    `yaml:"alert_every"`
 	// Other values
-	alertCount        int16
+	lastCheck    time.Time
-	failureCount      int16
+	lastOutput   string
-	lastCheck         time.Time
+	alertCount   int16
-	lastSuccess       time.Time
+	failureCount int16
-	lastOutput        string
+	lastSuccess  time.Time
-	lastCheckDuration time.Duration
 }
 
 // IsValid returns a boolean indicating if the Monitor has been correctly
@@ -43,9 +40,8 @@ func (monitor Monitor) ShouldCheck() bool {
 		return true
 	}
 
-	sinceLastCheck := time.Since(monitor.lastCheck)
+	sinceLastCheck := time.Now().Sub(monitor.lastCheck).Seconds()
-
+	return sinceLastCheck >= monitor.CheckInterval
-	return sinceLastCheck >= monitor.CheckInterval.Value()
 }
 
 // Check will run the command configured by the Monitor and return a status
@@ -58,14 +54,11 @@ func (monitor *Monitor) Check() (bool, *AlertNotice) {
 		cmd = ShellCommand(monitor.Command.ShellCommand)
 	}
 
-	checkStartTime := time.Now()
 	output, err := cmd.CombinedOutput()
 	monitor.lastCheck = time.Now()
 	monitor.lastOutput = string(output)
-	monitor.lastCheckDuration = monitor.lastCheck.Sub(checkStartTime)
 
 	var alertNotice *AlertNotice
-
 	isSuccess := (err == nil)
 	if isSuccess {
 		alertNotice = monitor.success()
@@ -73,11 +66,17 @@ func (monitor *Monitor) Check() (bool, *AlertNotice) {
 		alertNotice = monitor.failure()
 	}
 
-	slog.Debugf("Command output: %s", monitor.lastOutput)
+	if LogDebug {
-	slog.OnErrWarnf(err, "Command result: %v", err)
+		log.Printf("DEBUG: Command output: %s", monitor.lastOutput)
+	}
+	if err != nil {
+		if LogDebug {
+			log.Printf("DEBUG: Command result: %v", err)
+		}
+	}
 
-	slog.Infof(
+	log.Printf(
-		"%s success=%t, alert=%t",
+		"INFO: %s success=%t, alert=%t",
 		monitor.Name,
 		isSuccess,
 		alertNotice != nil,
@@ -91,17 +90,11 @@ func (monitor Monitor) IsUp() bool {
 	return monitor.alertCount == 0
 }
 
-// LastCheckMilliseconds gives number of miliseconds the last check ran for
-func (monitor Monitor) LastCheckMilliseconds() int64 {
-	return monitor.lastCheckDuration.Milliseconds()
-}
-
 func (monitor *Monitor) success() (notice *AlertNotice) {
 	if !monitor.IsUp() {
 		// Alert that we have recovered
 		notice = monitor.createAlertNotice(true)
 	}
-
 	monitor.failureCount = 0
 	monitor.alertCount = 0
 	monitor.lastSuccess = time.Now()
@@ -113,14 +106,15 @@ func (monitor *Monitor) failure() (notice *AlertNotice) {
 	monitor.failureCount++
 	// If we haven't hit the minimum failures, we can exit
 	if monitor.failureCount < monitor.getAlertAfter() {
-		slog.Debugf(
+		if LogDebug {
-			"%s failed but did not hit minimum failures. "+
+			log.Printf(
-				"Count: %v alert after: %v",
+				"DEBUG: %s failed but did not hit minimum failures. "+
-			monitor.Name,
+					"Count: %v alert after: %v",
-			monitor.failureCount,
+				monitor.Name,
-			monitor.getAlertAfter(),
+				monitor.failureCount,
-		)
+				monitor.getAlertAfter(),
-
+			)
+		}
 		return
 	}
 
@@ -128,20 +122,19 @@ func (monitor *Monitor) failure() (notice *AlertNotice) {
 	failureCount := (monitor.failureCount - monitor.getAlertAfter())
 
 	// Use alert cadence to determine if we should alert
-	switch {
+	if monitor.AlertEvery > 0 {
-	case monitor.AlertEvery > 0:
 		// Handle integer number of failures before alerting
 		if failureCount%monitor.AlertEvery == 0 {
 			notice = monitor.createAlertNotice(false)
 		}
-	case monitor.AlertEvery == 0:
+	} else if monitor.AlertEvery == 0 {
 		// Handle alerting on first failure only
 		if failureCount == 0 {
 			notice = monitor.createAlertNotice(false)
 		}
-	default:
+	} else {
 		// Handle negative numbers indicating an exponential backoff
-		if failureCount >= int16(math.Pow(2, float64(monitor.alertCount))-1) { //nolint:gomnd
+		if failureCount >= int16(math.Pow(2, float64(monitor.alertCount))-1) {
 			notice = monitor.createAlertNotice(false)
 		}
 	}
@@ -151,7 +144,7 @@ func (monitor *Monitor) failure() (notice *AlertNotice) {
 		monitor.alertCount++
 	}
 
-	return notice
+	return
 }
 
 func (monitor Monitor) getAlertAfter() int16 {
@@ -160,7 +153,6 @@ func (monitor Monitor) getAlertAfter() int16 {
 	if monitor.AlertAfter == 0 {
 		return 1
 	}
-
 	return monitor.AlertAfter
 }
 
@@ -169,7 +161,6 @@ func (monitor Monitor) GetAlertNames(up bool) []string {
 	if up {
 		return monitor.AlertUp
 	}
-
 	return monitor.AlertDown
 }
 

monitor_test.go

@@ -22,13 +22,11 @@ func TestMonitorIsValid(t *testing.T) {
 
 	for _, c := range cases {
 		log.Printf("Testing case %s", c.name)
-
 		actual := c.monitor.IsValid()
 		if actual != c.expected {
 			t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expected, actual)
 			log.Printf("Case failed: %s", c.name)
 		}
-
 		log.Println("-----")
 	}
 }
@@ -45,9 +43,9 @@ func TestMonitorShouldCheck(t *testing.T) {
 		name     string
 	}{
 		{Monitor{}, true, "Empty"},
-		{Monitor{lastCheck: timeNow, CheckInterval: SecondsOrDuration{time.Second * 15}}, false, "Just checked"},
+		{Monitor{lastCheck: timeNow, CheckInterval: 15}, false, "Just checked"},
-		{Monitor{lastCheck: timeTenSecAgo, CheckInterval: SecondsOrDuration{time.Second * 15}}, false, "-10s"},
+		{Monitor{lastCheck: timeTenSecAgo, CheckInterval: 15}, false, "-10s"},
-		{Monitor{lastCheck: timeTwentySecAgo, CheckInterval: SecondsOrDuration{time.Second * 15}}, true, "-20s"},
+		{Monitor{lastCheck: timeTwentySecAgo, CheckInterval: 15}, true, "-20s"},
 	}
 
 	for _, c := range cases {
@@ -73,13 +71,11 @@ func TestMonitorIsUp(t *testing.T) {
 
 	for _, c := range cases {
 		log.Printf("Testing case %s", c.name)
-
 		actual := c.monitor.IsUp()
 		if actual != c.expected {
 			t.Errorf("IsUp(%v), expected=%t actual=%t", c.name, c.expected, actual)
 			log.Printf("Case failed: %s", c.name)
 		}
-
 		log.Println("-----")
 	}
 }
@@ -100,13 +96,11 @@ func TestMonitorGetAlertNames(t *testing.T) {
 
 	for _, c := range cases {
 		log.Printf("Testing case %s", c.name)
-
 		actual := c.monitor.GetAlertNames(c.up)
 		if !EqualSliceString(actual, c.expected) {
 			t.Errorf("GetAlertNames(%v), expected=%v actual=%v", c.name, c.expected, actual)
 			log.Printf("Case failed: %s", c.name)
 		}
-
 		log.Println("-----")
 	}
 }
@@ -125,15 +119,12 @@ func TestMonitorSuccess(t *testing.T) {
 
 	for _, c := range cases {
 		log.Printf("Testing case %s", c.name)
-
 		notice := c.monitor.success()
 		hasNotice := (notice != nil)
-
 		if hasNotice != c.expectNotice {
 			t.Errorf("success(%v), expected=%t actual=%t", c.name, c.expectNotice, hasNotice)
 			log.Printf("Case failed: %s", c.name)
 		}
-
 		log.Println("-----")
 	}
 }
@@ -156,15 +147,12 @@ func TestMonitorFailureAlertAfter(t *testing.T) {
 
 	for _, c := range cases {
 		log.Printf("Testing case %s", c.name)
-
 		notice := c.monitor.failure()
 		hasNotice := (notice != nil)
-
 		if hasNotice != c.expectNotice {
 			t.Errorf("failure(%v), expected=%t actual=%t", c.name, c.expectNotice, hasNotice)
 			log.Printf("Case failed: %s", c.name)
 		}
-
 		log.Println("-----")
 	}
 }
@@ -207,12 +195,10 @@ func TestMonitorFailureAlertEvery(t *testing.T) {
 
 		notice := c.monitor.failure()
 		hasNotice := (notice != nil)
-
 		if hasNotice != c.expectNotice {
 			t.Errorf("failure(%v), expected=%t actual=%t", c.name, c.expectNotice, hasNotice)
 			log.Printf("Case failed: %s", c.name)
 		}
-
 		log.Println("-----")
 	}
 }
@@ -237,18 +223,15 @@ func TestMonitorFailureExponential(t *testing.T) {
 	// Unlike previous tests, this one requires a static Monitor with repeated
 	// calls to the failure method
 	monitor := Monitor{failureCount: 0, AlertAfter: 1, AlertEvery: -1}
-
 	for _, c := range cases {
 		log.Printf("Testing case %s", c.name)
 
 		notice := monitor.failure()
 		hasNotice := (notice != nil)
-
 		if hasNotice != c.expectNotice {
 			t.Errorf("failure(%v), expected=%t actual=%t", c.name, c.expectNotice, hasNotice)
 			log.Printf("Case failed: %s", c.name)
 		}
-
 		log.Println("-----")
 	}
 }
@@ -260,7 +243,6 @@ func TestMonitorCheck(t *testing.T) {
 		hasNotice  bool
 		lastOutput string
 	}
-
 	cases := []struct {
 		monitor Monitor
 		expect  expected
@@ -308,7 +290,6 @@ func TestMonitorCheck(t *testing.T) {
 			t.Errorf("Check(%v) (output), expected=%v actual=%v", c.name, c.expect.lastOutput, lastOutput)
 			log.Printf("Case failed: %s", c.name)
 		}
-
 		log.Println("-----")
 	}
 }

sample-config.yml

@@ -3,14 +3,14 @@ check_interval: 5
 
 monitors:
   - name: Fake Website
-    command: ["curl", "-s", "-o", "/dev/null", "https://minitor.mon"]
+    command: ['curl', '-s', '-o', '/dev/null', 'https://minitor.mon']
     alert_down: [log_down, mailgun_down, sms_down]
     alert_up: [log_up, email_up]
-    check_interval: 10 # Must be at minimum the global `check_interval`
+    check_interval: 10  # Must be at minimum the global `check_interval`
     alert_after: 3
-    alert_every: -1 # Defaults to -1 for exponential backoff. 0 to disable repeating
+    alert_every: -1  # Defaults to -1 for exponential backoff. 0 to disable repeating
   - name: Real Website
-    command: ["curl", "-s", "-o", "/dev/null", "https://google.com"]
+    command: ['curl', '-s', '-o', '/dev/null', 'https://google.com']
     alert_down: [log_down, mailgun_down, sms_down]
     alert_up: [log_up, email_up]
     check_interval: 5
@@ -23,8 +23,7 @@ alerts:
   log_up:
     command: ["echo", "Minitor recovery for {{.MonitorName}}"]
   email_up:
-    command:
+    command: [sendmail, "me@minitor.mon", "Recovered: {monitor_name}", "We're back!"]
-      [sendmail, "me@minitor.mon", "Recovered: {monitor_name}", "We're back!"]
   mailgun_down:
     command: >
       curl -s -X POST

test/valid-config.yml

@@ -3,23 +3,21 @@ check_interval: 1
 
 monitors:
   - name: Command
-    command: ["echo", "$PATH"]
+    command: ['echo', '$PATH']
-    alert_down: ["log_command", "log_shell"]
+    alert_down: ['log_command', 'log_shell']
     alert_every: 0
-    check_interval: 10s
   - name: Shell
     command: >
       echo 'Some string with stuff';
       echo 'another line';
       echo $PATH;
       exit 1
-    alert_down: ["log_command", "log_shell"]
+    alert_down: ['log_command', 'log_shell']
     alert_after: 5
     alert_every: 0
-    check_interval: 1m
 
 alerts:
   log_command:
-    command: ["echo", "regular", '"command!!!"', "{{.MonitorName}}"]
+    command: ['echo', 'regular', '"command!!!"', "{{.MonitorName}}"]
   log_shell:
     command: echo "Failure on {{.MonitorName}} User is $USER"

util.go

@@ -8,7 +8,7 @@ import (
 // ShellCommand takes a string and executes it as a command using `sh`
 func ShellCommand(command string) *exec.Cmd {
 	shellCommand := []string{"sh", "-c", strings.TrimSpace(command)}
-
+	//log.Printf("Shell command: %v", shellCommand)
 	return exec.Command(shellCommand[0], shellCommand[1:]...)
 }
 
@@ -17,12 +17,10 @@ func EqualSliceString(a, b []string) bool {
 	if len(a) != len(b) {
 		return false
 	}
-
 	for i, val := range a {
 		if val != b[i] {
 			return false
 		}
 	}
-
 	return true
 }