Compare commits

..

5 Commits

Author SHA1 Message Date
Ian Fijolek
f4a972747f Add notify after docker builds 2020-01-10 14:25:02 -08:00
Ian Fijolek
c7c82fabe8 Add qemu binaries 2020-01-10 14:21:48 -08:00
Ian Fijolek
f807caa1ad Add multi-arch builds 2020-01-10 13:58:17 -08:00
Ian Fijolek
3226be69e7 Fix issue with shell commands containing "<>" and unecessary (and poor) escaping 2020-01-07 10:37:53 -08:00
Ian Fijolek
0269ad3512 Add new test for multi-line YAML strings 2020-01-07 10:28:14 -08:00
16 changed files with 305 additions and 96 deletions
+58 -3
View File
@@ -45,15 +45,70 @@ trigger:
- refs/tags/v*
steps:
- name: build all binaries
image: golang:1.12
commands:
- make all
# Might consider moving this step into the previous pipeline
- name: push image
- name: push image - arm
image: plugins/docker
settings:
repo: iamthefij/minitor-go
dockerfile: Dockerfile.multi-stage
auto_tag: true
auto_tag_suffix: linux-arm
username:
from_secret: docker_username
password:
from_secret: docker_password
build_args:
- ARCH=arm
- REPO=arm32v7
- name: push image - arm64
image: plugins/docker
settings:
repo: iamthefij/minitor-go
auto_tag: true
auto_tag_suffix: linux-arm64
username:
from_secret: docker_username
password:
from_secret: docker_password
build_args:
- ARCH=arm64
- REPO=arm64v8
- name: push image - amd64
image: plugins/docker
settings:
repo: iamthefij/minitor-go
auto_tag: true
auto_tag_suffix: linux-amd64
username:
from_secret: docker_username
password:
from_secret: docker_password
- name: publish manifest
image: plugins/manifest
settings:
spec: manifest.tmpl
auto_tag: true
ignore_missing: true
username:
from_secret: docker_username
password:
from_secret: docker_password
- name: notify
image: drillster/drone-email
settings:
host:
from_secret: SMTP_HOST
username:
from_secret: SMTP_USER
password:
from_secret: SMTP_PASS
from: drone@iamthefij.com
when:
status: [changed, failure]
Vendored
+3 -1
View File
@@ -16,4 +16,6 @@
config.yml
# Output binary
minitor-go
minitor
minitor-linux-*
minitor-darwin-amd64
+9 -4
View File
@@ -1,12 +1,13 @@
ARG REPO=library
FROM multiarch/qemu-user-static:4.2.0-2 as qemu-user-static
FROM ${REPO}/alpine:3.10
# Copying all qemu files because amd64 doesn't exist and cannot condional copy
COPY --from=qemu-user-static /usr/bin/qemu-* /usr/bin/
RUN mkdir /app
WORKDIR /app/
# Copy minitor in
ARG ARCH=amd64
COPY ./minitor-go ./minitor
# Add common checking tools
RUN apk --no-cache add bash=~5.0 curl=~7.66 jq=~1.6
@@ -17,6 +18,10 @@ RUN addgroup -S minitor && adduser -S minitor -G minitor
COPY ./scripts /app/scripts
RUN chmod -R 755 /app/scripts
# Copy minitor in
ARG ARCH=amd64
COPY ./minitor-linux-${ARCH} ./minitor
# Drop to non-root user
USER minitor
+59 -11
View File
@@ -1,23 +1,28 @@
.PHONY: all
DOCKER_TAG ?= minitor-go-${USER}
GIT_TAG_NAME := $(shell git tag -l --contains HEAD)
GIT_SHA := $(shell git rev-parse HEAD)
VERSION := $(if $(GIT_TAG_NAME),$(GIT_TAG_NAME),$(GIT_SHA))
.PHONY: all
all: minitor-linux-amd64 minitor-linux-arm minitor-linux-arm64
.PHONY: default
default: test
.PHONY: build
build:
go build
build: minitor
minitor-go:
go build
minitor:
@echo Version: $(VERSION)
go build -ldflags '-X "main.version=${VERSION}"' -o minitor
.PHONY: run
run: minitor-go build
./minitor-go -debug
run: minitor build
./minitor -debug
.PHONY: run-metrics
run-metrics: minitor-go build
./minitor-go -debug -metrics
run-metrics: minitor build
./minitor -debug -metrics
.PHONY: test
test:
@@ -41,13 +46,56 @@ check:
.PHONY: clean
clean:
rm -f ./minitor-go
rm -f ./minitor
rm -f ./minitor-linux-*
rm -f ./minitor-darwin-amd64
rm -f ./coverage.out
.PHONY: docker-build
docker-build:
docker build -f ./Dockerfile.multi-stage -t $(DOCKER_TAG) .
docker build -f ./Dockerfile.multi-stage -t $(DOCKER_TAG)-linux-amd64 .
.PHONY: docker-run
docker-run: docker-build
docker run --rm -v $(shell pwd)/config.yml:/root/config.yml $(DOCKER_TAG)
## Multi-arch targets
# Arch specific go build targets
minitor-darwin-amd64:
GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 \
go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
-o minitor-darwin-amd64
minitor-linux-amd64:
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \
go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
-o minitor-linux-amd64
minitor-linux-arm:
GOOS=linux GOARCH=arm CGO_ENABLED=0 \
go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
-o minitor-linux-arm
minitor-linux-arm64:
GOOS=linux GOARCH=arm64 CGO_ENABLED=0 \
go build -ldflags '-X "main.version=${VERSION}"' -a -installsuffix nocgo \
-o minitor-linux-arm64
# Arch specific docker build targets
.PHONY: docker-build-arm
docker-build-arm: minitor-linux-arm
docker build --build-arg REPO=arm32v7 --build-arg ARCH=arm . -t ${DOCKER_TAG}-linux-arm
.PHONY: docker-build-arm
docker-build-arm64: minitor-linux-arm64
docker build --build-arg REPO=arm64v8 --build-arg ARCH=arm64 . -t ${DOCKER_TAG}-linux-arm64
# Cross run on host architechture
.PHONY: docker-run-arm
docker-run-arm: docker-build-arm
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock --name $(DOCKER_TAG)-run ${DOCKER_TAG}-linux-arm
.PHONY: docker-run-arm64
docker-run-arm64: docker-build-arm64
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock --name $(DOCKER_TAG)-run ${DOCKER_TAG}-linux-arm64
+12 -9
View File
@@ -3,11 +3,10 @@ package main
import (
"bytes"
"fmt"
"log"
"os/exec"
"text/template"
"time"
log "github.com/sirupsen/logrus"
)
// Alert is a config driven mechanism for sending a notice
@@ -39,7 +38,9 @@ func (alert Alert) IsValid() bool {
// BuildTemplates compiles command templates for the Alert
func (alert *Alert) BuildTemplates() error {
log.Debugf("Building template for alert %s", alert.Name)
if LogDebug {
log.Printf("DEBUG: Building template for alert %s", alert.Name)
}
if alert.commandTemplate == nil && alert.Command != nil {
alert.commandTemplate = []*template.Template{}
for i, cmdPart := range alert.Command {
@@ -59,8 +60,8 @@ func (alert *Alert) BuildTemplates() error {
}
// Send will send an alert notice by executing the command template
func (alert Alert) Send(notice AlertNotice) (outputStr string, err error) {
log.Infof("Sending alert %s for %s", alert.Name, notice.MonitorName)
func (alert Alert) Send(notice AlertNotice) (output_str string, err error) {
log.Printf("INFO: Sending alert %s for %s", alert.Name, notice.MonitorName)
var cmd *exec.Cmd
if alert.commandTemplate != nil {
command := []string{}
@@ -94,8 +95,10 @@ func (alert Alert) Send(notice AlertNotice) (outputStr string, err error) {
var output []byte
output, err = cmd.CombinedOutput()
outputStr = string(output)
log.Debugf("Alert output for: %s\n---\n%s\n---", alert.Name, outputStr)
return outputStr, err
output_str = string(output)
if LogDebug {
log.Printf("DEBUG: Alert output for: %s\n---\n%s\n---", alert.Name, output_str)
}
return output_str, err
}
+12 -13
View File
@@ -1,9 +1,8 @@
package main
import (
"log"
"testing"
log "github.com/sirupsen/logrus"
)
func TestAlertIsValid(t *testing.T) {
@@ -23,13 +22,13 @@ func TestAlertIsValid(t *testing.T) {
}
for _, c := range cases {
log.Debugf("Testing case %s", c.name)
log.Printf("Testing case %s", c.name)
actual := c.alert.IsValid()
if actual != c.expected {
t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expected, actual)
log.Debugf("Case failed: %s", c.name)
log.Printf("Case failed: %s", c.name)
}
log.Debugf("-----")
log.Println("-----")
}
}
@@ -72,19 +71,19 @@ func TestAlertSend(t *testing.T) {
}
for _, c := range cases {
log.Debugf("Testing case %s", c.name)
log.Printf("Testing case %s", c.name)
c.alert.BuildTemplates()
output, err := c.alert.Send(c.notice)
hasErr := (err != nil)
if output != c.expectedOutput {
t.Errorf("Send(%v output), expected=%v actual=%v", c.name, c.expectedOutput, output)
log.Debugf("Case failed: %s", c.name)
log.Printf("Case failed: %s", c.name)
}
if hasErr != c.expectErr {
t.Errorf("Send(%v err), expected=%v actual=%v", c.name, "Err", err)
log.Debugf("Case failed: %s", c.name)
log.Printf("Case failed: %s", c.name)
}
log.Debugf("-----")
log.Println("-----")
}
}
@@ -95,7 +94,7 @@ func TestAlertSendNoTemplates(t *testing.T) {
if err == nil {
t.Errorf("Send(no template), expected=%v actual=%v", "Err", output)
}
log.Debugf("-----")
log.Println("-----")
}
func TestAlertBuildTemplate(t *testing.T) {
@@ -110,13 +109,13 @@ func TestAlertBuildTemplate(t *testing.T) {
}
for _, c := range cases {
log.Debugf("Testing case %s", c.name)
log.Printf("Testing case %s", c.name)
err := c.alert.BuildTemplates()
hasErr := (err != nil)
if hasErr != c.expectErr {
t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expectErr, err)
log.Debugf("Case failed: %s", c.name)
log.Printf("Case failed: %s", c.name)
}
log.Debugf("-----")
log.Println("-----")
}
}
+11 -12
View File
@@ -3,9 +3,8 @@ package main
import (
"errors"
"io/ioutil"
"os"
"log"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
)
@@ -22,20 +21,20 @@ func (config Config) IsValid() (isValid bool) {
// Validate monitors
if config.Monitors == nil || len(config.Monitors) == 0 {
log.Errorf("Invalid monitor configuration: Must provide at least one monitor")
log.Printf("ERROR: Invalid monitor configuration: Must provide at least one monitor")
isValid = false
}
for _, monitor := range config.Monitors {
if !monitor.IsValid() {
log.Errorf("Invalid monitor configuration: %s", monitor.Name)
log.Printf("ERROR: Invalid monitor configuration: %s", monitor.Name)
isValid = false
}
// Check that all Monitor alerts actually exist
for _, isUp := range []bool{true, false} {
for _, alertName := range monitor.GetAlertNames(isUp) {
if _, ok := config.Alerts[alertName]; !ok {
log.Errorf(
"Invalid monitor configuration: %s. Unknown alert %s",
log.Printf(
"ERROR: Invalid monitor configuration: %s. Unknown alert %s",
monitor.Name, alertName,
)
isValid = false
@@ -46,12 +45,12 @@ func (config Config) IsValid() (isValid bool) {
// Validate alerts
if config.Alerts == nil || len(config.Alerts) == 0 {
log.Errorf("Invalid alert configuration: Must provide at least one alert")
log.Printf("ERROR: Invalid alert configuration: Must provide at least one alert")
isValid = false
}
for _, alert := range config.Alerts {
if !alert.IsValid() {
log.Errorf("Invalid alert configuration: %s", alert.Name)
log.Printf("ERROR: Invalid alert configuration: %s", alert.Name)
isValid = false
}
}
@@ -78,14 +77,14 @@ func LoadConfig(filePath string) (config Config, err error) {
return
}
// TODO: Decide if this is better expanded here, or only when executing
envExpanded := os.ExpandEnv(string(data))
err = yaml.Unmarshal([]byte(envExpanded), &config)
err = yaml.Unmarshal(data, &config)
if err != nil {
return
}
log.Debugf("Config values:\n%v\n", config)
if LogDebug {
log.Printf("DEBUG: Config values:\n%v\n", config)
}
if !config.IsValid() {
err = errors.New("Invalid configuration")
+66 -1
View File
@@ -23,9 +23,74 @@ func TestLoadConfig(t *testing.T) {
_, err := LoadConfig(c.configPath)
hasErr := (err != nil)
if hasErr != c.expectErr {
t.Errorf("LoadConfig(%v), expected=%v actual=%v", c.name, "Err", err)
t.Errorf("LoadConfig(%v), expected_error=%v actual=%v", c.name, c.expectErr, err)
log.Printf("Case failed: %s", c.name)
}
log.Println("-----")
}
}
// TestMultiLineConfig is a more complicated test stepping through the parsing
// and execution of mutli-line strings presented in YAML
func TestMultiLineConfig(t *testing.T) {
log.Println("Testing multi-line string config")
config, err := LoadConfig("./test/valid-verify-multi-line.yml")
if err != nil {
t.Fatalf("TestMultiLineConfig(load), expected=no_error actual=%v", err)
}
log.Println("-----")
log.Println("TestMultiLineConfig(parse > string)")
expected := "echo 'Some string with stuff'; echo \"<angle brackets>\"; exit 1\n"
actual := config.Monitors[0].CommandShell
if expected != actual {
t.Errorf("TestMultiLineConfig(>) failed")
t.Logf("string expected=`%v`", expected)
t.Logf("string actual =`%v`", actual)
t.Logf("bytes expected=%v", []byte(expected))
t.Logf("bytes actual =%v", []byte(actual))
}
log.Println("-----")
log.Println("TestMultiLineConfig(execute > string)")
_, notice := config.Monitors[0].Check()
if notice == nil {
t.Fatalf("Did not receive an alert notice")
}
expected = "Some string with stuff\n<angle brackets>\n"
actual = notice.LastCheckOutput
if expected != actual {
t.Errorf("TestMultiLineConfig(execute > string) check failed")
t.Logf("string expected=`%v`", expected)
t.Logf("string actual =`%v`", actual)
t.Logf("bytes expected=%v", []byte(expected))
t.Logf("bytes actual =%v", []byte(actual))
}
log.Println("-----")
log.Println("TestMultiLineConfig(parse | string)")
expected = "echo 'Some string with stuff'\necho '<angle brackets>'\n"
actual = config.Alerts["log_shell"].CommandShell
if expected != actual {
t.Errorf("TestMultiLineConfig(|) failed")
t.Logf("string expected=`%v`", expected)
t.Logf("string actual =`%v`", actual)
t.Logf("bytes expected=%v", []byte(expected))
t.Logf("bytes actual =%v", []byte(actual))
}
log.Println("-----")
log.Println("TestMultiLineConfig(execute | string)")
actual, err = config.Alerts["log_shell"].Send(AlertNotice{})
if err != nil {
t.Errorf("Execution of alert failed")
}
expected = "Some string with stuff\n<angle brackets>\n"
if expected != actual {
t.Errorf("TestMultiLineConfig(execute | string) check failed")
t.Logf("string expected=`%v`", expected)
t.Logf("string actual =`%v`", actual)
t.Logf("bytes expected=%v", []byte(expected))
t.Logf("bytes actual =%v", []byte(actual))
}
}
-1
View File
@@ -4,6 +4,5 @@ go 1.12
require (
github.com/prometheus/client_golang v1.2.1
github.com/sirupsen/logrus v1.4.2
gopkg.in/yaml.v2 v2.2.4
)
-2
View File
@@ -53,7 +53,6 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -69,7 +68,6 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+8 -9
View File
@@ -3,12 +3,14 @@ package main
import (
"flag"
"fmt"
"log"
"time"
log "github.com/sirupsen/logrus"
)
var (
// LogDebug will control whether debug messsages should be logged
LogDebug = false
// ExportMetrics will track whether or not we want to export metrics to prometheus
ExportMetrics = false
// MetricsPort is the port to expose metrics on
@@ -33,7 +35,9 @@ func checkMonitors(config *Config) error {
// Should probably consider refactoring everything below here
if alertNotice != nil {
log.Debugf("Recieved an alert notice from %s", alertNotice.MonitorName)
if LogDebug {
log.Printf("DEBUG: Recieved an alert notice from %s", alertNotice.MonitorName)
}
alertNames := monitor.GetAlertNames(alertNotice.IsUp)
if alertNames == nil {
// This should only happen for a recovery alert. AlertDown is validated not empty
@@ -77,16 +81,11 @@ func checkMonitors(config *Config) error {
func main() {
// Get debug flag
var debug = flag.Bool("debug", false, "Enables debug logs (default: false)")
flag.BoolVar(&LogDebug, "debug", false, "Enables debug logs (default: false)")
flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)")
var showVersion = flag.Bool("version", false, "Display the version of minitor and exit")
flag.Parse()
// Set debug if flag is set
if *debug {
log.SetLevel(log.DebugLevel)
}
// Print version if flag is provided
if *showVersion {
log.Println("Minitor version:", version)
+25
View File
@@ -0,0 +1,25 @@
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}
{{#if build.tags}}
tags:
{{#each build.tags}}
- {{this}}
{{/each}}
{{/if}}
manifests:
-
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64
platform:
architecture: amd64
os: linux
-
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64
platform:
architecture: arm64
os: linux
variant: v8
-
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm
platform:
architecture: arm
os: linux
variant: v7
+15 -12
View File
@@ -1,11 +1,10 @@
package main
import (
"log"
"math"
"os/exec"
"time"
log "github.com/sirupsen/logrus"
)
// Monitor represents a particular periodic check of a command
@@ -71,18 +70,20 @@ func (monitor *Monitor) Check() (bool, *AlertNotice) {
alertNotice = monitor.failure()
}
log.Debugf("Command output: %s", monitor.lastOutput)
if LogDebug {
log.Printf("DEBUG: Command output: %s", monitor.lastOutput)
}
if err != nil {
log.Debugf("Command result: %v", err)
if LogDebug {
log.Printf("DEBUG: Command result: %v", err)
}
}
log.WithFields(log.Fields{
"monitor": monitor.Name,
"success": isSuccess,
"alert": alertNotice != nil,
}).Infof(
"%s checked",
log.Printf(
"INFO: %s success=%t, alert=%t",
monitor.Name,
isSuccess,
alertNotice != nil,
)
return isSuccess, alertNotice
@@ -108,13 +109,15 @@ func (monitor *Monitor) failure() (notice *AlertNotice) {
monitor.failureCount++
// If we haven't hit the minimum failures, we can exit
if monitor.failureCount < monitor.getAlertAfter() {
log.Debugf(
"%s failed but did not hit minimum failures. "+
if LogDebug {
log.Printf(
"DEBUG: %s failed but did not hit minimum failures. "+
"Count: %v alert after: %v",
monitor.Name,
monitor.failureCount,
monitor.getAlertAfter(),
)
}
return
}
+1
View File
@@ -1,3 +1,4 @@
---
check_interval: 1
monitors:
+18
View File
@@ -0,0 +1,18 @@
---
check_interval: 1
monitors:
- name: Shell
command_shell: >
echo 'Some string with stuff';
echo "<angle brackets>";
exit 1
alert_down: ['log_shell']
alert_after: 1
alert_every: 0
alerts:
log_shell:
command_shell: |
echo 'Some string with stuff'
echo '<angle brackets>'
+1 -11
View File
@@ -5,19 +5,9 @@ import (
"strings"
)
// escapeCommandShell accepts a command to be executed by a shell and escapes it
func escapeCommandShell(command string) string {
// Remove extra spaces and newlines from ends
command = strings.TrimSpace(command)
// TODO: Not sure if this part is actually needed. Should verify
// Escape double quotes since this will be passed in as an argument
command = strings.Replace(command, `"`, `\"`, -1)
return command
}
// ShellCommand takes a string and executes it as a command using `sh`
func ShellCommand(command string) *exec.Cmd {
shellCommand := []string{"sh", "-c", escapeCommandShell(command)}
shellCommand := []string{"sh", "-c", strings.TrimSpace(command)}
//log.Printf("Shell command: %v", shellCommand)
return exec.Command(shellCommand[0], shellCommand[1:]...)
}