diff --git a/Dockerfile b/Dockerfile index 841d1c9..f512195 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,21 @@ -FROM alpine:latest -RUN apk add --update ca-certificates +FROM golang:latest COPY ./default.tmpl /templates/default.tmpl -COPY ./alertmanager-bot /usr/bin/alertmanager-bot +# Set go bin which doesn't appear to be set already. +ENV GOBIN /go/bin -EXPOSE 8080 +# build directories +RUN mkdir /app +RUN mkdir -p /go/src/github.com/vu-long/alertmanager-bot +ADD . /go/src/github.com/vu-long/alertmanager-bot +WORKDIR /go/src/github.com/vu-long/alertmanager-bot -ENTRYPOINT ["/usr/bin/alertmanager-bot"] +# Go dep! +RUN go get -u github.com/golang/dep/... +RUN dep ensure -v -vendor-only + +RUN make build + +EXPOSE 8080:8080 + +ENTRYPOINT ["/go/src/github.com/vu-long/alertmanager-bot/alertmanager-bot"] diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 0000000..73803f1 --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,498 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + branch = "master" + digest = "1:315c5f2f60c76d89b871c73f9bd5fe689cad96597afd50fb9992228ef80bdd34" + name = "github.com/alecthomas/template" + packages = [ + ".", + "parse", + ] + pruneopts = "UT" + revision = "a0175ee3bccc567396460bf5acd36800cb10c49c" + +[[projects]] + branch = "master" + digest = "1:c198fdc381e898e8fb62b8eb62758195091c313ad18e52a3067366e1dda2fb3c" + name = "github.com/alecthomas/units" + packages = ["."] + pruneopts = "UT" + revision = "2efee857e7cfd4f3d0138cc3cbb1b4966962b93a" + +[[projects]] + branch = "master" + digest = "1:ef5b0622d834c139454148b8fd0c92bb314828900532b267ae62da9fec109866" + name = "github.com/armon/go-metrics" + packages = ["."] + pruneopts = "UT" + revision = "f0300d1749da6fa982027e449ec0c7a145510c3c" + +[[projects]] + digest = "1:c47f4964978e211c6e566596ec6246c329912ea92e9bb99c00798bb4564c5b09" + name = "github.com/armon/go-radix" + packages = ["."] + pruneopts = "UT" + revision = "1a2de0c21c94309923825da3df33a4381872c795" + version = "v1.0.0" + +[[projects]] + branch = "master" + digest = "1:d6afaeed1502aa28e80a4ed0981d570ad91b2579193404256ce672ed0a609e0d" + name = "github.com/beorn7/perks" + packages = ["quantile"] + pruneopts = "UT" + revision = "3a771d992973f24aa725d07868b467d1ddfceafb" + +[[projects]] + digest = "1:0f98f59e9a2f4070d66f0c9c39561f68fcd1dc837b22a852d28d0003aebd1b1e" + name = "github.com/boltdb/bolt" + packages = ["."] + pruneopts = "UT" + revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8" + version = "v1.3.1" + +[[projects]] + digest = "1:166438587ed45ac211dab8a3ecebf4fa0c186d0db63430fb9127bbc2e5fcdc67" + name = "github.com/cenkalti/backoff" + packages = ["."] + pruneopts = "UT" + revision = "1e4cf3da559842a91afcb6ea6141451e6c30c618" + version = "v2.1.1" + +[[projects]] + digest = "1:998cf998358a303ac2430c386ba3fd3398477d6013153d3c6e11432765cc9ae6" + name = "github.com/cespare/xxhash" + packages = ["."] + pruneopts = "UT" + revision = "3b82fb7d186719faeedd0c2864f868c74fbf79a1" + version = "v2.0.0" + +[[projects]] + digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec" + name = "github.com/davecgh/go-spew" + packages = ["spew"] + pruneopts = "UT" + revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" + version = "v1.1.1" + +[[projects]] + digest = "1:1cf8b80d775788921f6e309a8d7da4a925780e55e887feca461cda04b8ceea45" + name = "github.com/docker/libkv" + packages = [ + ".", + "store", + "store/boltdb", + "store/consul", + ] + pruneopts = "UT" + revision = "aabc039ad04deb721e234f99cd1b4aa28ac71a40" + version = "v0.2.1" + +[[projects]] + digest = "1:0bec1c27a357178fe656168f96f7b89f338d73815ddca2af3a2eeb39f70c837f" + name = "github.com/go-kit/kit" + packages = [ + "log", + "log/level", + ] + pruneopts = "UT" + revision = "e2b298466b32c7cd5579a9b9b07e968fc9d9452c" + +[[projects]] + digest = "1:4062bc6de62d73e2be342243cf138cf499b34d558876db8d9430e2149388a4d8" + name = "github.com/go-logfmt/logfmt" + packages = ["."] + pruneopts = "UT" + revision = "07c9b44f60d7ffdfb7d8efe1ad539965737836dc" + version = "v0.4.0" + +[[projects]] + digest = "1:586ea76dbd0374d6fb649a91d70d652b7fe0ccffb8910a77468e7702e7901f3d" + name = "github.com/go-stack/stack" + packages = ["."] + pruneopts = "UT" + revision = "2fee6af1a9795aafbe0253a0cfbdf668e1fb8a9a" + version = "v1.8.0" + +[[projects]] + digest = "1:a1be218880631ef6719b9b565ffd744cf67907f2a6c19bba86f8f1dc66d9a62e" + name = "github.com/gogo/protobuf" + packages = [ + "gogoproto", + "proto", + "protoc-gen-gogo/descriptor", + "sortkeys", + "types", + ] + pruneopts = "UT" + revision = "616a82ed12d78d24d4839363e8f3c5d3f20627cf" + +[[projects]] + digest = "1:97df918963298c287643883209a2c3f642e6593379f97ab400c2a2e219ab647d" + name = "github.com/golang/protobuf" + packages = ["proto"] + pruneopts = "UT" + revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5" + version = "v1.2.0" + +[[projects]] + digest = "1:09970bb8daf100bdf11b48b72c71ef980cd2b50efbd903989b3d182956be0b83" + name = "github.com/hako/durafmt" + packages = ["."] + pruneopts = "UT" + revision = "7b7ae1e72eade09dbc9c2cfba3e6c4bae7b8bcac" + version = "1.0.0" + +[[projects]] + digest = "1:bfc483a051d3c7185ebeaa41b5bb67a4f76e742217bcaeab5661cc4b1320f392" + name = "github.com/hashicorp/consul" + packages = ["api"] + pruneopts = "UT" + revision = "0bddfa23a2ebe3c0773d917fc104f53d74f7a5ec" + version = "v1.4.0" + +[[projects]] + digest = "1:0ade334594e69404d80d9d323445d2297ff8161637f9b2d347cc6973d2d6f05b" + name = "github.com/hashicorp/errwrap" + packages = ["."] + pruneopts = "UT" + revision = "8a6fb523712970c966eefc6b39ed2c5e74880354" + version = "v1.0.0" + +[[projects]] + digest = "1:77cb3be9b21ba7f1a4701e870c84ea8b66e7d74c7c8951c58155fdadae9414ec" + name = "github.com/hashicorp/go-cleanhttp" + packages = ["."] + pruneopts = "UT" + revision = "d5fe4b57a186c716b0e00b8c301cbd9b4182694d" + +[[projects]] + digest = "1:2be5a35f0c5b35162c41bb24971e5dcf6ce825403296ee435429cdcc4e1e847e" + name = "github.com/hashicorp/go-immutable-radix" + packages = ["."] + pruneopts = "UT" + revision = "27df80928bb34bb1b0d6d0e01b9e679902e7a6b5" + version = "v1.0.0" + +[[projects]] + branch = "master" + digest = "1:b8ba23b0b493e601d5ebaf21079bd3433eb304a5d73c0fb40b8aea526e94f46b" + name = "github.com/hashicorp/go-msgpack" + packages = ["codec"] + pruneopts = "UT" + revision = "fa3f63826f7c23912c15263591e65d54d080b458" + +[[projects]] + digest = "1:f668349b83f7d779567c880550534addeca7ebadfdcf44b0b9c39be61864b4b7" + name = "github.com/hashicorp/go-multierror" + packages = ["."] + pruneopts = "UT" + revision = "886a7fbe3eb1c874d46f623bfa70af45f425b3d1" + version = "v1.0.0" + +[[projects]] + branch = "master" + digest = "1:45aad874d3c7d5e8610427c81870fb54970b981692930ec2a319ce4cb89d7a00" + name = "github.com/hashicorp/go-rootcerts" + packages = ["."] + pruneopts = "UT" + revision = "6bb64b370b90e7ef1fa532be9e591a81c3493e00" + +[[projects]] + digest = "1:ba05d01857d121ba6ab1613ec7a153de2aaf8598224dda71e669a01733d08e92" + name = "github.com/hashicorp/go-sockaddr" + packages = ["."] + pruneopts = "UT" + revision = "9b4c5fa5b10a683339a270d664474b9f4aee62fc" + +[[projects]] + digest = "1:67474f760e9ac3799f740db2c489e6423a4cde45520673ec123ac831ad849cb8" + name = "github.com/hashicorp/golang-lru" + packages = ["simplelru"] + pruneopts = "UT" + revision = "20f1fb78b0740ba8c3cb143a61e86ba5c8669768" + version = "v0.5.0" + +[[projects]] + digest = "1:83264bfcb4a76fe13c71fd2c36fc7c4cef3284f6ba5d81d291e036e93d7d10c7" + name = "github.com/hashicorp/memberlist" + packages = ["."] + pruneopts = "UT" + revision = "687988a0b5daaf7ed5051e5e374aef27f8254822" + +[[projects]] + digest = "1:acc81e4e4289587b257ccdfccbc6eaf16d4c2fb57dda73c6bb349bf50f02501f" + name = "github.com/hashicorp/serf" + packages = ["coordinate"] + pruneopts = "UT" + revision = "19bbd39e421bdf3559d5025fb2c760f5ffa56233" + +[[projects]] + digest = "1:ecd9aa82687cf31d1585d4ac61d0ba180e42e8a6182b85bd785fcca8dfeefc1b" + name = "github.com/joho/godotenv" + packages = ["."] + pruneopts = "UT" + revision = "23d116af351c84513e1946b527c88823e476be13" + version = "v1.3.0" + +[[projects]] + branch = "master" + digest = "1:a64e323dc06b73892e5bb5d040ced475c4645d456038333883f58934abbf6f72" + name = "github.com/kr/logfmt" + packages = ["."] + pruneopts = "UT" + revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0" + +[[projects]] + digest = "1:ff5ebae34cfbf047d505ee150de27e60570e8c394b3b8fdbb720ff6ac71985fc" + name = "github.com/matttproud/golang_protobuf_extensions" + packages = ["pbutil"] + pruneopts = "UT" + revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" + version = "v1.0.1" + +[[projects]] + digest = "1:478b8f5ee3eac843d206afd9c2c2a7133d0b7da7a65e8f9edbb1ebbf6758bdb0" + name = "github.com/miekg/dns" + packages = ["."] + pruneopts = "UT" + revision = "44a8c5f8ba7d0b3eac116182736dcfb5885ee405" + version = "v1.1.2" + +[[projects]] + digest = "1:78bbb1ba5b7c3f2ed0ea1eab57bdd3859aec7e177811563edc41198a760b06af" + name = "github.com/mitchellh/go-homedir" + packages = ["."] + pruneopts = "UT" + revision = "ae18d6b8b3205b561c79e8e5f69bff09736185f4" + version = "v1.0.0" + +[[projects]] + digest = "1:61332bb44d05257bbf0356d8400a8b30fe0b9fdc3b72b8b55661da8f0a4f39ae" + name = "github.com/mitchellh/hashstructure" + packages = ["."] + pruneopts = "UT" + revision = "a38c50148365edc8df43c1580c48fb2b3a1e9cd7" + version = "v1.0.0" + +[[projects]] + digest = "1:bba69fe95a60f59a68df402c23d57ffdc5596200901189958aea969c468d550f" + name = "github.com/mitchellh/mapstructure" + packages = ["."] + pruneopts = "UT" + revision = "5a380f224700b8a6c4eaad048804f5bff514cb35" + +[[projects]] + branch = "master" + digest = "1:9f07f801988b225662081432361c430cad8f5293b134e80bdf1998d14969d7a6" + name = "github.com/mwitkow/go-conntrack" + packages = ["."] + pruneopts = "UT" + revision = "cc309e4a22231782e8893f3c35ced0967807a33e" + +[[projects]] + digest = "1:9ec6cf1df5ad1d55cf41a43b6b1e7e118a91bade4f68ff4303379343e40c0e25" + name = "github.com/oklog/run" + packages = ["."] + pruneopts = "UT" + revision = "4dadeb3030eda0273a12382bb2348ffc7c9d1a39" + version = "v1.0.0" + +[[projects]] + digest = "1:15ddf3998dbeca86ffee1a3774353ef6e5868ff2e3898ab9ab1438b83079f9c9" + name = "github.com/oklog/ulid" + packages = ["."] + pruneopts = "UT" + revision = "66bb6560562feca7045b23db1ae85b01260f87c5" + +[[projects]] + digest = "1:cf31692c14422fa27c83a05292eb5cbe0fb2775972e8f1f8446a71549bd8980b" + name = "github.com/pkg/errors" + packages = ["."] + pruneopts = "UT" + revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4" + version = "v0.8.1" + +[[projects]] + digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe" + name = "github.com/pmezard/go-difflib" + packages = ["difflib"] + pruneopts = "UT" + revision = "792786c7400a136282c1664665ae0a8db921c6c2" + version = "v1.0.0" + +[[projects]] + digest = "1:5e278dfd778c44ca6b856e843a48449151fe531c989a8c9b983de715a1d42726" + name = "github.com/prometheus/alertmanager" + packages = [ + "cluster", + "cluster/clusterpb", + "config", + "nflog", + "nflog/nflogpb", + "notify", + "silence", + "silence/silencepb", + "template", + "template/internal/deftmpl", + "types", + ] + pruneopts = "UT" + revision = "d4a7697cc90f8bce62efe7c44b63b542578ec0a1" + version = "v0.15.3" + +[[projects]] + digest = "1:9e6e005a467ff9b1e4bb32e4c64ef36f6ab9c3313335acc66b83eaf9570d1d95" + name = "github.com/prometheus/client_golang" + packages = [ + "prometheus", + "prometheus/promhttp", + ] + pruneopts = "UT" + revision = "c3324c1198cf3374996e9d3098edd46a6b55afc9" + +[[projects]] + branch = "master" + digest = "1:2d5cd61daa5565187e1d96bae64dbbc6080dacf741448e9629c64fd93203b0d4" + name = "github.com/prometheus/client_model" + packages = ["go"] + pruneopts = "UT" + revision = "5c3871d89910bfb32f5fcab2aa4b9ec68e65a99f" + +[[projects]] + digest = "1:f5230e376231e489365dd86ceea2e5205a42f5ed4ceaab13c4e7f9eb75ceb666" + name = "github.com/prometheus/common" + packages = [ + "config", + "expfmt", + "internal/bitbucket.org/ww/goautoneg", + "model", + "version", + ] + pruneopts = "UT" + revision = "7600349dcfe1abd18d72d3a1770870d9800a7801" + +[[projects]] + branch = "master" + digest = "1:08eb8b60450efe841e37512d66ce366a87d187505d7c67b99307a6c1803483a2" + name = "github.com/prometheus/procfs" + packages = [ + ".", + "internal/util", + "nfs", + "xfs", + ] + pruneopts = "UT" + revision = "b1a0a9a36d7453ba0f62578b99712f3a6c5f82d1" + +[[projects]] + digest = "1:274f67cb6fed9588ea2521ecdac05a6d62a8c51c074c1fccc6a49a40ba80e925" + name = "github.com/satori/go.uuid" + packages = ["."] + pruneopts = "UT" + revision = "f58768cc1a7a7e77a3bd49e98cdd21419399b6a3" + version = "v1.2.0" + +[[projects]] + branch = "master" + digest = "1:579c4bbcc2e16d4caf871ba91c0e2c331b07c5560c80d142d82c0de01c57fa96" + name = "github.com/sean-/seed" + packages = ["."] + pruneopts = "UT" + revision = "e2103e2c35297fb7e17febb81e49b312087a2372" + +[[projects]] + digest = "1:972c2427413d41a1e06ca4897e8528e5a1622894050e2f527b38ddf0f343f759" + name = "github.com/stretchr/testify" + packages = ["assert"] + pruneopts = "UT" + revision = "ffdc059bfe9ce6a4e144ba849dbedead332c6053" + version = "v1.3.0" + +[[projects]] + digest = "1:6a519ce8a4f7e5da2e12c7b47da4711750c085d8e3a6449d046b17eea42069a9" + name = "github.com/tucnak/telebot" + packages = ["."] + pruneopts = "UT" + revision = "00cebf376d79ee4363a11cd66fbbdf565452e0ab" + version = "v1.0" + +[[projects]] + branch = "master" + digest = "1:d5891c5bca9c62e5d394ca26491d2b710a1dc08cedeb0ca8f9ac4c3305120b02" + name = "golang.org/x/crypto" + packages = [ + "ed25519", + "ed25519/internal/edwards25519", + ] + pruneopts = "UT" + revision = "ff983b9c42bc9fbf91556e191cc8efb585c16908" + +[[projects]] + branch = "master" + digest = "1:cc69b66377d151326324dd9225f034c9d4ba002d8424dbc2d1d82c950dbaa89b" + name = "golang.org/x/net" + packages = [ + "bpf", + "context", + "context/ctxhttp", + "internal/iana", + "internal/socket", + "internal/timeseries", + "ipv4", + "ipv6", + "trace", + ] + pruneopts = "UT" + revision = "1e06a53dbb7e2ed46e91183f219db23c6943c532" + +[[projects]] + branch = "master" + digest = "1:f106a08663d0031b576f61ab81ffd80124da17096eab36aceb25b1b0d847da9a" + name = "golang.org/x/sys" + packages = ["unix"] + pruneopts = "UT" + revision = "7fbe1cd0fcc20051e1fcb87fbabec4a1bacaaeba" + +[[projects]] + digest = "1:c06d9e11d955af78ac3bbb26bd02e01d2f61f689e1a3bce2ef6fb683ef8a7f2d" + name = "gopkg.in/alecthomas/kingpin.v2" + packages = ["."] + pruneopts = "UT" + revision = "947dcec5ba9c011838740e680966fd7087a71d0d" + version = "v2.2.6" + +[[projects]] + digest = "1:342378ac4dcb378a5448dd723f0784ae519383532f5e70ade24132c4c8693202" + name = "gopkg.in/yaml.v2" + packages = ["."] + pruneopts = "UT" + revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" + version = "v2.2.1" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + input-imports = [ + "github.com/cenkalti/backoff", + "github.com/docker/libkv/store", + "github.com/docker/libkv/store/boltdb", + "github.com/docker/libkv/store/consul", + "github.com/go-kit/kit/log", + "github.com/go-kit/kit/log/level", + "github.com/hako/durafmt", + "github.com/joho/godotenv", + "github.com/oklog/run", + "github.com/pkg/errors", + "github.com/prometheus/alertmanager/notify", + "github.com/prometheus/alertmanager/template", + "github.com/prometheus/alertmanager/types", + "github.com/prometheus/client_golang/prometheus", + "github.com/prometheus/client_golang/prometheus/promhttp", + "github.com/stretchr/testify/assert", + "github.com/tucnak/telebot", + "gopkg.in/alecthomas/kingpin.v2", + ] + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 0000000..17004f0 --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,70 @@ +# Gopkg.toml example +# +# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" +# +# [prune] +# non-go = false +# go-tests = true +# unused-packages = true + + +[[constraint]] + name = "github.com/cenkalti/backoff" + version = "2.1.1" + +[[constraint]] + name = "github.com/docker/libkv" + version = "0.2.1" + +[[constraint]] + name = "github.com/hako/durafmt" + version = "1.0.0" + +[[constraint]] + name = "github.com/joho/godotenv" + version = "1.3.0" + +[[constraint]] + name = "github.com/oklog/run" + version = "1.0.0" + +[[constraint]] + name = "github.com/pkg/errors" + version = "0.8.1" + +[[constraint]] + name = "github.com/prometheus/alertmanager" + version = "0.15.3" + +[[constraint]] + name = "github.com/stretchr/testify" + version = "1.3.0" + +[[constraint]] + name = "github.com/tucnak/telebot" + version = "1.0.0" + +[[constraint]] + name = "gopkg.in/alecthomas/kingpin.v2" + version = "2.2.6" + +[prune] + go-tests = true + unused-packages = true diff --git a/Makefile b/Makefile index 564b631..319a31f 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ EXECUTABLE ?= alertmanager-bot -IMAGE ?= metalmatze/$(EXECUTABLE) +IMAGE ?= vulong/$(EXECUTABLE) GO := CGO_ENABLED=0 go DATE := $(shell date -u '+%FT%T%z') diff --git a/cmd/alertmanager-bot/main.go b/cmd/alertmanager-bot/main.go index 83728aa..b535b0d 100644 --- a/cmd/alertmanager-bot/main.go +++ b/cmd/alertmanager-bot/main.go @@ -18,14 +18,14 @@ import ( "github.com/go-kit/kit/log/level" "github.com/hako/durafmt" "github.com/joho/godotenv" - "github.com/metalmatze/alertmanager-bot/pkg/alertmanager" - "github.com/metalmatze/alertmanager-bot/pkg/telegram" "github.com/oklog/run" "github.com/prometheus/alertmanager/notify" "github.com/prometheus/alertmanager/template" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - "gopkg.in/alecthomas/kingpin.v2" + "github.com/vu-long/alertmanager-bot/pkg/alertmanager" + "github.com/vu-long/alertmanager-bot/pkg/telegram" + kingpin "gopkg.in/alecthomas/kingpin.v2" ) const ( diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4499932 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,144 @@ +version: "3" + +networks: + monitor-net: + driver: bridge + +volumes: + prometheus_data: {} + grafana_data: {} + +services: + # alertmanager-bot: + # build: + # context: . + # dockerfile: Dockerfile + # container_name: alertmanager-bot + # # image: alertmanager-bot:latest + # environment: + # ALERTMANAGER_URL: http://alertmanager:9093 + # BOLT_PATH: /data/bot.db + # CONSUL_URL: consul-agent:8500 + # LISTEN_ADDR: 0.0.0.0:8080 + # STORE: consul + # TELEGRAM_ADMIN: 789593887 + # TELEGRAM_TOKEN: 715767920:AAHM_dFYTF8Q4FG3gnQjvljKCJx4n1dCWuI + # TEMPLATE_PATHS: /templates/default.tmpl + # ports: + # - 8080:8080 + # restart: unless-stopped + # networks: + # - monitor-net + # # volumes: + # # - /srv/monitoring/alertmanager-bot:/data + + # consul-agent: + # image: consul:latest + # container_name: consul-agent + # restart: unless-stopped + # networks: + # - monitor-net + # command: "agent -dev" + # ports: + # - 8500:8500 + + prometheus: + image: prom/prometheus:latest + container_name: prometheus + volumes: + - ./services/prometheus/:/etc/prometheus/ + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/etc/prometheus/console_libraries' + - '--web.console.templates=/etc/prometheus/consoles' + - '--storage.tsdb.retention=200h' + - '--web.enable-lifecycle' + restart: unless-stopped + expose: + - 9090 + ports: + - "9090:9090" + networks: + - monitor-net + + # nodeexporter: + # image: prom/node-exporter:latest + # container_name: nodeexporter + # user: root + # privileged: true + # volumes: + # - /proc:/host/proc:ro + # - /sys:/host/sys:ro + # - /:/rootfs:ro + # command: + # - '--path.procfs=/host/proc' + # - '--path.sysfs=/host/sys' + # - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)' + # restart: unless-stopped + # expose: + # - 9100 + # networks: + # - monitor-net + + alertmanager: + image: prom/alertmanager + container_name: alertmanager + volumes: + - ./services/alertmanager/:/etc/alertmanager/ + command: + - '--config.file=/etc/alertmanager/webhookconfig.yml' + - '--storage.path=/alertmanager' + - '--web.listen-address=0.0.0.0:9093' + restart: unless-stopped + ports: + - "9093:9093" + networks: + - monitor-net + + blackbox-exporter: + image: prom/blackbox-exporter + container_name: blackbox + restart: unless-stopped + ports: + - "9115:9115" + networks: + - monitor-net + + grafana: + image: grafana/grafana:latest + container_name: grafana + volumes: + - grafana_data:/var/lib/grafana + - ./services/grafana/datasources:/etc/grafana/datasources + - ./services/grafana/dashboards:/etc/grafana/dashboards + - ./services/grafana/setup.sh:/setup.sh + entrypoint: /setup.sh + environment: + - GF_SECURITY_ADMIN_USER=${ADMIN_USER:-admin} + - GF_SECURITY_ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin} + - GF_USERS_ALLOW_SIGN_UP=false + restart: unless-stopped + expose: + - 3000 + ports: + - 3000:3000 + networks: + - monitor-net + + httpd: + image: httpd + ports: + - "5000:80" + container_name: "httpd" + networks: + - monitor-net + + nginx: + image: nginx + ports: + - "5001:80" + container_name: "nginx" + networks: + - monitor-net diff --git a/docs/Assignment.md b/docs/Assignment.md new file mode 100644 index 0000000..f05b69b --- /dev/null +++ b/docs/Assignment.md @@ -0,0 +1,69 @@ +# Alerting System + +## Requirements: +- Write your own code to create a Chat Bot that follow the description as below. +- You can use a code base from some open source project, but should show your ability +as a software engineer. + +## Specifications: + +### 1. Node Exporter: +- Running an exporter agent to collect service/server metrics and show via +HTTP/HTTPS endpoint. +Example: CPU, memory, network, disk utilization of server. + +### 2. Prometheus: +- Automatically scrape metrics from Node Exporter after an interval time. +- Set rules for alerting. +Example: Monitor memory usage of node, if node_memory_Active exceeds 80% +then send alert to Alertmanager. + +### 3. Alertmanager: +- Receiving signal from Prometheus and forward the alert to Telegram Bot. + +### 4. Levels of alerting: +- Define 3 levels of alert + + Level 1: owners of each service (or server) + + Level 2: Tech Manager + + Level 3: SRE +- When alert is sent, each level need to do response in 5 minutes by one of 2 +actions: + + Acknowledge + + Forward +- If there is no any response in 5 minutes, the alert will be sent to next levels. + +### 5. Database: +- Storing service/server name, owners with different levels, Telegram ID of each +person. + +### 6. Telegram Chat Group: +- A Telegram chat group that including all Tech owners. Everyone disable +notification of that group by default, but not tagging. + +### 7. Telegram Bot: +Description: Handle the alerts from Alertmanager and send to Telegram chat group. +- Receiving alert/resolved messages from AlertManager. +- Handling messages: + + The alert message will be send to Telegram Group with tag a TelegramID for level 1 recipients (ex: wukong - owner in below photo) and two buttons: *'Acknowledge'*, *'Forward'*. + + If anyone in chat group click to _'Acknowledge'_ button, then: + + ● Hide all buttons. + + ● Show username who did acknowledge the alert. + + ● Stop auto forwarding to next Level recipients. + + If anyone in chat group click to ‘Forward’ button, then: + + ● Show username that did forward the alert and username of Level 2 of recipients. + + ● Button ‘Forward’ will be hide. + + If no one action that message in 5 minutes, then: + + ● The message will be auto forward for next level (Level 2). + + ● When the highest level was forwarded, the button *‘Forward’* will be hide. + + If the bot received a resolved message, then: + + ● Hide all buttons of previous alert message. + + ● Stop auto forward to next Level of previous alert message. \ No newline at end of file diff --git a/docs/DatabaseDesign.md b/docs/DatabaseDesign.md new file mode 100644 index 0000000..321f5bf --- /dev/null +++ b/docs/DatabaseDesign.md @@ -0,0 +1,11 @@ +# Tables: +### Servers +- id +- name +- owner_id (foreign key) + +### Technicans +- id +- name +- level +- telegram_id \ No newline at end of file diff --git a/docs/Reference.md b/docs/Reference.md new file mode 100644 index 0000000..8622832 --- /dev/null +++ b/docs/Reference.md @@ -0,0 +1,8 @@ +### Prometheus +https://github.com/prometheus/alertmanager +https://rahulwa.com/post/monitoring-using-prometheus/ +https://github.com/kjanshair/docker-prometheus +### Telegram bot +https://github.com/tucnak/telebot +https://github.com/go-telegram-bot-api/telegram-bot-api +https://github.com/metalmatze/alertmanager-bot \ No newline at end of file diff --git a/docs/SystemDiagram.xml b/docs/SystemDiagram.xml new file mode 100644 index 0000000..802fd67 --- /dev/null +++ b/docs/SystemDiagram.xml @@ -0,0 +1 @@ +7Zndc5s4EMD/Gj8mwzfmMXbi9mYuN5lJp20eZZBBqUCMELHdv74SrPiMEy7nJj5P/ZCglbSstD9WuzCzl+nuE0d5cssiTGeWEe1m9vXMsnzXlX+VYF8LAieoBTEnUS0yW8E9+YlBaIC0JBEuegMFY1SQvC8MWZbhUPRkiHO27Q/bMNq/a45iPBLch4iOpd9IJBKQml7QdnzGJE7g1nPLrzvWKPwRc1ZmcL+ZZW+qX92dIq0LFlokKGLbjsi+mdlLzpior9LdElO1tXrb6nmrA72N3RxnYtIE3wFXPSFaYm10ZZrY6+3AkdwdaGYsk/8W1Rqx0mLI1mOZ5oMBiUipbJnyslmkGkrRGtNFs01LRhlvZxUCcXGlPDiQrQiloAFnkR4RUlQUJKyFMETd8RELsQesUCmYFDEuEhazDNG/Gcu1ZYKzH1jbIJ1lrdyFv2x6tPNtKdmwTKxQSqhi+jOmT1iQEEEH3Mv0oN1VaTuOe61U5igkWSylrjJ4R8R3MENdP6jVXVY9meD7781iZaPtG/sXXF6wkofgrv1ft/Ov1MAm8x6v6UNZZHf0wvTnwJzczhiLxv+AtPJxRyPg8gmzFEsL5ACOKRLkqf+EIHjQ4mZcC5u8AN4OsuedLHstWV38zLPBbxJILR19ZnzzA5nx/zDzf2TGMz6QmfnJMnPeZ1xzkrUH24NeIZx/0AUnoDkdM9ceY/bKyXc89mDqHSPSRsuAvFcmvvUUSHtt3+irqFcAswYEN2ZMhToYQf0FUxxzlErpgokR4i3Papu3CRH4XvpL9W5lGv8fgO4S1hP24JLsPofhW6FbVT8pR5TEmXpSJDCY64mwbnNEJeIhaPVlK8ecyH3H/L4ZZL0E4RPmAu9epAh6ncC/HPAwd7Vk2ykrXKAk6VQUQ3LeFvj0M9Jh5I4rRQkuixPnA89dz7Gm8uH8Gz76PAxpeS8+LMPo0WHpI/I1NryjsKG1dNi4RgKtUYFPnIyF5XmVY8+VjHHkcE1jauQ4Eh3OiI5/WCTJMG52ucwz5G6dNiQrY+4b7hlDMgwfphW8LyK6Cnw+AVkmSFTv91iZnzgqpoXQ+pxR8e0+KioPcYLO7/2wOZCh++Nk9k+F9oEVmn4rOa7QjAbJQTX1xpJNHzVHL8YcbwC97U0qxl5V1Ly814rq5RyjqjtUv44jbRHKOKmOY6WdhONkXoYG0X8AOC7IT7SuBigv5srgagnuYlaxMQpfKsRIvOgVdKQkitT8Q89P+7lDQzIKUs1HG7Bk1mxnF58Xw8TBOHdhXJrBPOh56wISmbfSpNWYA6ezzabAv8vf49T8isqNvEUZik8+8zK9IIgmx6qPOU6fwXIUvSZnXnbwO5Nz2Wy/B9aMtd9c7Ztf \ No newline at end of file diff --git a/docs/TelegramBotforAlerting.docx b/docs/TelegramBotforAlerting.docx new file mode 100644 index 0000000..4fbbe5c Binary files /dev/null and b/docs/TelegramBotforAlerting.docx differ diff --git a/go.mod b/go.mod index 5c0017a..a47e941 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/metalmatze/alertmanager-bot +module github.com/vu-long/alertmanager-bot require ( cloud.google.com/go v0.34.0 // indirect diff --git a/pkg/telegram/bot.go b/pkg/telegram/bot.go index e578f65..6a51a57 100644 --- a/pkg/telegram/bot.go +++ b/pkg/telegram/bot.go @@ -11,13 +11,13 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/hako/durafmt" - "github.com/metalmatze/alertmanager-bot/pkg/alertmanager" "github.com/oklog/run" "github.com/prometheus/alertmanager/notify" "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" "github.com/prometheus/client_golang/prometheus" "github.com/tucnak/telebot" + "github.com/vu-long/alertmanager-bot/pkg/alertmanager" ) const ( diff --git a/services/.gitattributes b/services/.gitattributes new file mode 100644 index 0000000..1ff0c42 --- /dev/null +++ b/services/.gitattributes @@ -0,0 +1,63 @@ +############################################################################### +# Set default behavior to automatically normalize line endings. +############################################################################### +* text=auto + +############################################################################### +# Set default behavior for command prompt diff. +# +# This is need for earlier builds of msysgit that does not have it on by +# default for csharp files. +# Note: This is only used by command line +############################################################################### +#*.cs diff=csharp + +############################################################################### +# Set the merge driver for project and solution files +# +# Merging from the command prompt will add diff markers to the files if there +# are conflicts (Merging from VS is not affected by the settings below, in VS +# the diff markers are never inserted). Diff markers may cause the following +# file extensions to fail to load in VS. An alternative would be to treat +# these files as binary and thus will always conflict and require user +# intervention with every merge. To do so, just uncomment the entries below +############################################################################### +#*.sln merge=binary +#*.csproj merge=binary +#*.vbproj merge=binary +#*.vcxproj merge=binary +#*.vcproj merge=binary +#*.dbproj merge=binary +#*.fsproj merge=binary +#*.lsproj merge=binary +#*.wixproj merge=binary +#*.modelproj merge=binary +#*.sqlproj merge=binary +#*.wwaproj merge=binary + +############################################################################### +# behavior for image files +# +# image files are treated as binary by default. +############################################################################### +#*.jpg binary +#*.png binary +#*.gif binary + +############################################################################### +# diff behavior for common document formats +# +# Convert binary document formats to text before diffing them. This feature +# is only available from the command line. Turn it on by uncommenting the +# entries below. +############################################################################### +#*.doc diff=astextplain +#*.DOC diff=astextplain +#*.docx diff=astextplain +#*.DOCX diff=astextplain +#*.dot diff=astextplain +#*.DOT diff=astextplain +#*.pdf diff=astextplain +#*.PDF diff=astextplain +#*.rtf diff=astextplain +#*.RTF diff=astextplain diff --git a/services/.gitignore b/services/.gitignore new file mode 100644 index 0000000..2f6da49 --- /dev/null +++ b/services/.gitignore @@ -0,0 +1,5 @@ +*DS_Store +.vs/ +.vscode/ +.idea/ +alertmanager/config-email.yml diff --git a/services/README.md b/services/README.md new file mode 100644 index 0000000..e539f10 --- /dev/null +++ b/services/README.md @@ -0,0 +1,11 @@ +## Prometheus Monitoring + +This repository contains minimal Prometheus Server, NodeExporter, BlackBoxExporter, AlertManager and Grafana implementation for monitoring various services. You can use this repository to monitor a bare-metal Linux instance or to monitor Apache, NGINX or other HTTP based services using Prometheus. + +## Monitoring a Bare-Metal Linux Server + +To monitor a stand-alone Linux Server, you have to checkout against the tag v1.0 of the repository. Where all the configurations for monitoring a stand-alone Linux Server are available. Just `docker-compose up -d` and you're good to go. (You have to map alerts manually against tag v1.0) + +## Monitoring HTTP-based Web Services + +The v1.1 tag of the repository monitors 2 HTTP-based Web Services by default: An Apache httpd server and NGINX server both running in Docker Containers. If either or both of them goes down, an Prometheus will fire alerts in the form emails specified in the `config.yml` file in the AlertManager folder. \ No newline at end of file diff --git a/services/alertmanager/config.yml b/services/alertmanager/config.yml new file mode 100644 index 0000000..f4b9df5 --- /dev/null +++ b/services/alertmanager/config.yml @@ -0,0 +1,30 @@ +route: + repeat_interval: 2h + receiver: email-1 + routes: + - match: + alertname: httpd_down + receiver: email-1 + + - match: + alertname: nginx_down + receiver: email-2 + +receivers: +- name: email-1 + email_configs: + - to: + from: + smarthost: + auth_username: "" + auth_identity: "" + auth_password: "" + +- name: email-2 + email_configs: + - to: + from: + smarthost: + auth_username: "" + auth_identity: "" + auth_password: "" diff --git a/services/alertmanager/data/nflog b/services/alertmanager/data/nflog new file mode 100644 index 0000000..e69de29 diff --git a/services/alertmanager/data/silences b/services/alertmanager/data/silences new file mode 100644 index 0000000..e69de29 diff --git a/services/alertmanager/webhookconfig.yml b/services/alertmanager/webhookconfig.yml new file mode 100644 index 0000000..51d8e5f --- /dev/null +++ b/services/alertmanager/webhookconfig.yml @@ -0,0 +1,18 @@ +route: + repeat_interval: 2h + receiver: group_1 + routes: + - match: + alertname: httpd_down + receiver: group_1 + + - match: + alertname: nginx_down + receiver: group_1 + +receivers: +- name: group_1 + webhook_configs: + - url: 'http://alertmanager-bot:8080' + send_resolved: true + diff --git a/services/docker-compose.yml b/services/docker-compose.yml new file mode 100644 index 0000000..6374404 --- /dev/null +++ b/services/docker-compose.yml @@ -0,0 +1,111 @@ +version: '3' + +networks: + monitor-net: + driver: bridge + +volumes: + prometheus_data: {} + grafana_data: {} + +services: + prometheus: + image: prom/prometheus:latest + container_name: prometheus + volumes: + - ./prometheus/:/etc/prometheus/ + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/etc/prometheus/console_libraries' + - '--web.console.templates=/etc/prometheus/consoles' + - '--storage.tsdb.retention=200h' + - '--web.enable-lifecycle' + restart: unless-stopped + expose: + - 9090 + ports: + - "9090:9090" + networks: + - monitor-net + + # nodeexporter: + # image: prom/node-exporter:latest + # container_name: nodeexporter + # user: root + # privileged: true + # volumes: + # - /proc:/host/proc:ro + # - /sys:/host/sys:ro + # - /:/rootfs:ro + # command: + # - '--path.procfs=/host/proc' + # - '--path.sysfs=/host/sys' + # - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)' + # restart: unless-stopped + # expose: + # - 9100 + # networks: + # - monitor-net + + alertmanager: + image: prom/alertmanager + container_name: alertmanager + volumes: + - ./alertmanager/:/etc/alertmanager/ + command: + - '--config.file=/etc/alertmanager/webhookconfig.yml' + - '--storage.path=/alertmanager' + - '--web.listen-address=0.0.0.0:9093' + restart: unless-stopped + ports: + - "9093:9093" + networks: + - monitor-net + + blackbox-exporter: + image: prom/blackbox-exporter + container_name: blackbox + restart: unless-stopped + ports: + - "9115:9115" + networks: + - monitor-net + + grafana: + image: grafana/grafana:latest + container_name: grafana + volumes: + - grafana_data:/var/lib/grafana + - ./grafana/datasources:/etc/grafana/datasources + - ./grafana/dashboards:/etc/grafana/dashboards + - ./grafana/setup.sh:/setup.sh + entrypoint: /setup.sh + environment: + - GF_SECURITY_ADMIN_USER=${ADMIN_USER:-admin} + - GF_SECURITY_ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin} + - GF_USERS_ALLOW_SIGN_UP=false + restart: unless-stopped + expose: + - 3000 + ports: + - 3000:3000 + networks: + - monitor-net + + httpd: + image: httpd + ports: + - "5000:80" + container_name: "httpd" + networks: + - monitor-net + + nginx: + image: nginx + ports: + - "5001:80" + container_name: "nginx" + networks: + - monitor-net diff --git a/services/grafana/dashboards/node-stat.json b/services/grafana/dashboards/node-stat.json new file mode 100644 index 0000000..f825052 --- /dev/null +++ b/services/grafana/dashboards/node-stat.json @@ -0,0 +1,511 @@ +{ + "__inputs": [ + { + "name": "Prometheus", + "label": "", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "3.1.1" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "id": null, + "title": "Host Stats - Prometheus Node Exporter", + "tags": [], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": false, + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 1, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg without (cpu)(irate(node_cpu{job=\"node\",instance=\"$instance\",mode!=\"idle\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{mode}}", + "metric": "node_cpu", + "refId": "A", + "step": 60 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "shared": true, + "value_type": "individual", + "sort": 0, + "msResolution": false + }, + "type": "graph", + "yaxes": [ + { + "show": true, + "min": null, + "max": 1, + "logBase": 1, + "format": "percentunit" + }, + { + "show": true, + "min": null, + "max": null, + "logBase": 1, + "format": "short" + } + ], + "xaxis": { + "show": true + } + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 2, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": null, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_MemTotal{job='node',instance='$instance'} - node_memory_MemFree{job='node',instance='$instance'} - node_memory_Buffers{job='node',instance='$instance'} - node_memory_Cached{job='node',instance='$instance'}", + "intervalFactor": 2, + "legendFormat": "Used", + "refId": "A", + "step": 60 + }, + { + "expr": "node_memory_Buffers{job='node',instance='$instance'}", + "intervalFactor": 2, + "legendFormat": "Buffers", + "refId": "B", + "step": 60 + }, + { + "expr": "node_memory_Cached{job='node',instance='$instance'}", + "intervalFactor": 2, + "legendFormat": "Cached", + "refId": "D", + "step": 60 + }, + { + "expr": "node_memory_MemFree{job='node',instance='$instance'}", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Free", + "refId": "C", + "step": 60 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "value_type": "individual", + "sort": 0, + "msResolution": false + }, + "type": "graph", + "yaxes": [ + { + "show": true, + "min": 0, + "max": null, + "logBase": 1, + "format": "bytes", + "label": "" + }, + { + "show": true, + "min": null, + "max": null, + "logBase": 1, + "format": "short" + } + ], + "xaxis": { + "show": true + } + } + ], + "title": "Row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 3, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_io_time_ms{job='node',instance='$instance',device!~'^(md\\\\d+$|dm-)'}[5m]) / 1000", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "refId": "A", + "step": 120 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O Utilisation", + "tooltip": { + "shared": true, + "value_type": "cumulative", + "sort": 0, + "msResolution": false + }, + "type": "graph", + "yaxes": [ + { + "show": true, + "min": null, + "max": 1, + "logBase": 1, + "format": "percentunit" + }, + { + "show": true, + "min": null, + "max": null, + "logBase": 1, + "format": "short" + } + ], + "xaxis": { + "show": true + } + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 4, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - node_filesystem_free{job='node',instance='$instance',fstype!='rootfs',mountpoint!~'/(run|var).*',mountpoint!=''} / node_filesystem_size{job='node',instance='$instance'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{mountpoint}}", + "refId": "A", + "step": 120 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Filesystem Fullness", + "tooltip": { + "shared": true, + "value_type": "cumulative", + "sort": 0, + "msResolution": false + }, + "type": "graph", + "yaxes": [ + { + "show": true, + "min": 0, + "max": 1, + "logBase": 1, + "format": "percentunit" + }, + { + "show": true, + "min": null, + "max": null, + "logBase": 1, + "format": "short" + } + ], + "xaxis": { + "show": true + } + }, + { + "columns": [ + { + "text": "Current", + "value": "current" + } + ], + "editable": true, + "error": false, + "fontSize": "100%", + "hideTimeOverride": true, + "id": 5, + "isNew": true, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "span": 4, + "styles": [ + { + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 0, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "s" + } + ], + "targets": [ + { + "expr": "(node_filesystem_size{job='node',instance='$instance'} - node_filesystem_free{job='node',instance='$instance'}) / deriv(node_filesystem_free{job='node',instance='$instance',fstype!='rootfs',mountpoint!~'/(run|var).*',mountpoint!=''}[3d]) > 0", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{mountpoint}}", + "refId": "A", + "step": 20 + } + ], + "timeFrom": "1h", + "timeShift": null, + "title": "Filesystem Fill Up Time", + "transform": "timeseries_aggregations", + "type": "table" + } + ], + "title": "New row" + } + ], + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "templating": { + "list": [ + { + "allFormat": "glob", + "current": {}, + "datasource": "prometheus", + "hideLabel": false, + "includeAll": false, + "label": "Machine", + "multi": false, + "multiFormat": "glob", + "name": "instance", + "options": [], + "query": "up{job=\"node\"}", + "refresh": 1, + "regex": ".*instance=\"(.*?)\".*", + "type": "query", + "hide": 0 + } + ] + }, + "annotations": { + "list": [] + }, + "schemaVersion": 12, + "version": 12, + "links": [], + "gnetId": 718, + "description": "Basic host stats: CPU, Memory Usage, Disk Utilisation, Filesystem usage and Predicted time to filesystems filling" +} diff --git a/services/grafana/datasources/Prometheus.json b/services/grafana/datasources/Prometheus.json new file mode 100644 index 0000000..1f19f1a --- /dev/null +++ b/services/grafana/datasources/Prometheus.json @@ -0,0 +1,7 @@ +{ + "name":"prometheus", + "type":"prometheus", + "url":"http://prometheus:9090", + "access":"proxy", + "basicAuth":false + } diff --git a/services/grafana/setup.sh b/services/grafana/setup.sh new file mode 100755 index 0000000..03abc0a --- /dev/null +++ b/services/grafana/setup.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# Taken from https://github.com/grafana/grafana-docker/issues/74 + +# Script to configure grafana datasources and dashboards. +# Intended to be run before grafana entrypoint... +# Image: grafana/grafana:4.1.2 +# ENTRYPOINT [\"/run.sh\"]" + +GRAFANA_URL=${GRAFANA_URL:-http://$GF_SECURITY_ADMIN_USER:$GF_SECURITY_ADMIN_PASSWORD@localhost:3000} +#GRAFANA_URL=http://grafana-plain.k8s.playground1.aws.ad.zopa.com +DATASOURCES_PATH=${DATASOURCES_PATH:-/etc/grafana/datasources} +DASHBOARDS_PATH=${DASHBOARDS_PATH:-/etc/grafana/dashboards} + +# Generic function to call the Vault API +grafana_api() { + local verb=$1 + local url=$2 + local params=$3 + local bodyfile=$4 + local response + local cmd + + cmd="curl -L -s --fail -H \"Accept: application/json\" -H \"Content-Type: application/json\" -X ${verb} -k ${GRAFANA_URL}${url}" + [[ -n "${params}" ]] && cmd="${cmd} -d \"${params}\"" + [[ -n "${bodyfile}" ]] && cmd="${cmd} --data @${bodyfile}" + echo "Running ${cmd}" + eval ${cmd} || return 1 + return 0 +} + +wait_for_api() { + while ! grafana_api GET /api/user/preferences + do + sleep 5 + done +} + +install_datasources() { + local datasource + + for datasource in ${DATASOURCES_PATH}/*.json + do + if [[ -f "${datasource}" ]]; then + echo "Installing datasource ${datasource}" + if grafana_api POST /api/datasources "" "${datasource}"; then + echo "installed ok" + else + echo "install failed" + fi + fi + done +} + +install_dashboards() { + local dashboard + + for dashboard in ${DASHBOARDS_PATH}/*.json + do + if [[ -f "${dashboard}" ]]; then + echo "Installing dashboard ${dashboard}" + + echo "{\"dashboard\": `cat $dashboard`}" > "${dashboard}.wrapped" + + if grafana_api POST /api/dashboards/db "" "${dashboard}.wrapped"; then + echo "installed ok" + else + echo "install failed" + fi + + rm "${dashboard}.wrapped" + fi + done +} + +configure_grafana() { + wait_for_api + install_datasources + install_dashboards +} + +echo "Running configure_grafana in the background..." +configure_grafana & +/run.sh +exit 0 \ No newline at end of file diff --git a/services/prometheus/alert.rules b/services/prometheus/alert.rules new file mode 100644 index 0000000..025c1d3 --- /dev/null +++ b/services/prometheus/alert.rules @@ -0,0 +1,21 @@ +groups: + +- name: httpd + rules: + - alert: httpd_down + expr: probe_success{instance="http://httpd:80",job="httpd"} == 0 + for: 1s + labels: + severity: critical + annotations: + summary: "httpd is down" + +- name: nginx + rules: + - alert: nginx_down + expr: probe_success{instance="http://nginx:80",job="nginx"} == 0 + for: 1s + labels: + severity: warning + annotations: + summary: "nginx is down" diff --git a/services/prometheus/prometheus.yml b/services/prometheus/prometheus.yml new file mode 100644 index 0000000..ada5d71 --- /dev/null +++ b/services/prometheus/prometheus.yml @@ -0,0 +1,57 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + + external_labels: + monitor: 'prometheus-grafana-exporter' + +# Alertmanager configuration +alerting: + alertmanagers: + - static_configs: + - targets: + - alertmanager:9093 + +rule_files: + - "alert.rules" + +scrape_configs: + # - job_name: "kjanshair" + # scrape_interval: 5s + # static_configs: + # - targets: ['nodeexporter:9100'] + + - job_name: 'prometheus' + scrape_interval: 10s + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'httpd' + metrics_path: /probe + params: + module: [http_2xx] + static_configs: + - targets: + - http://httpd:80 + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox:9115 + + - job_name: 'nginx' + metrics_path: /probe + params: + module: [http_2xx] + static_configs: + - targets: + - http://nginx:80 + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox:9115