diff --git a/Dockerfile b/Dockerfile
index 841d1c9..f512195 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,9 +1,21 @@
-FROM alpine:latest
-RUN apk add --update ca-certificates
+FROM golang:latest
COPY ./default.tmpl /templates/default.tmpl
-COPY ./alertmanager-bot /usr/bin/alertmanager-bot
+# Set go bin which doesn't appear to be set already.
+ENV GOBIN /go/bin
-EXPOSE 8080
+# build directories
+RUN mkdir /app
+RUN mkdir -p /go/src/github.com/vu-long/alertmanager-bot
+ADD . /go/src/github.com/vu-long/alertmanager-bot
+WORKDIR /go/src/github.com/vu-long/alertmanager-bot
-ENTRYPOINT ["/usr/bin/alertmanager-bot"]
+# Go dep!
+RUN go get -u github.com/golang/dep/...
+RUN dep ensure -v -vendor-only
+
+RUN make build
+
+EXPOSE 8080:8080
+
+ENTRYPOINT ["/go/src/github.com/vu-long/alertmanager-bot/alertmanager-bot"]
diff --git a/Gopkg.lock b/Gopkg.lock
new file mode 100644
index 0000000..73803f1
--- /dev/null
+++ b/Gopkg.lock
@@ -0,0 +1,498 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+ branch = "master"
+ digest = "1:315c5f2f60c76d89b871c73f9bd5fe689cad96597afd50fb9992228ef80bdd34"
+ name = "github.com/alecthomas/template"
+ packages = [
+ ".",
+ "parse",
+ ]
+ pruneopts = "UT"
+ revision = "a0175ee3bccc567396460bf5acd36800cb10c49c"
+
+[[projects]]
+ branch = "master"
+ digest = "1:c198fdc381e898e8fb62b8eb62758195091c313ad18e52a3067366e1dda2fb3c"
+ name = "github.com/alecthomas/units"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "2efee857e7cfd4f3d0138cc3cbb1b4966962b93a"
+
+[[projects]]
+ branch = "master"
+ digest = "1:ef5b0622d834c139454148b8fd0c92bb314828900532b267ae62da9fec109866"
+ name = "github.com/armon/go-metrics"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "f0300d1749da6fa982027e449ec0c7a145510c3c"
+
+[[projects]]
+ digest = "1:c47f4964978e211c6e566596ec6246c329912ea92e9bb99c00798bb4564c5b09"
+ name = "github.com/armon/go-radix"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "1a2de0c21c94309923825da3df33a4381872c795"
+ version = "v1.0.0"
+
+[[projects]]
+ branch = "master"
+ digest = "1:d6afaeed1502aa28e80a4ed0981d570ad91b2579193404256ce672ed0a609e0d"
+ name = "github.com/beorn7/perks"
+ packages = ["quantile"]
+ pruneopts = "UT"
+ revision = "3a771d992973f24aa725d07868b467d1ddfceafb"
+
+[[projects]]
+ digest = "1:0f98f59e9a2f4070d66f0c9c39561f68fcd1dc837b22a852d28d0003aebd1b1e"
+ name = "github.com/boltdb/bolt"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8"
+ version = "v1.3.1"
+
+[[projects]]
+ digest = "1:166438587ed45ac211dab8a3ecebf4fa0c186d0db63430fb9127bbc2e5fcdc67"
+ name = "github.com/cenkalti/backoff"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "1e4cf3da559842a91afcb6ea6141451e6c30c618"
+ version = "v2.1.1"
+
+[[projects]]
+ digest = "1:998cf998358a303ac2430c386ba3fd3398477d6013153d3c6e11432765cc9ae6"
+ name = "github.com/cespare/xxhash"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "3b82fb7d186719faeedd0c2864f868c74fbf79a1"
+ version = "v2.0.0"
+
+[[projects]]
+ digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec"
+ name = "github.com/davecgh/go-spew"
+ packages = ["spew"]
+ pruneopts = "UT"
+ revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73"
+ version = "v1.1.1"
+
+[[projects]]
+ digest = "1:1cf8b80d775788921f6e309a8d7da4a925780e55e887feca461cda04b8ceea45"
+ name = "github.com/docker/libkv"
+ packages = [
+ ".",
+ "store",
+ "store/boltdb",
+ "store/consul",
+ ]
+ pruneopts = "UT"
+ revision = "aabc039ad04deb721e234f99cd1b4aa28ac71a40"
+ version = "v0.2.1"
+
+[[projects]]
+ digest = "1:0bec1c27a357178fe656168f96f7b89f338d73815ddca2af3a2eeb39f70c837f"
+ name = "github.com/go-kit/kit"
+ packages = [
+ "log",
+ "log/level",
+ ]
+ pruneopts = "UT"
+ revision = "e2b298466b32c7cd5579a9b9b07e968fc9d9452c"
+
+[[projects]]
+ digest = "1:4062bc6de62d73e2be342243cf138cf499b34d558876db8d9430e2149388a4d8"
+ name = "github.com/go-logfmt/logfmt"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "07c9b44f60d7ffdfb7d8efe1ad539965737836dc"
+ version = "v0.4.0"
+
+[[projects]]
+ digest = "1:586ea76dbd0374d6fb649a91d70d652b7fe0ccffb8910a77468e7702e7901f3d"
+ name = "github.com/go-stack/stack"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "2fee6af1a9795aafbe0253a0cfbdf668e1fb8a9a"
+ version = "v1.8.0"
+
+[[projects]]
+ digest = "1:a1be218880631ef6719b9b565ffd744cf67907f2a6c19bba86f8f1dc66d9a62e"
+ name = "github.com/gogo/protobuf"
+ packages = [
+ "gogoproto",
+ "proto",
+ "protoc-gen-gogo/descriptor",
+ "sortkeys",
+ "types",
+ ]
+ pruneopts = "UT"
+ revision = "616a82ed12d78d24d4839363e8f3c5d3f20627cf"
+
+[[projects]]
+ digest = "1:97df918963298c287643883209a2c3f642e6593379f97ab400c2a2e219ab647d"
+ name = "github.com/golang/protobuf"
+ packages = ["proto"]
+ pruneopts = "UT"
+ revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5"
+ version = "v1.2.0"
+
+[[projects]]
+ digest = "1:09970bb8daf100bdf11b48b72c71ef980cd2b50efbd903989b3d182956be0b83"
+ name = "github.com/hako/durafmt"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "7b7ae1e72eade09dbc9c2cfba3e6c4bae7b8bcac"
+ version = "1.0.0"
+
+[[projects]]
+ digest = "1:bfc483a051d3c7185ebeaa41b5bb67a4f76e742217bcaeab5661cc4b1320f392"
+ name = "github.com/hashicorp/consul"
+ packages = ["api"]
+ pruneopts = "UT"
+ revision = "0bddfa23a2ebe3c0773d917fc104f53d74f7a5ec"
+ version = "v1.4.0"
+
+[[projects]]
+ digest = "1:0ade334594e69404d80d9d323445d2297ff8161637f9b2d347cc6973d2d6f05b"
+ name = "github.com/hashicorp/errwrap"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "8a6fb523712970c966eefc6b39ed2c5e74880354"
+ version = "v1.0.0"
+
+[[projects]]
+ digest = "1:77cb3be9b21ba7f1a4701e870c84ea8b66e7d74c7c8951c58155fdadae9414ec"
+ name = "github.com/hashicorp/go-cleanhttp"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "d5fe4b57a186c716b0e00b8c301cbd9b4182694d"
+
+[[projects]]
+ digest = "1:2be5a35f0c5b35162c41bb24971e5dcf6ce825403296ee435429cdcc4e1e847e"
+ name = "github.com/hashicorp/go-immutable-radix"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "27df80928bb34bb1b0d6d0e01b9e679902e7a6b5"
+ version = "v1.0.0"
+
+[[projects]]
+ branch = "master"
+ digest = "1:b8ba23b0b493e601d5ebaf21079bd3433eb304a5d73c0fb40b8aea526e94f46b"
+ name = "github.com/hashicorp/go-msgpack"
+ packages = ["codec"]
+ pruneopts = "UT"
+ revision = "fa3f63826f7c23912c15263591e65d54d080b458"
+
+[[projects]]
+ digest = "1:f668349b83f7d779567c880550534addeca7ebadfdcf44b0b9c39be61864b4b7"
+ name = "github.com/hashicorp/go-multierror"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "886a7fbe3eb1c874d46f623bfa70af45f425b3d1"
+ version = "v1.0.0"
+
+[[projects]]
+ branch = "master"
+ digest = "1:45aad874d3c7d5e8610427c81870fb54970b981692930ec2a319ce4cb89d7a00"
+ name = "github.com/hashicorp/go-rootcerts"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "6bb64b370b90e7ef1fa532be9e591a81c3493e00"
+
+[[projects]]
+ digest = "1:ba05d01857d121ba6ab1613ec7a153de2aaf8598224dda71e669a01733d08e92"
+ name = "github.com/hashicorp/go-sockaddr"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "9b4c5fa5b10a683339a270d664474b9f4aee62fc"
+
+[[projects]]
+ digest = "1:67474f760e9ac3799f740db2c489e6423a4cde45520673ec123ac831ad849cb8"
+ name = "github.com/hashicorp/golang-lru"
+ packages = ["simplelru"]
+ pruneopts = "UT"
+ revision = "20f1fb78b0740ba8c3cb143a61e86ba5c8669768"
+ version = "v0.5.0"
+
+[[projects]]
+ digest = "1:83264bfcb4a76fe13c71fd2c36fc7c4cef3284f6ba5d81d291e036e93d7d10c7"
+ name = "github.com/hashicorp/memberlist"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "687988a0b5daaf7ed5051e5e374aef27f8254822"
+
+[[projects]]
+ digest = "1:acc81e4e4289587b257ccdfccbc6eaf16d4c2fb57dda73c6bb349bf50f02501f"
+ name = "github.com/hashicorp/serf"
+ packages = ["coordinate"]
+ pruneopts = "UT"
+ revision = "19bbd39e421bdf3559d5025fb2c760f5ffa56233"
+
+[[projects]]
+ digest = "1:ecd9aa82687cf31d1585d4ac61d0ba180e42e8a6182b85bd785fcca8dfeefc1b"
+ name = "github.com/joho/godotenv"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "23d116af351c84513e1946b527c88823e476be13"
+ version = "v1.3.0"
+
+[[projects]]
+ branch = "master"
+ digest = "1:a64e323dc06b73892e5bb5d040ced475c4645d456038333883f58934abbf6f72"
+ name = "github.com/kr/logfmt"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0"
+
+[[projects]]
+ digest = "1:ff5ebae34cfbf047d505ee150de27e60570e8c394b3b8fdbb720ff6ac71985fc"
+ name = "github.com/matttproud/golang_protobuf_extensions"
+ packages = ["pbutil"]
+ pruneopts = "UT"
+ revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c"
+ version = "v1.0.1"
+
+[[projects]]
+ digest = "1:478b8f5ee3eac843d206afd9c2c2a7133d0b7da7a65e8f9edbb1ebbf6758bdb0"
+ name = "github.com/miekg/dns"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "44a8c5f8ba7d0b3eac116182736dcfb5885ee405"
+ version = "v1.1.2"
+
+[[projects]]
+ digest = "1:78bbb1ba5b7c3f2ed0ea1eab57bdd3859aec7e177811563edc41198a760b06af"
+ name = "github.com/mitchellh/go-homedir"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "ae18d6b8b3205b561c79e8e5f69bff09736185f4"
+ version = "v1.0.0"
+
+[[projects]]
+ digest = "1:61332bb44d05257bbf0356d8400a8b30fe0b9fdc3b72b8b55661da8f0a4f39ae"
+ name = "github.com/mitchellh/hashstructure"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "a38c50148365edc8df43c1580c48fb2b3a1e9cd7"
+ version = "v1.0.0"
+
+[[projects]]
+ digest = "1:bba69fe95a60f59a68df402c23d57ffdc5596200901189958aea969c468d550f"
+ name = "github.com/mitchellh/mapstructure"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "5a380f224700b8a6c4eaad048804f5bff514cb35"
+
+[[projects]]
+ branch = "master"
+ digest = "1:9f07f801988b225662081432361c430cad8f5293b134e80bdf1998d14969d7a6"
+ name = "github.com/mwitkow/go-conntrack"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "cc309e4a22231782e8893f3c35ced0967807a33e"
+
+[[projects]]
+ digest = "1:9ec6cf1df5ad1d55cf41a43b6b1e7e118a91bade4f68ff4303379343e40c0e25"
+ name = "github.com/oklog/run"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "4dadeb3030eda0273a12382bb2348ffc7c9d1a39"
+ version = "v1.0.0"
+
+[[projects]]
+ digest = "1:15ddf3998dbeca86ffee1a3774353ef6e5868ff2e3898ab9ab1438b83079f9c9"
+ name = "github.com/oklog/ulid"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "66bb6560562feca7045b23db1ae85b01260f87c5"
+
+[[projects]]
+ digest = "1:cf31692c14422fa27c83a05292eb5cbe0fb2775972e8f1f8446a71549bd8980b"
+ name = "github.com/pkg/errors"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4"
+ version = "v0.8.1"
+
+[[projects]]
+ digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe"
+ name = "github.com/pmezard/go-difflib"
+ packages = ["difflib"]
+ pruneopts = "UT"
+ revision = "792786c7400a136282c1664665ae0a8db921c6c2"
+ version = "v1.0.0"
+
+[[projects]]
+ digest = "1:5e278dfd778c44ca6b856e843a48449151fe531c989a8c9b983de715a1d42726"
+ name = "github.com/prometheus/alertmanager"
+ packages = [
+ "cluster",
+ "cluster/clusterpb",
+ "config",
+ "nflog",
+ "nflog/nflogpb",
+ "notify",
+ "silence",
+ "silence/silencepb",
+ "template",
+ "template/internal/deftmpl",
+ "types",
+ ]
+ pruneopts = "UT"
+ revision = "d4a7697cc90f8bce62efe7c44b63b542578ec0a1"
+ version = "v0.15.3"
+
+[[projects]]
+ digest = "1:9e6e005a467ff9b1e4bb32e4c64ef36f6ab9c3313335acc66b83eaf9570d1d95"
+ name = "github.com/prometheus/client_golang"
+ packages = [
+ "prometheus",
+ "prometheus/promhttp",
+ ]
+ pruneopts = "UT"
+ revision = "c3324c1198cf3374996e9d3098edd46a6b55afc9"
+
+[[projects]]
+ branch = "master"
+ digest = "1:2d5cd61daa5565187e1d96bae64dbbc6080dacf741448e9629c64fd93203b0d4"
+ name = "github.com/prometheus/client_model"
+ packages = ["go"]
+ pruneopts = "UT"
+ revision = "5c3871d89910bfb32f5fcab2aa4b9ec68e65a99f"
+
+[[projects]]
+ digest = "1:f5230e376231e489365dd86ceea2e5205a42f5ed4ceaab13c4e7f9eb75ceb666"
+ name = "github.com/prometheus/common"
+ packages = [
+ "config",
+ "expfmt",
+ "internal/bitbucket.org/ww/goautoneg",
+ "model",
+ "version",
+ ]
+ pruneopts = "UT"
+ revision = "7600349dcfe1abd18d72d3a1770870d9800a7801"
+
+[[projects]]
+ branch = "master"
+ digest = "1:08eb8b60450efe841e37512d66ce366a87d187505d7c67b99307a6c1803483a2"
+ name = "github.com/prometheus/procfs"
+ packages = [
+ ".",
+ "internal/util",
+ "nfs",
+ "xfs",
+ ]
+ pruneopts = "UT"
+ revision = "b1a0a9a36d7453ba0f62578b99712f3a6c5f82d1"
+
+[[projects]]
+ digest = "1:274f67cb6fed9588ea2521ecdac05a6d62a8c51c074c1fccc6a49a40ba80e925"
+ name = "github.com/satori/go.uuid"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "f58768cc1a7a7e77a3bd49e98cdd21419399b6a3"
+ version = "v1.2.0"
+
+[[projects]]
+ branch = "master"
+ digest = "1:579c4bbcc2e16d4caf871ba91c0e2c331b07c5560c80d142d82c0de01c57fa96"
+ name = "github.com/sean-/seed"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "e2103e2c35297fb7e17febb81e49b312087a2372"
+
+[[projects]]
+ digest = "1:972c2427413d41a1e06ca4897e8528e5a1622894050e2f527b38ddf0f343f759"
+ name = "github.com/stretchr/testify"
+ packages = ["assert"]
+ pruneopts = "UT"
+ revision = "ffdc059bfe9ce6a4e144ba849dbedead332c6053"
+ version = "v1.3.0"
+
+[[projects]]
+ digest = "1:6a519ce8a4f7e5da2e12c7b47da4711750c085d8e3a6449d046b17eea42069a9"
+ name = "github.com/tucnak/telebot"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "00cebf376d79ee4363a11cd66fbbdf565452e0ab"
+ version = "v1.0"
+
+[[projects]]
+ branch = "master"
+ digest = "1:d5891c5bca9c62e5d394ca26491d2b710a1dc08cedeb0ca8f9ac4c3305120b02"
+ name = "golang.org/x/crypto"
+ packages = [
+ "ed25519",
+ "ed25519/internal/edwards25519",
+ ]
+ pruneopts = "UT"
+ revision = "ff983b9c42bc9fbf91556e191cc8efb585c16908"
+
+[[projects]]
+ branch = "master"
+ digest = "1:cc69b66377d151326324dd9225f034c9d4ba002d8424dbc2d1d82c950dbaa89b"
+ name = "golang.org/x/net"
+ packages = [
+ "bpf",
+ "context",
+ "context/ctxhttp",
+ "internal/iana",
+ "internal/socket",
+ "internal/timeseries",
+ "ipv4",
+ "ipv6",
+ "trace",
+ ]
+ pruneopts = "UT"
+ revision = "1e06a53dbb7e2ed46e91183f219db23c6943c532"
+
+[[projects]]
+ branch = "master"
+ digest = "1:f106a08663d0031b576f61ab81ffd80124da17096eab36aceb25b1b0d847da9a"
+ name = "golang.org/x/sys"
+ packages = ["unix"]
+ pruneopts = "UT"
+ revision = "7fbe1cd0fcc20051e1fcb87fbabec4a1bacaaeba"
+
+[[projects]]
+ digest = "1:c06d9e11d955af78ac3bbb26bd02e01d2f61f689e1a3bce2ef6fb683ef8a7f2d"
+ name = "gopkg.in/alecthomas/kingpin.v2"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "947dcec5ba9c011838740e680966fd7087a71d0d"
+ version = "v2.2.6"
+
+[[projects]]
+ digest = "1:342378ac4dcb378a5448dd723f0784ae519383532f5e70ade24132c4c8693202"
+ name = "gopkg.in/yaml.v2"
+ packages = ["."]
+ pruneopts = "UT"
+ revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
+ version = "v2.2.1"
+
+[solve-meta]
+ analyzer-name = "dep"
+ analyzer-version = 1
+ input-imports = [
+ "github.com/cenkalti/backoff",
+ "github.com/docker/libkv/store",
+ "github.com/docker/libkv/store/boltdb",
+ "github.com/docker/libkv/store/consul",
+ "github.com/go-kit/kit/log",
+ "github.com/go-kit/kit/log/level",
+ "github.com/hako/durafmt",
+ "github.com/joho/godotenv",
+ "github.com/oklog/run",
+ "github.com/pkg/errors",
+ "github.com/prometheus/alertmanager/notify",
+ "github.com/prometheus/alertmanager/template",
+ "github.com/prometheus/alertmanager/types",
+ "github.com/prometheus/client_golang/prometheus",
+ "github.com/prometheus/client_golang/prometheus/promhttp",
+ "github.com/stretchr/testify/assert",
+ "github.com/tucnak/telebot",
+ "gopkg.in/alecthomas/kingpin.v2",
+ ]
+ solver-name = "gps-cdcl"
+ solver-version = 1
diff --git a/Gopkg.toml b/Gopkg.toml
new file mode 100644
index 0000000..17004f0
--- /dev/null
+++ b/Gopkg.toml
@@ -0,0 +1,70 @@
+# Gopkg.toml example
+#
+# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+# name = "github.com/user/project"
+# version = "1.0.0"
+#
+# [[constraint]]
+# name = "github.com/user/project2"
+# branch = "dev"
+# source = "github.com/myfork/project2"
+#
+# [[override]]
+# name = "github.com/x/y"
+# version = "2.4.0"
+#
+# [prune]
+# non-go = false
+# go-tests = true
+# unused-packages = true
+
+
+[[constraint]]
+ name = "github.com/cenkalti/backoff"
+ version = "2.1.1"
+
+[[constraint]]
+ name = "github.com/docker/libkv"
+ version = "0.2.1"
+
+[[constraint]]
+ name = "github.com/hako/durafmt"
+ version = "1.0.0"
+
+[[constraint]]
+ name = "github.com/joho/godotenv"
+ version = "1.3.0"
+
+[[constraint]]
+ name = "github.com/oklog/run"
+ version = "1.0.0"
+
+[[constraint]]
+ name = "github.com/pkg/errors"
+ version = "0.8.1"
+
+[[constraint]]
+ name = "github.com/prometheus/alertmanager"
+ version = "0.15.3"
+
+[[constraint]]
+ name = "github.com/stretchr/testify"
+ version = "1.3.0"
+
+[[constraint]]
+ name = "github.com/tucnak/telebot"
+ version = "1.0.0"
+
+[[constraint]]
+ name = "gopkg.in/alecthomas/kingpin.v2"
+ version = "2.2.6"
+
+[prune]
+ go-tests = true
+ unused-packages = true
diff --git a/Makefile b/Makefile
index 564b631..319a31f 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
EXECUTABLE ?= alertmanager-bot
-IMAGE ?= metalmatze/$(EXECUTABLE)
+IMAGE ?= vulong/$(EXECUTABLE)
GO := CGO_ENABLED=0 go
DATE := $(shell date -u '+%FT%T%z')
diff --git a/cmd/alertmanager-bot/main.go b/cmd/alertmanager-bot/main.go
index 83728aa..b535b0d 100644
--- a/cmd/alertmanager-bot/main.go
+++ b/cmd/alertmanager-bot/main.go
@@ -18,14 +18,14 @@ import (
"github.com/go-kit/kit/log/level"
"github.com/hako/durafmt"
"github.com/joho/godotenv"
- "github.com/metalmatze/alertmanager-bot/pkg/alertmanager"
- "github.com/metalmatze/alertmanager-bot/pkg/telegram"
"github.com/oklog/run"
"github.com/prometheus/alertmanager/notify"
"github.com/prometheus/alertmanager/template"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
- "gopkg.in/alecthomas/kingpin.v2"
+ "github.com/vu-long/alertmanager-bot/pkg/alertmanager"
+ "github.com/vu-long/alertmanager-bot/pkg/telegram"
+ kingpin "gopkg.in/alecthomas/kingpin.v2"
)
const (
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000..4499932
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,144 @@
+version: "3"
+
+networks:
+ monitor-net:
+ driver: bridge
+
+volumes:
+ prometheus_data: {}
+ grafana_data: {}
+
+services:
+ # alertmanager-bot:
+ # build:
+ # context: .
+ # dockerfile: Dockerfile
+ # container_name: alertmanager-bot
+ # # image: alertmanager-bot:latest
+ # environment:
+ # ALERTMANAGER_URL: http://alertmanager:9093
+ # BOLT_PATH: /data/bot.db
+ # CONSUL_URL: consul-agent:8500
+ # LISTEN_ADDR: 0.0.0.0:8080
+ # STORE: consul
+ # TELEGRAM_ADMIN: 789593887
+ # TELEGRAM_TOKEN: 715767920:AAHM_dFYTF8Q4FG3gnQjvljKCJx4n1dCWuI
+ # TEMPLATE_PATHS: /templates/default.tmpl
+ # ports:
+ # - 8080:8080
+ # restart: unless-stopped
+ # networks:
+ # - monitor-net
+ # # volumes:
+ # # - /srv/monitoring/alertmanager-bot:/data
+
+ # consul-agent:
+ # image: consul:latest
+ # container_name: consul-agent
+ # restart: unless-stopped
+ # networks:
+ # - monitor-net
+ # command: "agent -dev"
+ # ports:
+ # - 8500:8500
+
+ prometheus:
+ image: prom/prometheus:latest
+ container_name: prometheus
+ volumes:
+ - ./services/prometheus/:/etc/prometheus/
+ - prometheus_data:/prometheus
+ command:
+ - '--config.file=/etc/prometheus/prometheus.yml'
+ - '--storage.tsdb.path=/prometheus'
+ - '--web.console.libraries=/etc/prometheus/console_libraries'
+ - '--web.console.templates=/etc/prometheus/consoles'
+ - '--storage.tsdb.retention=200h'
+ - '--web.enable-lifecycle'
+ restart: unless-stopped
+ expose:
+ - 9090
+ ports:
+ - "9090:9090"
+ networks:
+ - monitor-net
+
+ # nodeexporter:
+ # image: prom/node-exporter:latest
+ # container_name: nodeexporter
+ # user: root
+ # privileged: true
+ # volumes:
+ # - /proc:/host/proc:ro
+ # - /sys:/host/sys:ro
+ # - /:/rootfs:ro
+ # command:
+ # - '--path.procfs=/host/proc'
+ # - '--path.sysfs=/host/sys'
+ # - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)'
+ # restart: unless-stopped
+ # expose:
+ # - 9100
+ # networks:
+ # - monitor-net
+
+ alertmanager:
+ image: prom/alertmanager
+ container_name: alertmanager
+ volumes:
+ - ./services/alertmanager/:/etc/alertmanager/
+ command:
+ - '--config.file=/etc/alertmanager/webhookconfig.yml'
+ - '--storage.path=/alertmanager'
+ - '--web.listen-address=0.0.0.0:9093'
+ restart: unless-stopped
+ ports:
+ - "9093:9093"
+ networks:
+ - monitor-net
+
+ blackbox-exporter:
+ image: prom/blackbox-exporter
+ container_name: blackbox
+ restart: unless-stopped
+ ports:
+ - "9115:9115"
+ networks:
+ - monitor-net
+
+ grafana:
+ image: grafana/grafana:latest
+ container_name: grafana
+ volumes:
+ - grafana_data:/var/lib/grafana
+ - ./services/grafana/datasources:/etc/grafana/datasources
+ - ./services/grafana/dashboards:/etc/grafana/dashboards
+ - ./services/grafana/setup.sh:/setup.sh
+ entrypoint: /setup.sh
+ environment:
+ - GF_SECURITY_ADMIN_USER=${ADMIN_USER:-admin}
+ - GF_SECURITY_ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin}
+ - GF_USERS_ALLOW_SIGN_UP=false
+ restart: unless-stopped
+ expose:
+ - 3000
+ ports:
+ - 3000:3000
+ networks:
+ - monitor-net
+
+ httpd:
+ image: httpd
+ ports:
+ - "5000:80"
+ container_name: "httpd"
+ networks:
+ - monitor-net
+
+ nginx:
+ image: nginx
+ ports:
+ - "5001:80"
+ container_name: "nginx"
+ networks:
+ - monitor-net
diff --git a/docs/Assignment.md b/docs/Assignment.md
new file mode 100644
index 0000000..f05b69b
--- /dev/null
+++ b/docs/Assignment.md
@@ -0,0 +1,69 @@
+# Alerting System
+
+## Requirements:
+- Write your own code to create a Chat Bot that follow the description as below.
+- You can use a code base from some open source project, but should show your ability
+as a software engineer.
+
+## Specifications:
+
+### 1. Node Exporter:
+- Running an exporter agent to collect service/server metrics and show via
+HTTP/HTTPS endpoint.
+Example: CPU, memory, network, disk utilization of server.
+
+### 2. Prometheus:
+- Automatically scrape metrics from Node Exporter after an interval time.
+- Set rules for alerting.
+Example: Monitor memory usage of node, if node_memory_Active exceeds 80%
+then send alert to Alertmanager.
+
+### 3. Alertmanager:
+- Receiving signal from Prometheus and forward the alert to Telegram Bot.
+
+### 4. Levels of alerting:
+- Define 3 levels of alert
+ + Level 1: owners of each service (or server)
+ + Level 2: Tech Manager
+ + Level 3: SRE
+- When alert is sent, each level need to do response in 5 minutes by one of 2
+actions:
+ + Acknowledge
+ + Forward
+- If there is no any response in 5 minutes, the alert will be sent to next levels.
+
+### 5. Database:
+- Storing service/server name, owners with different levels, Telegram ID of each
+person.
+
+### 6. Telegram Chat Group:
+- A Telegram chat group that including all Tech owners. Everyone disable
+notification of that group by default, but not tagging.
+
+### 7. Telegram Bot:
+Description: Handle the alerts from Alertmanager and send to Telegram chat group.
+- Receiving alert/resolved messages from AlertManager.
+- Handling messages:
+ + The alert message will be send to Telegram Group with tag a TelegramID for level 1 recipients (ex: wukong - owner in below photo) and two buttons: *'Acknowledge'*, *'Forward'*.
+ + If anyone in chat group click to _'Acknowledge'_ button, then:
+
+ ● Hide all buttons.
+
+ ● Show username who did acknowledge the alert.
+
+ ● Stop auto forwarding to next Level recipients.
+ + If anyone in chat group click to ‘Forward’ button, then:
+
+ ● Show username that did forward the alert and username of Level 2 of recipients.
+
+ ● Button ‘Forward’ will be hide.
+ + If no one action that message in 5 minutes, then:
+
+ ● The message will be auto forward for next level (Level 2).
+
+ ● When the highest level was forwarded, the button *‘Forward’* will be hide.
+ + If the bot received a resolved message, then:
+
+ ● Hide all buttons of previous alert message.
+
+ ● Stop auto forward to next Level of previous alert message.
\ No newline at end of file
diff --git a/docs/DatabaseDesign.md b/docs/DatabaseDesign.md
new file mode 100644
index 0000000..321f5bf
--- /dev/null
+++ b/docs/DatabaseDesign.md
@@ -0,0 +1,11 @@
+# Tables:
+### Servers
+- id
+- name
+- owner_id (foreign key)
+
+### Technicans
+- id
+- name
+- level
+- telegram_id
\ No newline at end of file
diff --git a/docs/Reference.md b/docs/Reference.md
new file mode 100644
index 0000000..8622832
--- /dev/null
+++ b/docs/Reference.md
@@ -0,0 +1,8 @@
+### Prometheus
+https://github.com/prometheus/alertmanager
+https://rahulwa.com/post/monitoring-using-prometheus/
+https://github.com/kjanshair/docker-prometheus
+### Telegram bot
+https://github.com/tucnak/telebot
+https://github.com/go-telegram-bot-api/telegram-bot-api
+https://github.com/metalmatze/alertmanager-bot
\ No newline at end of file
diff --git a/docs/SystemDiagram.xml b/docs/SystemDiagram.xml
new file mode 100644
index 0000000..802fd67
--- /dev/null
+++ b/docs/SystemDiagram.xml
@@ -0,0 +1 @@
+7Zndc5s4EMD/Gj8mwzfmMXbi9mYuN5lJp20eZZBBqUCMELHdv74SrPiMEy7nJj5P/ZCglbSstD9WuzCzl+nuE0d5cssiTGeWEe1m9vXMsnzXlX+VYF8LAieoBTEnUS0yW8E9+YlBaIC0JBEuegMFY1SQvC8MWZbhUPRkiHO27Q/bMNq/a45iPBLch4iOpd9IJBKQml7QdnzGJE7g1nPLrzvWKPwRc1ZmcL+ZZW+qX92dIq0LFlokKGLbjsi+mdlLzpior9LdElO1tXrb6nmrA72N3RxnYtIE3wFXPSFaYm10ZZrY6+3AkdwdaGYsk/8W1Rqx0mLI1mOZ5oMBiUipbJnyslmkGkrRGtNFs01LRhlvZxUCcXGlPDiQrQiloAFnkR4RUlQUJKyFMETd8RELsQesUCmYFDEuEhazDNG/Gcu1ZYKzH1jbIJ1lrdyFv2x6tPNtKdmwTKxQSqhi+jOmT1iQEEEH3Mv0oN1VaTuOe61U5igkWSylrjJ4R8R3MENdP6jVXVY9meD7781iZaPtG/sXXF6wkofgrv1ft/Ov1MAm8x6v6UNZZHf0wvTnwJzczhiLxv+AtPJxRyPg8gmzFEsL5ACOKRLkqf+EIHjQ4mZcC5u8AN4OsuedLHstWV38zLPBbxJILR19ZnzzA5nx/zDzf2TGMz6QmfnJMnPeZ1xzkrUH24NeIZx/0AUnoDkdM9ceY/bKyXc89mDqHSPSRsuAvFcmvvUUSHtt3+irqFcAswYEN2ZMhToYQf0FUxxzlErpgokR4i3Papu3CRH4XvpL9W5lGv8fgO4S1hP24JLsPofhW6FbVT8pR5TEmXpSJDCY64mwbnNEJeIhaPVlK8ecyH3H/L4ZZL0E4RPmAu9epAh6ncC/HPAwd7Vk2ykrXKAk6VQUQ3LeFvj0M9Jh5I4rRQkuixPnA89dz7Gm8uH8Gz76PAxpeS8+LMPo0WHpI/I1NryjsKG1dNi4RgKtUYFPnIyF5XmVY8+VjHHkcE1jauQ4Eh3OiI5/WCTJMG52ucwz5G6dNiQrY+4b7hlDMgwfphW8LyK6Cnw+AVkmSFTv91iZnzgqpoXQ+pxR8e0+KioPcYLO7/2wOZCh++Nk9k+F9oEVmn4rOa7QjAbJQTX1xpJNHzVHL8YcbwC97U0qxl5V1Ly814rq5RyjqjtUv44jbRHKOKmOY6WdhONkXoYG0X8AOC7IT7SuBigv5srgagnuYlaxMQpfKsRIvOgVdKQkitT8Q89P+7lDQzIKUs1HG7Bk1mxnF58Xw8TBOHdhXJrBPOh56wISmbfSpNWYA6ezzabAv8vf49T8isqNvEUZik8+8zK9IIgmx6qPOU6fwXIUvSZnXnbwO5Nz2Wy/B9aMtd9c7Ztf
\ No newline at end of file
diff --git a/docs/TelegramBotforAlerting.docx b/docs/TelegramBotforAlerting.docx
new file mode 100644
index 0000000..4fbbe5c
Binary files /dev/null and b/docs/TelegramBotforAlerting.docx differ
diff --git a/go.mod b/go.mod
index 5c0017a..a47e941 100644
--- a/go.mod
+++ b/go.mod
@@ -1,4 +1,4 @@
-module github.com/metalmatze/alertmanager-bot
+module github.com/vu-long/alertmanager-bot
require (
cloud.google.com/go v0.34.0 // indirect
diff --git a/pkg/telegram/bot.go b/pkg/telegram/bot.go
index e578f65..6a51a57 100644
--- a/pkg/telegram/bot.go
+++ b/pkg/telegram/bot.go
@@ -11,13 +11,13 @@ import (
"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/hako/durafmt"
- "github.com/metalmatze/alertmanager-bot/pkg/alertmanager"
"github.com/oklog/run"
"github.com/prometheus/alertmanager/notify"
"github.com/prometheus/alertmanager/template"
"github.com/prometheus/alertmanager/types"
"github.com/prometheus/client_golang/prometheus"
"github.com/tucnak/telebot"
+ "github.com/vu-long/alertmanager-bot/pkg/alertmanager"
)
const (
diff --git a/services/.gitattributes b/services/.gitattributes
new file mode 100644
index 0000000..1ff0c42
--- /dev/null
+++ b/services/.gitattributes
@@ -0,0 +1,63 @@
+###############################################################################
+# Set default behavior to automatically normalize line endings.
+###############################################################################
+* text=auto
+
+###############################################################################
+# Set default behavior for command prompt diff.
+#
+# This is need for earlier builds of msysgit that does not have it on by
+# default for csharp files.
+# Note: This is only used by command line
+###############################################################################
+#*.cs diff=csharp
+
+###############################################################################
+# Set the merge driver for project and solution files
+#
+# Merging from the command prompt will add diff markers to the files if there
+# are conflicts (Merging from VS is not affected by the settings below, in VS
+# the diff markers are never inserted). Diff markers may cause the following
+# file extensions to fail to load in VS. An alternative would be to treat
+# these files as binary and thus will always conflict and require user
+# intervention with every merge. To do so, just uncomment the entries below
+###############################################################################
+#*.sln merge=binary
+#*.csproj merge=binary
+#*.vbproj merge=binary
+#*.vcxproj merge=binary
+#*.vcproj merge=binary
+#*.dbproj merge=binary
+#*.fsproj merge=binary
+#*.lsproj merge=binary
+#*.wixproj merge=binary
+#*.modelproj merge=binary
+#*.sqlproj merge=binary
+#*.wwaproj merge=binary
+
+###############################################################################
+# behavior for image files
+#
+# image files are treated as binary by default.
+###############################################################################
+#*.jpg binary
+#*.png binary
+#*.gif binary
+
+###############################################################################
+# diff behavior for common document formats
+#
+# Convert binary document formats to text before diffing them. This feature
+# is only available from the command line. Turn it on by uncommenting the
+# entries below.
+###############################################################################
+#*.doc diff=astextplain
+#*.DOC diff=astextplain
+#*.docx diff=astextplain
+#*.DOCX diff=astextplain
+#*.dot diff=astextplain
+#*.DOT diff=astextplain
+#*.pdf diff=astextplain
+#*.PDF diff=astextplain
+#*.rtf diff=astextplain
+#*.RTF diff=astextplain
diff --git a/services/.gitignore b/services/.gitignore
new file mode 100644
index 0000000..2f6da49
--- /dev/null
+++ b/services/.gitignore
@@ -0,0 +1,5 @@
+*DS_Store
+.vs/
+.vscode/
+.idea/
+alertmanager/config-email.yml
diff --git a/services/README.md b/services/README.md
new file mode 100644
index 0000000..e539f10
--- /dev/null
+++ b/services/README.md
@@ -0,0 +1,11 @@
+## Prometheus Monitoring
+
+This repository contains minimal Prometheus Server, NodeExporter, BlackBoxExporter, AlertManager and Grafana implementation for monitoring various services. You can use this repository to monitor a bare-metal Linux instance or to monitor Apache, NGINX or other HTTP based services using Prometheus.
+
+## Monitoring a Bare-Metal Linux Server
+
+To monitor a stand-alone Linux Server, you have to checkout against the tag v1.0 of the repository. Where all the configurations for monitoring a stand-alone Linux Server are available. Just `docker-compose up -d` and you're good to go. (You have to map alerts manually against tag v1.0)
+
+## Monitoring HTTP-based Web Services
+
+The v1.1 tag of the repository monitors 2 HTTP-based Web Services by default: An Apache httpd server and NGINX server both running in Docker Containers. If either or both of them goes down, an Prometheus will fire alerts in the form emails specified in the `config.yml` file in the AlertManager folder.
\ No newline at end of file
diff --git a/services/alertmanager/config.yml b/services/alertmanager/config.yml
new file mode 100644
index 0000000..f4b9df5
--- /dev/null
+++ b/services/alertmanager/config.yml
@@ -0,0 +1,30 @@
+route:
+ repeat_interval: 2h
+ receiver: email-1
+ routes:
+ - match:
+ alertname: httpd_down
+ receiver: email-1
+
+ - match:
+ alertname: nginx_down
+ receiver: email-2
+
+receivers:
+- name: email-1
+ email_configs:
+ - to:
+ from:
+ smarthost:
+ auth_username: ""
+ auth_identity: ""
+ auth_password: ""
+
+- name: email-2
+ email_configs:
+ - to:
+ from:
+ smarthost:
+ auth_username: ""
+ auth_identity: ""
+ auth_password: ""
diff --git a/services/alertmanager/data/nflog b/services/alertmanager/data/nflog
new file mode 100644
index 0000000..e69de29
diff --git a/services/alertmanager/data/silences b/services/alertmanager/data/silences
new file mode 100644
index 0000000..e69de29
diff --git a/services/alertmanager/webhookconfig.yml b/services/alertmanager/webhookconfig.yml
new file mode 100644
index 0000000..51d8e5f
--- /dev/null
+++ b/services/alertmanager/webhookconfig.yml
@@ -0,0 +1,18 @@
+route:
+ repeat_interval: 2h
+ receiver: group_1
+ routes:
+ - match:
+ alertname: httpd_down
+ receiver: group_1
+
+ - match:
+ alertname: nginx_down
+ receiver: group_1
+
+receivers:
+- name: group_1
+ webhook_configs:
+ - url: 'http://alertmanager-bot:8080'
+ send_resolved: true
+
diff --git a/services/docker-compose.yml b/services/docker-compose.yml
new file mode 100644
index 0000000..6374404
--- /dev/null
+++ b/services/docker-compose.yml
@@ -0,0 +1,111 @@
+version: '3'
+
+networks:
+ monitor-net:
+ driver: bridge
+
+volumes:
+ prometheus_data: {}
+ grafana_data: {}
+
+services:
+ prometheus:
+ image: prom/prometheus:latest
+ container_name: prometheus
+ volumes:
+ - ./prometheus/:/etc/prometheus/
+ - prometheus_data:/prometheus
+ command:
+ - '--config.file=/etc/prometheus/prometheus.yml'
+ - '--storage.tsdb.path=/prometheus'
+ - '--web.console.libraries=/etc/prometheus/console_libraries'
+ - '--web.console.templates=/etc/prometheus/consoles'
+ - '--storage.tsdb.retention=200h'
+ - '--web.enable-lifecycle'
+ restart: unless-stopped
+ expose:
+ - 9090
+ ports:
+ - "9090:9090"
+ networks:
+ - monitor-net
+
+ # nodeexporter:
+ # image: prom/node-exporter:latest
+ # container_name: nodeexporter
+ # user: root
+ # privileged: true
+ # volumes:
+ # - /proc:/host/proc:ro
+ # - /sys:/host/sys:ro
+ # - /:/rootfs:ro
+ # command:
+ # - '--path.procfs=/host/proc'
+ # - '--path.sysfs=/host/sys'
+ # - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)'
+ # restart: unless-stopped
+ # expose:
+ # - 9100
+ # networks:
+ # - monitor-net
+
+ alertmanager:
+ image: prom/alertmanager
+ container_name: alertmanager
+ volumes:
+ - ./alertmanager/:/etc/alertmanager/
+ command:
+ - '--config.file=/etc/alertmanager/webhookconfig.yml'
+ - '--storage.path=/alertmanager'
+ - '--web.listen-address=0.0.0.0:9093'
+ restart: unless-stopped
+ ports:
+ - "9093:9093"
+ networks:
+ - monitor-net
+
+ blackbox-exporter:
+ image: prom/blackbox-exporter
+ container_name: blackbox
+ restart: unless-stopped
+ ports:
+ - "9115:9115"
+ networks:
+ - monitor-net
+
+ grafana:
+ image: grafana/grafana:latest
+ container_name: grafana
+ volumes:
+ - grafana_data:/var/lib/grafana
+ - ./grafana/datasources:/etc/grafana/datasources
+ - ./grafana/dashboards:/etc/grafana/dashboards
+ - ./grafana/setup.sh:/setup.sh
+ entrypoint: /setup.sh
+ environment:
+ - GF_SECURITY_ADMIN_USER=${ADMIN_USER:-admin}
+ - GF_SECURITY_ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin}
+ - GF_USERS_ALLOW_SIGN_UP=false
+ restart: unless-stopped
+ expose:
+ - 3000
+ ports:
+ - 3000:3000
+ networks:
+ - monitor-net
+
+ httpd:
+ image: httpd
+ ports:
+ - "5000:80"
+ container_name: "httpd"
+ networks:
+ - monitor-net
+
+ nginx:
+ image: nginx
+ ports:
+ - "5001:80"
+ container_name: "nginx"
+ networks:
+ - monitor-net
diff --git a/services/grafana/dashboards/node-stat.json b/services/grafana/dashboards/node-stat.json
new file mode 100644
index 0000000..f825052
--- /dev/null
+++ b/services/grafana/dashboards/node-stat.json
@@ -0,0 +1,511 @@
+{
+ "__inputs": [
+ {
+ "name": "Prometheus",
+ "label": "",
+ "description": "",
+ "type": "datasource",
+ "pluginId": "prometheus",
+ "pluginName": "Prometheus"
+ }
+ ],
+ "__requires": [
+ {
+ "type": "panel",
+ "id": "graph",
+ "name": "Graph",
+ "version": ""
+ },
+ {
+ "type": "panel",
+ "id": "table",
+ "name": "Table",
+ "version": ""
+ },
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "3.1.1"
+ },
+ {
+ "type": "datasource",
+ "id": "prometheus",
+ "name": "Prometheus",
+ "version": "1.0.0"
+ }
+ ],
+ "id": null,
+ "title": "Host Stats - Prometheus Node Exporter",
+ "tags": [],
+ "style": "dark",
+ "timezone": "browser",
+ "editable": true,
+ "hideControls": false,
+ "sharedCrosshair": false,
+ "rows": [
+ {
+ "collapse": false,
+ "editable": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": "prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 1,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "span": 6,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "avg without (cpu)(irate(node_cpu{job=\"node\",instance=\"$instance\",mode!=\"idle\"}[5m]))",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{mode}}",
+ "metric": "node_cpu",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "CPU",
+ "tooltip": {
+ "shared": true,
+ "value_type": "individual",
+ "sort": 0,
+ "msResolution": false
+ },
+ "type": "graph",
+ "yaxes": [
+ {
+ "show": true,
+ "min": null,
+ "max": 1,
+ "logBase": 1,
+ "format": "percentunit"
+ },
+ {
+ "show": true,
+ "min": null,
+ "max": null,
+ "logBase": 1,
+ "format": "short"
+ }
+ ],
+ "xaxis": {
+ "show": true
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": "prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 2,
+ "isNew": true,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "minSpan": null,
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "repeat": null,
+ "seriesOverrides": [],
+ "span": 6,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "node_memory_MemTotal{job='node',instance='$instance'} - node_memory_MemFree{job='node',instance='$instance'} - node_memory_Buffers{job='node',instance='$instance'} - node_memory_Cached{job='node',instance='$instance'}",
+ "intervalFactor": 2,
+ "legendFormat": "Used",
+ "refId": "A",
+ "step": 60
+ },
+ {
+ "expr": "node_memory_Buffers{job='node',instance='$instance'}",
+ "intervalFactor": 2,
+ "legendFormat": "Buffers",
+ "refId": "B",
+ "step": 60
+ },
+ {
+ "expr": "node_memory_Cached{job='node',instance='$instance'}",
+ "intervalFactor": 2,
+ "legendFormat": "Cached",
+ "refId": "D",
+ "step": 60
+ },
+ {
+ "expr": "node_memory_MemFree{job='node',instance='$instance'}",
+ "hide": false,
+ "intervalFactor": 2,
+ "legendFormat": "Free",
+ "refId": "C",
+ "step": 60
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Memory",
+ "tooltip": {
+ "shared": true,
+ "value_type": "individual",
+ "sort": 0,
+ "msResolution": false
+ },
+ "type": "graph",
+ "yaxes": [
+ {
+ "show": true,
+ "min": 0,
+ "max": null,
+ "logBase": 1,
+ "format": "bytes",
+ "label": ""
+ },
+ {
+ "show": true,
+ "min": null,
+ "max": null,
+ "logBase": 1,
+ "format": "short"
+ }
+ ],
+ "xaxis": {
+ "show": true
+ }
+ }
+ ],
+ "title": "Row"
+ },
+ {
+ "collapse": false,
+ "editable": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": "prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 3,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "irate(node_disk_io_time_ms{job='node',instance='$instance',device!~'^(md\\\\d+$|dm-)'}[5m]) / 1000",
+ "intervalFactor": 2,
+ "legendFormat": "{{device}}",
+ "refId": "A",
+ "step": 120
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Disk I/O Utilisation",
+ "tooltip": {
+ "shared": true,
+ "value_type": "cumulative",
+ "sort": 0,
+ "msResolution": false
+ },
+ "type": "graph",
+ "yaxes": [
+ {
+ "show": true,
+ "min": null,
+ "max": 1,
+ "logBase": 1,
+ "format": "percentunit"
+ },
+ {
+ "show": true,
+ "min": null,
+ "max": null,
+ "logBase": 1,
+ "format": "short"
+ }
+ ],
+ "xaxis": {
+ "show": true
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": "prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 4,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "1 - node_filesystem_free{job='node',instance='$instance',fstype!='rootfs',mountpoint!~'/(run|var).*',mountpoint!=''} / node_filesystem_size{job='node',instance='$instance'}",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{mountpoint}}",
+ "refId": "A",
+ "step": 120
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Filesystem Fullness",
+ "tooltip": {
+ "shared": true,
+ "value_type": "cumulative",
+ "sort": 0,
+ "msResolution": false
+ },
+ "type": "graph",
+ "yaxes": [
+ {
+ "show": true,
+ "min": 0,
+ "max": 1,
+ "logBase": 1,
+ "format": "percentunit"
+ },
+ {
+ "show": true,
+ "min": null,
+ "max": null,
+ "logBase": 1,
+ "format": "short"
+ }
+ ],
+ "xaxis": {
+ "show": true
+ }
+ },
+ {
+ "columns": [
+ {
+ "text": "Current",
+ "value": "current"
+ }
+ ],
+ "editable": true,
+ "error": false,
+ "fontSize": "100%",
+ "hideTimeOverride": true,
+ "id": 5,
+ "isNew": true,
+ "links": [],
+ "pageSize": null,
+ "scroll": true,
+ "showHeader": true,
+ "sort": {
+ "col": 0,
+ "desc": true
+ },
+ "span": 4,
+ "styles": [
+ {
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "Time",
+ "type": "date"
+ },
+ {
+ "colorMode": null,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "decimals": 0,
+ "pattern": "/.*/",
+ "thresholds": [],
+ "type": "number",
+ "unit": "s"
+ }
+ ],
+ "targets": [
+ {
+ "expr": "(node_filesystem_size{job='node',instance='$instance'} - node_filesystem_free{job='node',instance='$instance'}) / deriv(node_filesystem_free{job='node',instance='$instance',fstype!='rootfs',mountpoint!~'/(run|var).*',mountpoint!=''}[3d]) > 0",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{mountpoint}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "timeFrom": "1h",
+ "timeShift": null,
+ "title": "Filesystem Fill Up Time",
+ "transform": "timeseries_aggregations",
+ "type": "table"
+ }
+ ],
+ "title": "New row"
+ }
+ ],
+ "time": {
+ "from": "now-6h",
+ "to": "now"
+ },
+ "timepicker": {
+ "now": true,
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "templating": {
+ "list": [
+ {
+ "allFormat": "glob",
+ "current": {},
+ "datasource": "prometheus",
+ "hideLabel": false,
+ "includeAll": false,
+ "label": "Machine",
+ "multi": false,
+ "multiFormat": "glob",
+ "name": "instance",
+ "options": [],
+ "query": "up{job=\"node\"}",
+ "refresh": 1,
+ "regex": ".*instance=\"(.*?)\".*",
+ "type": "query",
+ "hide": 0
+ }
+ ]
+ },
+ "annotations": {
+ "list": []
+ },
+ "schemaVersion": 12,
+ "version": 12,
+ "links": [],
+ "gnetId": 718,
+ "description": "Basic host stats: CPU, Memory Usage, Disk Utilisation, Filesystem usage and Predicted time to filesystems filling"
+}
diff --git a/services/grafana/datasources/Prometheus.json b/services/grafana/datasources/Prometheus.json
new file mode 100644
index 0000000..1f19f1a
--- /dev/null
+++ b/services/grafana/datasources/Prometheus.json
@@ -0,0 +1,7 @@
+{
+ "name":"prometheus",
+ "type":"prometheus",
+ "url":"http://prometheus:9090",
+ "access":"proxy",
+ "basicAuth":false
+ }
diff --git a/services/grafana/setup.sh b/services/grafana/setup.sh
new file mode 100755
index 0000000..03abc0a
--- /dev/null
+++ b/services/grafana/setup.sh
@@ -0,0 +1,85 @@
+#!/bin/bash
+
+# Taken from https://github.com/grafana/grafana-docker/issues/74
+
+# Script to configure grafana datasources and dashboards.
+# Intended to be run before grafana entrypoint...
+# Image: grafana/grafana:4.1.2
+# ENTRYPOINT [\"/run.sh\"]"
+
+GRAFANA_URL=${GRAFANA_URL:-http://$GF_SECURITY_ADMIN_USER:$GF_SECURITY_ADMIN_PASSWORD@localhost:3000}
+#GRAFANA_URL=http://grafana-plain.k8s.playground1.aws.ad.zopa.com
+DATASOURCES_PATH=${DATASOURCES_PATH:-/etc/grafana/datasources}
+DASHBOARDS_PATH=${DASHBOARDS_PATH:-/etc/grafana/dashboards}
+
+# Generic function to call the Vault API
+grafana_api() {
+ local verb=$1
+ local url=$2
+ local params=$3
+ local bodyfile=$4
+ local response
+ local cmd
+
+ cmd="curl -L -s --fail -H \"Accept: application/json\" -H \"Content-Type: application/json\" -X ${verb} -k ${GRAFANA_URL}${url}"
+ [[ -n "${params}" ]] && cmd="${cmd} -d \"${params}\""
+ [[ -n "${bodyfile}" ]] && cmd="${cmd} --data @${bodyfile}"
+ echo "Running ${cmd}"
+ eval ${cmd} || return 1
+ return 0
+}
+
+wait_for_api() {
+ while ! grafana_api GET /api/user/preferences
+ do
+ sleep 5
+ done
+}
+
+install_datasources() {
+ local datasource
+
+ for datasource in ${DATASOURCES_PATH}/*.json
+ do
+ if [[ -f "${datasource}" ]]; then
+ echo "Installing datasource ${datasource}"
+ if grafana_api POST /api/datasources "" "${datasource}"; then
+ echo "installed ok"
+ else
+ echo "install failed"
+ fi
+ fi
+ done
+}
+
+install_dashboards() {
+ local dashboard
+
+ for dashboard in ${DASHBOARDS_PATH}/*.json
+ do
+ if [[ -f "${dashboard}" ]]; then
+ echo "Installing dashboard ${dashboard}"
+
+ echo "{\"dashboard\": `cat $dashboard`}" > "${dashboard}.wrapped"
+
+ if grafana_api POST /api/dashboards/db "" "${dashboard}.wrapped"; then
+ echo "installed ok"
+ else
+ echo "install failed"
+ fi
+
+ rm "${dashboard}.wrapped"
+ fi
+ done
+}
+
+configure_grafana() {
+ wait_for_api
+ install_datasources
+ install_dashboards
+}
+
+echo "Running configure_grafana in the background..."
+configure_grafana &
+/run.sh
+exit 0
\ No newline at end of file
diff --git a/services/prometheus/alert.rules b/services/prometheus/alert.rules
new file mode 100644
index 0000000..025c1d3
--- /dev/null
+++ b/services/prometheus/alert.rules
@@ -0,0 +1,21 @@
+groups:
+
+- name: httpd
+ rules:
+ - alert: httpd_down
+ expr: probe_success{instance="http://httpd:80",job="httpd"} == 0
+ for: 1s
+ labels:
+ severity: critical
+ annotations:
+ summary: "httpd is down"
+
+- name: nginx
+ rules:
+ - alert: nginx_down
+ expr: probe_success{instance="http://nginx:80",job="nginx"} == 0
+ for: 1s
+ labels:
+ severity: warning
+ annotations:
+ summary: "nginx is down"
diff --git a/services/prometheus/prometheus.yml b/services/prometheus/prometheus.yml
new file mode 100644
index 0000000..ada5d71
--- /dev/null
+++ b/services/prometheus/prometheus.yml
@@ -0,0 +1,57 @@
+global:
+ scrape_interval: 5s
+ evaluation_interval: 5s
+
+ external_labels:
+ monitor: 'prometheus-grafana-exporter'
+
+# Alertmanager configuration
+alerting:
+ alertmanagers:
+ - static_configs:
+ - targets:
+ - alertmanager:9093
+
+rule_files:
+ - "alert.rules"
+
+scrape_configs:
+ # - job_name: "kjanshair"
+ # scrape_interval: 5s
+ # static_configs:
+ # - targets: ['nodeexporter:9100']
+
+ - job_name: 'prometheus'
+ scrape_interval: 10s
+ static_configs:
+ - targets: ['localhost:9090']
+
+ - job_name: 'httpd'
+ metrics_path: /probe
+ params:
+ module: [http_2xx]
+ static_configs:
+ - targets:
+ - http://httpd:80
+ relabel_configs:
+ - source_labels: [__address__]
+ target_label: __param_target
+ - source_labels: [__param_target]
+ target_label: instance
+ - target_label: __address__
+ replacement: blackbox:9115
+
+ - job_name: 'nginx'
+ metrics_path: /probe
+ params:
+ module: [http_2xx]
+ static_configs:
+ - targets:
+ - http://nginx:80
+ relabel_configs:
+ - source_labels: [__address__]
+ target_label: __param_target
+ - source_labels: [__param_target]
+ target_label: instance
+ - target_label: __address__
+ replacement: blackbox:9115